Source code for pyathena.filesystem

# Copyright 2026 The PyAthena authors
#
# Licensed under the MIT License.
# See LICENSE or https://opensource.org/licenses/MIT.
#
# SPDX-License-Identifier: MIT

"""fsspec filesystem implementations for Amazon S3."""

from __future__ import annotations

import logging
from typing import TYPE_CHECKING, Any, TypeVar, overload

import fsspec
from fsspec import AbstractFileSystem

from pyathena.filesystem.s3 import S3FileSystem

if TYPE_CHECKING:
    from pyathena.connection import Connection

_logger = logging.getLogger(__name__)

_FileSystemT = TypeVar("_FileSystemT", bound=AbstractFileSystem)


@overload
def s3_filesystem(connection: Connection[Any], **kwargs: Any) -> S3FileSystem: ...


@overload
def s3_filesystem(
    connection: Connection[Any], filesystem_class: type[_FileSystemT], **kwargs: Any
) -> _FileSystemT: ...


[docs] def s3_filesystem( connection: Connection[Any], filesystem_class: type[AbstractFileSystem] = S3FileSystem, **kwargs: Any, ) -> AbstractFileSystem: """Create an S3 filesystem with the S3 client and retry policy of a connection. The filesystem borrows ``connection.s3_client``, which the connection shares with its result sets, and uses ``connection.retry_config``. A policy with the default ``exceptions`` also retries the transient S3 error codes (:attr:`~pyathena.filesystem.s3_core.S3Core.RETRY_ERROR_CODES`). Closing the filesystem does not close the client. A filesystem created this way cannot be pickled; see :class:`~pyathena.filesystem.s3.S3FileSystem`. Example: >>> from pyathena import connect >>> from pyathena.filesystem import s3_filesystem >>> fs = s3_filesystem(connect(s3_staging_dir="s3://my-bucket/path/")) Args: connection: The PyAthena connection. filesystem_class: ``S3FileSystem``, ``AioS3FileSystem``, or a subclass of either. **kwargs: The other arguments of the filesystem, except the arguments from which it builds a client. Returns: The filesystem. """ return filesystem_class( client=connection.s3_client, retry_config=connection.retry_config, **kwargs )
def register_s3_filesystem() -> None: """Register PyAthena's S3 filesystem as fsspec's "s3" / "s3a" protocols. PyAthena registers its own filesystem so that the pandas/polars result sets can read query results from S3 without depending on s3fs. The registration replaces fsspec's default lazy mapping of the "s3" protocol to s3fs, which means ``fsspec.filesystem("s3")`` returns PyAthena's implementation and s3fs-specific settings (e.g., the ``S3FS_LOGGING_LEVEL`` environment variable) have no effect. The registered class is synchronous (``async_impl`` is False): like fsspec's other synchronous filesystems, it accepts and ignores ``asynchronous=True``. ``AioS3FileSystem`` is the asynchronous filesystem. A filesystem class that has already been registered explicitly is also overwritten, with a warning log. To restore another implementation, re-register it after importing ``pyathena.pandas`` / ``pyathena.polars``:: fsspec.register_implementation("s3", s3fs.S3FileSystem, clobber=True) """ for protocol in ("s3", "s3a"): registered = fsspec.registry.get(protocol) if registered is not None and registered is not S3FileSystem: _logger.warning( f"The fsspec {protocol!r} protocol is already registered as " f"{registered.__module__}.{registered.__qualname__} and will be overwritten by " f"{S3FileSystem.__module__}.{S3FileSystem.__qualname__}." ) _logger.debug(f"Registering {S3FileSystem} as the fsspec {protocol!r} protocol.") fsspec.register_implementation(protocol, S3FileSystem, clobber=True)