Source code for pyathena.filesystem.s3

"""fsspec filesystem and file implementations for Amazon S3."""

from __future__ import annotations

import _thread
import atexit
import contextlib
import contextvars
import functools
import gc
import hashlib
import logging
import mimetypes
import os.path
import sys
import threading
import time
import warnings
from collections.abc import Callable, Generator, Iterable, Mapping
from copy import deepcopy
from dataclasses import dataclass, field
from datetime import datetime
from glob import has_magic
from io import BytesIO
from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, cast
from urllib.parse import unquote_plus

import fsspec
from boto3 import Session
from botocore import UNSIGNED
from botocore.client import BaseClient, Config
from fsspec import AbstractFileSystem
from fsspec.callbacks import _DEFAULT_CALLBACK, Callback
from fsspec.compression import compr
from fsspec.config import apply_config
from fsspec.core import get_compression
from fsspec.implementations.local import LocalFileSystem, make_path_posix, trailing_sep
from fsspec.spec import AbstractBufferedFile
from fsspec.utils import check_contained, isfilelike, other_paths, stringify_path, tokenize

import pyathena
from pyathena.filesystem.s3_core import S3Core, S3DeleteBatch, S3DeleteResult
from pyathena.filesystem.s3_errors import S3ClientError
from pyathena.filesystem.s3_executor import (
    ItemT,
    S3Executor,
    S3MapResult,
    S3ThreadPool,
    T,
    _S3PoolScope,
)
from pyathena.filesystem.s3_object import (
    S3Metadata,
    S3MultipartUpload,
    S3MultipartUploadPart,
    S3Object,
    S3ObjectType,
    S3ObjectVersion,
    S3StorageClass,
)
from pyathena.filesystem.s3_path import S3Path
from pyathena.filesystem.s3_path_pairing import S3PathPairing
from pyathena.filesystem.s3_writer import _S3WriteSession
from pyathena.util import RetryConfig, override, raise_if_stopped, unstoppable_retries

if TYPE_CHECKING:
    from pyathena.connection import Connection

_logger = logging.getLogger(__name__)

# The message of the TypeError that a filesystem with an injected client, and
# the files that it opens for reading, raise when they are pickled.
_UNPICKLABLE_CLIENT_MESSAGE = (
    "{cls} built with client= or connection= cannot be pickled, as its boto3 client "
    "cannot be. To pickle it, for example to send it to Dask, Ray, or multiprocessing "
    "workers, build it from s3fs-style arguments such as profile=, region_name=, or "
    "key=, secret=, and token=."
)
# The DeprecationWarning of the connection argument of the filesystems.
_CONNECTION_DEPRECATION_MESSAGE = (
    "The connection argument of {cls} is deprecated and will be removed in the next "
    "major version. Use pyathena.filesystem.s3_filesystem(connection), or pass "
    "client=connection.s3_client and retry_config=connection.retry_config."
)

# Whether the cyclic garbage collector runs in a thread, and the closes of
# files that it handed over (S3File._track_collection).
_collecting = threading.local()

# The ends of the closes that run in threads of the _thread module, which an
# exit waits for (S3File._wait_for_handed_over_closes), and whether it has.
_handed_over_closes: set[threading.Event] = set()
_handed_over_closes_waited = False

# The request parameters on which the authorization of a lookup (HeadObject,
# HeadBucket, or ListObjectsV2) depends.
_LOOKUP_REQUEST_PARAMETERS = frozenset(
    {
        "ExpectedBucketOwner",
        "RequestPayer",
        "SSECustomerAlgorithm",
        "SSECustomerKey",
        "SSECustomerKeyMD5",
    }
)
# The second element of the dircache key, ``(path, _LOOKUPS_CACHE_KEY)``, of
# the lookup results of a path made with lookup request parameters.
_LOOKUPS_CACHE_KEY = "lookups"
# The second element of the dircache key,
# ``(name, _VERSION_CACHE_KEY, version_id)``, of the lookup of an explicit
# version, which no key of an unversioned path, a name string, can equal.
_VERSION_CACHE_KEY = "versionId"


class CompressedBuffer(BytesIO):
    """An in-memory buffer of data compressed with a codec of fsspec.

    The buffer keeps its data when it is closed, as some codecs, such as
    the ``zstandard`` stream writer, close the file that they write to.
    """

    @override
    def close(self) -> None:
        """Do nothing, so that the data can still be read after a codec closes the buffer."""

    @classmethod
    def compress(cls, value: bytes | bytearray | memoryview, compression: str) -> bytes:
        """Compress a value with a codec of fsspec.

        Args:
            value: The bytes to compress.
            compression: Name of a codec in ``fsspec.compression.compr``.

        Returns:
            The compressed bytes.

        Raises:
            ValueError: If the codec is not supported.
        """
        if compression not in compr:
            raise ValueError(f"Compression type {compression} not supported")
        if isinstance(value, memoryview) and not value.c_contiguous:
            # Codecs cannot compress a non-contiguous memoryview.
            value = value.tobytes()
        buffer = cls()
        with compr[compression](buffer, mode="w") as f:
            f.write(value)
        return buffer.getvalue()


class _S3LockedCallback(Callback):
    """A progress callback whose uses, and those of its branches, are serialized.

    The per-object calls of a batch operation run in parallel threads and
    update the callback of the operation and the branches that it creates.
    fsspec does not require a callback to be thread-safe, nor its branches to
    be independent of it, so every use goes through one lock.

    Args:
        callback: The callback of the operation, or a branch of it.
        lock: The lock of the operation.
    """

    def __init__(self, callback: Callback, lock: threading.Lock) -> None:
        super().__init__()
        self._callback = callback
        self._lock = lock

    def __enter__(self) -> _S3LockedCallback:
        """Enter the callback.

        Returns:
            This callback, or a proxy of the callback that the wrapped one
            returned from ``__enter__()``, if that is another one.
        """
        with self._lock:
            entered = self._callback.__enter__()
        if entered is self._callback:
            return self
        return _S3LockedCallback(entered, self._lock)

    def __exit__(self, *exc_args: Any) -> Any:
        """Exit the callback.

        Args:
            *exc_args: The exception of the block, if any.

        Returns:
            What the callback's ``__exit__()`` returns.
        """
        with self._lock:
            return self._callback.__exit__(*exc_args)

    def close(self) -> None:
        """Close the callback."""
        with self._lock:
            self._callback.close()

    def branched(self, path_1, path_2, **kwargs) -> Callback:
        """Return a branch of the callback for the transfer of one file.

        Args:
            path_1: The source of the transfer.
            path_2: The destination of the transfer.
            **kwargs: Passed to the callback's ``branched()``.

        Returns:
            The branch, whose uses take the lock of this callback.
        """
        with self._lock:
            return _S3LockedCallback(self._callback.branched(path_1, path_2, **kwargs), self._lock)

    def set_size(self, size) -> None:
        """Set the size of the callback.

        Args:
            size: The total.
        """
        with self._lock:
            self._callback.set_size(size)

    def absolute_update(self, value) -> None:
        """Set the progress of the callback.

        Args:
            value: The progress.
        """
        with self._lock:
            self._callback.absolute_update(value)

    def relative_update(self, inc=1) -> None:
        """Advance the progress of the callback.

        Args:
            inc: The increment.
        """
        with self._lock:
            self._callback.relative_update(inc)

    def call(self, hook_name=None, **kwargs) -> Any:
        """Run the hooks of the callback.

        Args:
            hook_name: The hook to run, or None for all of them.
            **kwargs: Passed to the hooks.

        Returns:
            What the callback's ``call()`` returns.
        """
        with self._lock:
            return self._callback.call(hook_name, **kwargs)


class _S3TreeListing:
    """The listings of one ``tree()`` of an ``S3FileSystem``.

    fsspec's ``tree()`` lists each directory with ``ls()`` of its name, which
    a version ID query or an empty segment at the end of the name changes.
    This object runs fsspec's ``tree()`` in place of the filesystem: a
    directory that a listing of the same ``tree()`` returned is listed by
    its key prefix (see :meth:`S3FileSystem._directory_prefix`), and any
    other path with ``ls()``.
    """

    def __init__(self, fs: S3FileSystem) -> None:
        """Initialize the listings with no listed directory.

        Args:
            fs: The filesystem to list.
        """
        self._fs = fs
        self._directories: dict[str, S3Object] = {}

    def ls(self, path: str, detail: bool = True) -> list[S3Object]:
        """List a path as ``S3FileSystem.ls(detail=True)`` does.

        Args:
            path: The path, or the name of a directory listed before.
            detail: Unused; the entries are always returned, as fsspec's
                ``tree()`` asks for them.

        Returns:
            A new list of the entries, which fsspec's ``tree()`` sorts in place.
        """
        directory = self._directories.get(path)
        if directory is None:
            files = cast(list[S3Object], self._fs.ls(path, detail=True))
        else:
            files = self._fs._ls_directory(directory)
        for f in files:
            if f.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY:
                self._directories[f.name] = f
        return files

    def tree(self, **kwargs: Any) -> str:
        """Run fsspec's ``tree()`` with these listings.

        Args:
            **kwargs: The arguments of ``tree()``.

        Returns:
            The tree as a string.
        """
        return cast(str, AbstractFileSystem.tree(self, **kwargs))


[docs] class S3FileSystem(AbstractFileSystem): """A filesystem interface for Amazon S3 that implements the fsspec protocol. This class provides a file-system like interface to Amazon S3, allowing you to use familiar file operations (ls, open, cp, rm, etc.) with S3 objects. It's designed to be compatible with s3fs while offering PyAthena-specific optimizations. The filesystem supports standard S3 operations including: - Listing objects and directories - Reading and writing files - Copying and moving objects - Reading and writing object metadata, tags, and canned ACLs - Multipart uploads for large files, including management of incomplete uploads - Version-aware reads and object version listing (see ``version_aware``) - Creating and removing buckets (disabled by default; see ``allow_bucket_creation`` / ``allow_bucket_deletion``) - Various S3 storage classes and encryption options - Translating S3 error responses into standard Python exceptions (e.g., ``404`` -> ``FileNotFoundError``, ``403`` -> ``PermissionError``) Attributes: allow_bucket_creation: Whether mkdir/makedirs may create buckets. Defaults to False. allow_bucket_deletion: Whether rmdir may delete buckets. Defaults to False. version_aware: Whether reads pin the object version observed at open time and ls may list all versions. Requires the s3:GetObjectVersion / s3:ListBucketVersions permissions. Defaults to False. Example: >>> from pyathena.filesystem.s3 import S3FileSystem >>> fs = S3FileSystem() >>> >>> # List objects in a bucket >>> files = fs.ls('s3://my-bucket/data/') >>> >>> # Read a file >>> with fs.open('s3://my-bucket/data/file.csv', 'r') as f: ... content = f.read() >>> >>> # Write a file >>> with fs.open('s3://my-bucket/output/result.txt', 'w') as f: ... f.write('Hello, S3!') >>> >>> # Copy files >>> fs.cp('s3://source-bucket/file.txt', 's3://dest-bucket/file.txt') Note: This filesystem is used internally by PyAthena for handling query results stored in S3, but can also be used independently for S3 file operations. """ DEFAULT_BLOCK_SIZE: int = 5 * 2**20 # 5MiB protocol = ("s3", "s3a") _extra_tokenize_attributes = ("default_block_size",) # The boto3 session arguments of the client that the filesystem builds. _SESSION_PASSING_ARGS: ClassVar[frozenset[str]] = frozenset( { "aws_access_key_id", "aws_secret_access_key", "aws_session_token", "region_name", "botocore_session", "profile_name", } ) # The arguments of ``Session.client()`` taken from ``client_kwargs``. _CLIENT_PASSING_ARGS: ClassVar[frozenset[str]] = frozenset( { "aws_access_key_id", "aws_secret_access_key", "aws_session_token", "api_version", "use_ssl", "verify", "endpoint_url", "region_name", "config", } ) # The arguments from which the filesystem builds its client, which an # injected client (``client`` or ``connection``) does not take. _CLIENT_BUILDING_ARGS: ClassVar[frozenset[str]] = _SESSION_PASSING_ARGS | { "key", "username", "secret", "password", "token", "profile", "anon", "use_ssl", "endpoint_url", "connect_timeout", "read_timeout", "client_kwargs", "config_kwargs", }
[docs] def __init__( self, connection: Connection[Any] | None = None, default_block_size: int | None = None, default_cache_type: str | None = None, max_workers: int | None = None, s3_additional_kwargs=None, allow_bucket_creation: bool = False, allow_bucket_deletion: bool = False, version_aware: bool = False, *args, batch_size: int | None = None, client: BaseClient | None = None, retry_config: RetryConfig | None = None, **kwargs, ) -> None: """Create a filesystem for Amazon S3. The filesystem sends its requests with the boto3 S3 client given as ``client``, or with a client that it builds from s3fs-compatible arguments in ``kwargs``. :func:`pyathena.filesystem.s3_filesystem` creates one with the S3 client and retry policy of a PyAthena connection. Args: connection: Deprecated; use ``client`` and ``retry_config``, or :func:`pyathena.filesystem.s3_filesystem`. A PyAthena connection whose S3 client (``Connection.s3_client``) and retry policy the filesystem uses, with a ``DeprecationWarning``. default_block_size: The block size for reads and writes; defaults to ``DEFAULT_BLOCK_SIZE``. default_cache_type: The fsspec cache type for reads, the name of a cache in ``fsspec.caching.caches``; defaults to ``"bytes"``. The available names depend on the installed fsspec version. Common ones are ``"none"``, which keeps nothing and requests the bytes of each read; ``"bytes"``, which reads one block beyond each read and keeps one contiguous range; ``"readahead"``, which reads one block beyond each read and keeps only the last range read; ``"blockcache"``, which requests whole blocks and keeps the most recently used ones; and ``"background"``, which is like ``"blockcache"`` and also reads the next block in a thread. max_workers: The number of threads of the request pool of the filesystem, which its files and parallel operations share, and the default number of parallel requests of a file. If None, the ``max_pool_connections`` of the S3 client (10 unless configured). A client that the filesystem builds gets ``max_pool_connections`` from ``max_workers`` unless ``config_kwargs`` sets it; the connection pool of a given ``client`` keeps its size. s3_additional_kwargs: Extra arguments for the object requests of ``open()``, ``pipe_file()`` and ``put_file()``. Their authorization parameters (``RequestPayer``, ``ExpectedBucketOwner`` and ``SSECustomer*``) are sent with every request, listings and copies included; a copy also sends them for its source (``ExpectedSourceBucketOwner`` and ``CopySourceSSECustomer*``). Each request receives those that its operation accepts, and the parameters of a call take precedence. allow_bucket_creation: Whether ``mkdir``/``makedirs`` may create a bucket. allow_bucket_deletion: Whether ``rmdir`` may delete a bucket. version_aware: Whether reads pin the object version observed at open time. *args: Passed to ``fsspec.AbstractFileSystem``. batch_size: The number of per-object calls of a batch operation (``cat``, ``get``, ``put``, ``copy``, and the like) that run at once, and of DeleteObjects requests of ``rm``, unless the call gives its own. None or -1 is ``max_workers``. client: The boto3 S3 client that the filesystem sends its requests with. The filesystem does not close it. A filesystem with a given client cannot be pickled. retry_config: The retry policy of the requests; defaults to ``RetryConfig()``. A policy with the default ``exceptions`` also retries the transient S3 error codes (:attr:`~pyathena.filesystem.s3_core.S3Core.RETRY_ERROR_CODES`). **kwargs: Passed to ``fsspec.AbstractFileSystem``; without ``client`` or ``connection``, also s3fs-compatible client arguments. ``requester_pays=True`` sends requester-pays requests with the operations that accept ``RequestPayer``. Raises: ValueError: If ``batch_size`` is neither positive, -1, nor None. TypeError: If ``connection`` is given with ``client`` or ``retry_config``, or ``client`` or ``connection`` with an argument from which the filesystem builds a client, such as ``key``, ``profile``, ``anon``, ``endpoint_url``, ``client_kwargs``, or ``region_name``, other than one that fsspec's configuration supplies. """ super().__init__(*args, **kwargs) self.batch_size = self._check_batch_size(batch_size) self._client_injected = client is not None or connection is not None client, retry_config = self._resolve_injected_client( type(self), connection, client, retry_config, kwargs ) if client is None: client = self._get_client_compatible_with_s3fs(max_workers=max_workers, **kwargs) self.default_block_size = ( default_block_size if default_block_size else self.DEFAULT_BLOCK_SIZE ) self.default_cache_type = default_cache_type if default_cache_type else "bytes" # The requests of the files and parallel operations of the # filesystem run in this pool, at most max_workers at once. self._request_pool = S3ThreadPool( max_workers if max_workers is not None else client.meta.config.max_pool_connections ) self.max_workers: int = self._request_pool.max_workers self.s3_additional_kwargs = s3_additional_kwargs if s3_additional_kwargs else {} self.allow_bucket_creation = allow_bucket_creation self.allow_bucket_deletion = allow_bucket_deletion self.version_aware = version_aware requester_pays = kwargs.pop("requester_pays", False) # The parameters that authorize the access to the objects, sent with # every request that accepts them, and with a copy for its source. request_kwargs = self._get_lookup_kwargs(self.s3_additional_kwargs) if requester_pays: request_kwargs.setdefault("RequestPayer", "requester") self._core = S3Core( client, retry_config=retry_config if retry_config is not None else RetryConfig(), request_kwargs={**request_kwargs, **S3Core.copy_from_source_params(request_kwargs)}, )
@classmethod def _resolve_injected_client( cls, filesystem_class: type[AbstractFileSystem], connection: Connection[Any] | None, client: BaseClient | None, retry_config: RetryConfig | None, kwargs: Mapping[str, Any], ) -> tuple[BaseClient | None, RetryConfig | None]: """Check the injected client of a filesystem and resolve it from a connection. Warns that ``connection`` is deprecated, and rejects the arguments that conflict with an injected client before the client of the connection is used. ``AioS3FileSystem`` resolves its arguments with it too. Args: filesystem_class: The class of the filesystem being created, whose name the warning shows and whose fsspec configuration applies. connection: The deprecated ``connection`` argument. client: The ``client`` argument. retry_config: The ``retry_config`` argument. kwargs: The other keyword arguments of the filesystem, merged with fsspec's configuration for the protocol. Returns: The injected client and retry policy, or the given ones if ``connection`` is None. Raises: TypeError: If the arguments conflict with an injected client. """ if connection is not None: message = _CONNECTION_DEPRECATION_MESSAGE.format(cls=filesystem_class.__name__) if sys.version_info >= (3, 12): # Attributed to the first caller outside fsspec and the filesystems, # such as through fsspec.filesystem() or a subclass. warnings.warn( message, DeprecationWarning, skip_file_prefixes=( os.path.dirname(fsspec.__file__) + os.sep, os.path.dirname(__file__) + os.sep, ), ) else: # Past this method, __init__, and fsspec's metaclass. warnings.warn(message, DeprecationWarning, stacklevel=4) if client is not None or retry_config is not None: raise TypeError("connection cannot be combined with client or retry_config.") if client is not None or connection is not None: # The settings of fsspec's configuration for the protocol, such as # the FSSPEC_S3_* environment variables, apply to every filesystem # and are ignored instead. configured = apply_config(filesystem_class, {}) building_args = sorted( k for k in cls._CLIENT_BUILDING_ARGS if kwargs.get(k) is not None and (k not in configured or kwargs[k] != configured[k]) ) if building_args: raise TypeError( f"{', '.join(building_args)} cannot be combined with client or " "connection, whose client the filesystem uses as it is." ) if connection is not None: return connection.s3_client, connection.retry_config return client, retry_config
[docs] @override def __reduce__(self) -> tuple[Any, ...]: """Reduce the filesystem for pickling, as fsspec does. Returns: The reduction of ``fsspec.AbstractFileSystem``, which creates the filesystem again from its arguments. Raises: TypeError: If the filesystem was built with ``client`` or ``connection``, whose boto3 client cannot be pickled. """ if self._client_injected: raise TypeError(_UNPICKLABLE_CLIENT_MESSAGE.format(cls=type(self).__name__)) return cast(tuple[Any, ...], super().__reduce__())
@property def core(self) -> S3Core: """The typed S3 operations that the filesystem sends its requests with.""" return self._core @property def _client(self) -> BaseClient: """The boto3 S3 client of the filesystem (``core.client``).""" return self._core.client @property def _retry_config(self) -> RetryConfig: """The retry policy of the requests (``core.retry_config``).""" return self._core.retry_config @property def request_kwargs(self) -> dict[str, Any]: """The parameters sent with every operation that accepts them. ``{"RequestPayer": "requester"}`` with ``requester_pays=True``, and the authorization parameters of ``s3_additional_kwargs``, also under the names of a copy source; see ``core.request_kwargs``. """ return self._core.request_kwargs
[docs] def close(self) -> None: """Release the threads of the request pool, which stays usable, once they are idle. See :meth:`S3ThreadPool.release() <pyathena.filesystem.s3_executor.S3ThreadPool.release>`. The calls in progress, including the requests of open files, run to their end, and a call after the release creates new threads. """ self._request_pool.release()
[docs] def __enter__(self) -> S3FileSystem: """Return the filesystem, whose threads the end of the block releases.""" return self
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: self.close() def _get_client_compatible_with_s3fs( self, max_workers: int | None = None, **kwargs ) -> BaseClient: """Build a boto3 S3 client from s3fs-compatible constructor arguments. Accepts the constructor arguments that s3fs users pass through fsspec storage options — ``key``/``username``, ``secret``/``password``, ``token``, ``profile``, ``anon``, ``use_ssl``, ``endpoint_url``, ``connect_timeout``/``read_timeout``, and the ``client_kwargs`` / ``config_kwargs`` dictionaries — in addition to boto3 session arguments such as ``region_name`` and ``profile_name``. ``profile`` is used as ``profile_name`` when ``profile_name`` is not given. Args: max_workers: The size of the request pool of the filesystem, which becomes the ``max_pool_connections`` of the client unless ``config_kwargs`` sets it; None keeps botocore's default. **kwargs: The filesystem constructor arguments. Returns: A boto3 S3 client configured from the arguments. """ config_kwargs = deepcopy(kwargs.pop("config_kwargs", {})) client_kwargs = deepcopy(kwargs.pop("client_kwargs", {})) if max_workers is not None: config_kwargs.setdefault("max_pool_connections", max_workers) user_agent_extra = config_kwargs.pop("user_agent_extra", None) if user_agent_extra and pyathena.user_agent_extra not in user_agent_extra: user_agent_extra = f"{pyathena.user_agent_extra} {user_agent_extra}" config_kwargs.update({"user_agent_extra": user_agent_extra or pyathena.user_agent_extra}) if connect_timeout := kwargs.pop("connect_timeout", None): config_kwargs.update({"connect_timeout": connect_timeout}) if read_timeout := kwargs.pop("read_timeout", None): config_kwargs.update({"read_timeout": read_timeout}) use_ssl = kwargs.pop("use_ssl", None) if use_ssl is not None: client_kwargs.update({"use_ssl": use_ssl}) if endpoint_url := kwargs.pop("endpoint_url", None): client_kwargs.update({"endpoint_url": endpoint_url}) if kwargs.pop("anon", False): config_kwargs.update({"signature_version": UNSIGNED}) else: creds = { key: value for key, value in { "aws_access_key_id": kwargs.pop("key", kwargs.pop("username", None)), "aws_secret_access_key": kwargs.pop("secret", kwargs.pop("password", None)), "aws_session_token": kwargs.pop("token", None), }.items() if value is not None } kwargs.update(creds) client_kwargs.update(creds) if profile := kwargs.pop("profile", None): kwargs.setdefault("profile_name", profile) session = Session(**{k: v for k, v in kwargs.items() if k in self._SESSION_PASSING_ARGS}) return session.client( "s3", config=Config(**config_kwargs), **{k: v for k, v in client_kwargs.items() if k in self._CLIENT_PASSING_ARGS}, ) @staticmethod def _directory_object(bucket: str, key: str | None, version_id: str | None = None) -> S3Object: """Build an S3Object representing a directory entry.""" return S3Object( init={ "ContentLength": 0, "ContentType": None, "StorageClass": S3StorageClass.S3_STORAGE_CLASS_DIRECTORY, "ETag": None, "LastModified": None, }, type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY, bucket=bucket, key=key, version_id=version_id, ) def _head_bucket( self, bucket, refresh: bool = False, lookup_kwargs: Mapping[str, Any] | None = None, ) -> S3Object | None: """Get the bucket as a directory object with HeadBucket. The result is cached under the bucket name, apart for each set of lookup parameters (see ``_get_cached_lookup``). A missing bucket evicts its entries and the cached bucket listing that still lists it. Args: bucket: The bucket name. refresh: If True, bypass the cache and call HeadBucket. lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``) to send with the request. Returns: The bucket object, or None if the bucket does not exist. """ lookup_kwargs = lookup_kwargs or {} file = None if refresh else self._get_cached_lookup(bucket, lookup_kwargs) if file is None: try: bucket_entry = self.core.head_bucket( bucket, **self.core.operation_params("head_bucket", lookup_kwargs) ) except FileNotFoundError: self._evict_cache(bucket) self._evict_cache((bucket, _LOOKUPS_CACHE_KEY)) # Evict the cached bucket listing only if it still lists the bucket. buckets = self.dircache.get("") if buckets and any(b.name == bucket for b in buckets): self._evict_cache("") return None file = S3Object.from_bucket(bucket_entry) self._cache_lookup(bucket, lookup_kwargs, file) return file def _head_object( self, path: S3Path, refresh: bool = False, lookup_kwargs: Mapping[str, Any] | None = None, ) -> S3Object | None: """Get the object with HeadObject. The result is cached under the lookup cache key of the path (see :meth:`_lookup_cache_key`), which keeps an explicit version apart from the key and from other versions, apart for each set of lookup parameters (see ``_get_cached_lookup``). An explicitly requested ``"null"`` version is not cached. A missing object evicts its entries and, unless a version was requested, the cached listing of its parent that still lists it. Args: path: The path of the object, optionally with a version ID. refresh: If True, bypass the cache and call HeadObject. lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``) to send with the request. Returns: The object, or None if it does not exist. """ version_id = path.version_id # Chosen before a version_aware lookup pins the version, so that the # lookup of the key stays cached under the key. cache_key = self._lookup_cache_key(path) # Writes invalidate only the path without the version, and an # overwrite replaces the "null" version of a bucket without # versioning, so that version is looked up every time. cacheable = version_id != "null" lookup_kwargs = lookup_kwargs or {} file = None if refresh else self._get_cached_lookup(cache_key, lookup_kwargs) if file is None: try: metadata = self.core.head_object( path, **self.core.operation_params("head_object", lookup_kwargs) ) except FileNotFoundError: self._evict_cache(cache_key) self._evict_cache((cache_key, _LOOKUPS_CACHE_KEY)) if not version_id: # Evict the cached listing of the parent only if it still # lists the path. parent_key = (self._listing_name(path), "/") files = self.dircache.get(parent_key) if files and any(f.name == path.name for f in files): self._evict_cache(parent_key) return None if self.version_aware and not version_id: # Pin the version of the object so that subsequent reads see # the version observed here even if the object is overwritten. version_id = metadata.version_id file = S3Object.from_metadata(metadata, version_id=version_id) if cacheable: self._cache_lookup(cache_key, lookup_kwargs, file) return file @staticmethod def _lookup_cache_key(path: S3Path) -> str | tuple[str, str, str]: """Return the dircache key of the lookup of a path. A path without a version is cached under its name, so that a key that ends in a query such as ``?versionId=v1`` is cached under its own name. An explicit version is cached under ``(name, _VERSION_CACHE_KEY, version_id)``, which no name equals. Args: path: The path of the bucket or the object. Returns: The dircache key. """ if path.version_id: return path.name, _VERSION_CACHE_KEY, path.version_id return path.name @staticmethod def _listing_name(path: S3Path) -> str: """Return the name of the directory whose listing holds the key of a path. The name of the key is cut at its last slash, without parsing it, so that the directory is named as its listing names it: ``bucket/a`` for the key ``a/x``, ``bucket/a/`` for ``a//x``, and ``bucket/a/dir?versionId=v1`` for ``a/dir?versionId=v1/x``. Args: path: The path of an object. Returns: The name of the directory, or the bucket for a key without a slash. """ return path.name.rpartition("/")[0] def _ls_buckets(self, refresh: bool = False) -> list[S3Object]: """List the buckets with ListBuckets, following its pages. The listing is cached under ``""``. Args: refresh: If True, bypass the cache and call ListBuckets. Returns: The buckets as directory objects. """ buckets = None if refresh else self.dircache.get("") if buckets is None: buckets = [ S3Object.from_bucket(b) for page in self.core.list_buckets() for b in page.buckets ] self.dircache[""] = buckets return buckets def _ls_dirs( self, path: str, prefix: str = "", delimiter: str = "/", refresh: bool = False, ) -> list[S3Object]: """List the objects and common prefixes under a bucket or key path. Lists the directory that the path names, as :meth:`_ls_prefix` describes. Args: path: The bucket or key path to list, without the protocol or a trailing slash. prefix: Key prefix to filter by, relative to the path. A prefixed listing is neither read from nor written to the cache. delimiter: Delimiter to group keys by; ``""`` lists recursively. refresh: If True, bypass the cache and list from S3. Returns: The listed directories and files. """ s3_path = S3Path.parse(path) return self._ls_prefix( self._directory_object(s3_path.bucket, s3_path.key), prefix=prefix, delimiter=delimiter, refresh=refresh, ) @staticmethod def _directory_prefix(directory: S3Object) -> str: """Return the key prefix that a listed directory or bucket lists. A directory is named after its common prefix without the final delimiter, which may leave the name with a version ID query or with trailing slashes. The prefix is taken from its key instead, so that the directory is listed as it was listed. Args: directory: A directory or bucket entry of a listing. Returns: The key prefix: ``""`` for a bucket, and otherwise the key of the directory followed by ``/``. """ return "" if directory.key is None else f"{directory.key}/" def _ls_prefix( self, directory: S3Object, prefix: str = "", delimiter: str = "/", refresh: bool = False, ) -> list[S3Object]: """List the objects and common prefixes under the key prefix of a directory. A complete, non-empty listing of the directory is cached under ``(name, delimiter)``, with the name of the directory, and an empty one, or a listing that raises FileNotFoundError because the bucket does not exist, evicts it. ``invalidate_cache`` drops it when the path or a path under it is invalidated. A common prefix is named after its prefix without the final delimiter, so the common prefix ``dir//`` of a key with an empty segment, such as ``dir//x``, is the directory ``bucket/dir/``, and ``/`` at the top of the bucket is ``bucket/``. Its key is the prefix without the final delimiter (see :meth:`_directory_prefix`). Args: directory: The bucket or directory to list, as a listing or :meth:`_directory_object` returns it. prefix: Key prefix to filter by, relative to the directory. A prefixed listing is neither read from nor written to the cache. delimiter: Delimiter to group keys by; ``""`` lists recursively. refresh: If True, bypass the cache and list from S3. Returns: The listed directories and files. """ bucket = directory.bucket use_cache = not prefix cache_key = (directory.name, delimiter) cached = self.dircache.get(cache_key) if use_cache and not refresh else None if cached is not None: return cast(list[S3Object], cached) files: list[S3Object] = [] try: for page in self.core.list_objects( bucket, prefix=self._directory_prefix(directory) + prefix, delimiter=delimiter, ): files.extend( self._directory_object(bucket, c.prefix[:-1]) for c in page.common_prefixes ) files.extend(S3Object.from_summary(o) for o in page.objects) except FileNotFoundError: # The bucket does not exist, so the cached listing is stale. self._evict_cache(cache_key) raise if use_cache: if files: self.dircache[cache_key] = files else: self._evict_cache(cache_key) return files
[docs] def ls( self, path: str | os.PathLike[str], detail: bool = False, refresh: bool = False, **kwargs ) -> list[S3Object] | list[str]: """List contents of an S3 path. Lists buckets (when path is root) or objects within a bucket/prefix. Compatible with fsspec interface for filesystem operations. Args: path: S3 path to list (e.g., "s3://bucket" or "s3://bucket/prefix"). detail: If True, return S3Object instances; if False, return paths as strings. refresh: If True, bypass cache and fetch fresh results from S3. **kwargs: Additional arguments including: versions: If True, list all versions of the objects, named ``bucket/key?versionId=<id>`` except for the ``null`` version. Requires the filesystem to be constructed with ``version_aware=True``. Returns: List of S3Object instances (if detail=True) or paths as strings (if detail=False). Raises: FileNotFoundError: If the bucket does not exist, or if a key path is neither an object nor a directory, a path that some key starts with followed by ``/``, or if HeadObject reports the version that a key path with a version ID names missing. Example: >>> fs = S3FileSystem() >>> fs.ls("s3://my-bucket") # List objects in bucket >>> fs.ls("s3://my-bucket/", detail=True) # Get detailed object info """ versions = kwargs.pop("versions", False) if versions and not self.version_aware: raise ValueError( "Cannot list the object versions unless the filesystem is version aware." ) path = self._strip_protocol(path).rstrip("/") if path in ["", "/"]: files = self._ls_buckets(refresh) else: s3_path = S3Path.parse(path) key = s3_path.key if key and s3_path.version_id: files = self._ls_version(s3_path, refresh=refresh) elif versions: files = self._ls_object_versions(path) else: files = self._ls_dirs(path, refresh=refresh) if not files and key: file = self._head_object(s3_path, refresh=refresh) if file: files = [file] # An empty bucket lists nothing, while a missing bucket raises # from the listing. if not files and key: raise FileNotFoundError(path) return list(files) if detail else [f.name for f in files]
def _ls_version(self, s3_path: S3Path, refresh: bool = False) -> list[S3Object]: """List the object version that a key path with a version ID names. A version names one object, which has nothing below it, so no directory is listed. The entry is named as ``ls(versions=True)`` names a version. Args: s3_path: The key path with a version ID. refresh: If True, bypass the cache and call HeadObject. Returns: The entry of the version, or an empty list if it does not exist. """ file = self._head_object(s3_path, refresh=refresh) if not file: return [] if s3_path.version_id != "null": # The cached entry of the lookup is named without the version. file = file.copy() file.name = str(s3_path) return [file] def _ls_object_versions(self, path: str) -> list[S3Object]: """List a prefix including all versions of the objects. The listing is always fetched from S3 and is not cached, because the dircache stores the current view of a path. A key path with nothing under it lists the versions of its object. Args: path: The bucket or key path, without the protocol. Returns: The listed directories and versions. """ s3_path = S3Path.parse(path) files = self._ls_prefix_versions(self._directory_object(s3_path.bucket, s3_path.key)) if not files and s3_path.key: # The path may point at an object rather than a key prefix. files = [ S3Object.from_version(v) for page in self.core.list_object_versions( s3_path.bucket, prefix=s3_path.key, delimiter="/" ) for v in page.versions if v.key == s3_path.key ] return files def _ls_prefix_versions(self, directory: S3Object) -> list[S3Object]: """List the common prefixes and the versions of the objects under a directory. Args: directory: The bucket or directory to list (see :meth:`_directory_prefix`). Returns: The listed directories and versions, not cached. """ bucket = directory.bucket files: list[S3Object] = [] for page in self.core.list_object_versions( bucket, prefix=self._directory_prefix(directory), delimiter="/" ): files.extend( self._directory_object(bucket, c.prefix[:-1]) for c in page.common_prefixes ) files.extend(S3Object.from_version(v) for v in page.versions) return files def _ls_directory( self, directory: S3Object, refresh: bool = False, versions: bool = False ) -> list[S3Object]: """List a listed directory or bucket by its key prefix, as ``ls()`` lists a path. Args: directory: A directory or bucket entry of a listing (see :meth:`_directory_prefix`). refresh: If True, bypass the cache and list from S3. versions: Whether to list all versions of the objects, as ``ls(versions=True)`` does. Returns: A new list of the listed directories and files. Raises: FileNotFoundError: If nothing is listed under a directory, such as one whose keys have been deleted since it was listed. """ if versions: files = self._ls_prefix_versions(directory) else: files = self._ls_prefix(directory, refresh=refresh) if not files and directory.key is not None: raise FileNotFoundError(directory.name) return list(files)
[docs] def info(self, path: str | os.PathLike[str] | S3Path, **kwargs) -> S3Object: """Return information about an S3 path. Uses the directory cache first: a cached entry for the path is returned, the entry of the path in a cached listing of its parent is returned, preferring an object to a key prefix of the same name as HeadObject does, and a cached listing of its parent without it means it does not exist. The cached bucket listing holds only the buckets that the caller owns, so a bucket missing from it is looked up with HeadBucket. Otherwise, a key path is looked up with HeadObject and, if no object exists, with a ListObjectsV2 request (``Delimiter="/"``, ``MaxKeys=1``) that checks whether it is a key prefix; a bucket path is looked up with HeadBucket. If these requests find a listed object missing, or find a key prefix, the cached listing of the parent is removed. With ``version_aware``, a cached file entry without a version ID is looked up again. With an explicit version, the cached entries of the path are skipped, the HeadObject result is cached under the name and the version ID, apart from the key and other versions, except for the ``null`` version, which an overwrite replaces, and a missing version is not looked up as a key prefix, since a version names an object. With request parameters on which the authorization of the requests depends (``ExpectedBucketOwner``, ``RequestPayer``, and the ``SSECustomer*`` parameters of an object encrypted with a customer-provided key), each request receives those that its operation accepts, and only the cached HeadObject or HeadBucket results of lookups with the same values of these parameters are used. Args: path: S3 path (e.g., "s3://bucket" or "s3://bucket/key"), or an ``S3Path``, whose bucket, key and version are looked up as they are, such as a key that a listing returns. **kwargs: Additional arguments including: refresh: If True, bypass the cache and query S3. version_id: The version ID to look up when the path has none. ExpectedBucketOwner, RequestPayer, SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5: The request parameters described above. Other request parameters are ignored. Returns: S3Object describing the bucket, directory, or file. The root path (``""``, ``"/"`` or ``"s3://"``) is a directory. Raises: FileNotFoundError: If the path does not exist. """ refresh = kwargs.pop("refresh", False) version_id = kwargs.pop("version_id", None) if isinstance(path, S3Path): s3_path = path else: name = self._strip_protocol(path) if name in ["/", ""]: # S3Path.parse rejects the root path. return S3Object( init={ "ContentLength": 0, "ContentType": None, "StorageClass": S3StorageClass.S3_STORAGE_CLASS_BUCKET, "ETag": None, "LastModified": None, }, type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY, bucket="", key=None, version_id=None, ) s3_path = S3Path.parse(name) if version_id and not s3_path.version_id: s3_path = s3_path.with_version_id(version_id) return self._path_info( s3_path, refresh=refresh, lookup_kwargs=self._get_lookup_kwargs(kwargs) )
def _path_info( self, path: S3Path, refresh: bool = False, lookup_kwargs: Mapping[str, Any] | None = None, ) -> S3Object: """Return information about the bucket, object or directory of a path. As :meth:`info` describes, for a path whose bucket, key and version are given as they are, such as a key listed with a version ID query in its name. Args: path: The path, with the version to look up, if any. refresh: If True, bypass the cache and query S3. lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``). Returns: S3Object describing the bucket, directory, or file. Raises: FileNotFoundError: If the path does not exist. """ version_id = path.version_id lookup_kwargs = lookup_kwargs or {} name = path.name # Cached entries describe the current version of a path as looked up # without lookup parameters, so an explicit version or lookup # parameters use only the HeadObject cache of that version and those # parameters. if not refresh and not version_id and not lookup_kwargs: caches: list[S3Object] | S3Object | None = self._ls_from_cache(path) if caches is not None: if isinstance(caches, list): matches = [c for c in caches if c.name == name] # A key can be both an object and a key prefix. cache = next( (c for c in matches if c.type == S3ObjectType.S3_OBJECT_TYPE_FILE), next(iter(matches), None), ) elif caches.name == name: cache = caches else: cache = None if cache: if ( self.version_aware and cache.get("type") == S3ObjectType.S3_OBJECT_TYPE_FILE and not cache.get("version_id") ): # A version-aware lookup needs the version to pin; # treat a version-less cached entry (e.g., populated # by a listing) as stale and head the object again. refresh = True else: return cache else: return self._directory_object( path.bucket, path.key.rstrip("/") if path.key else None, version_id ) if path.key: object_info = self._head_object(path, refresh=refresh, lookup_kwargs=lookup_kwargs) if object_info: return object_info if version_id: raise FileNotFoundError(str(path)) else: bucket_info = self._head_bucket( path.bucket, refresh=refresh, lookup_kwargs=lookup_kwargs ) if bucket_info: return bucket_info raise FileNotFoundError(str(path)) page = self.core.list_objects_page( path.bucket, prefix=f"{path.key.rstrip('/')}/" if path.key else "", delimiter="/", max_keys=1, **self.core.operation_params("list_objects_v2", lookup_kwargs), ) if (page.key_count or 0) > 0 or page.objects or page.common_prefixes: # Nothing caches the key prefix, and the cached listing of the # parent may predate it. self._evict_cache((self._listing_name(path), "/")) return self._directory_object( path.bucket, path.key.rstrip("/") if path.key else None, version_id ) raise FileNotFoundError(str(path)) def _extract_parent_directories( self, files: list[S3Object], bucket: str, base_prefix: str ) -> list[S3Object]: """Extract parent directory objects from file paths. When listing files without delimiter, S3 doesn't return directory entries. This method creates directory objects by analyzing file paths. Args: files: List of S3Object instances representing files. bucket: S3 bucket name. base_prefix: The key prefix below which the directories are derived: ``""``, or a prefix that ends in ``/``, such as ``a//`` for the keys ``a//x/y``. Returns: List of S3Object instances representing directories. """ dirs = set() for f in files: if ( f.key and f.type == S3ObjectType.S3_OBJECT_TYPE_FILE and f.key.startswith(base_prefix) ): parts = f.key[len(base_prefix) :].split("/") for i in range(1, len(parts)): dirs.add(base_prefix + "/".join(parts[:i])) return [self._directory_object(bucket, dir_path) for dir_path in dirs] def _find( self, path: str | os.PathLike[str], maxdepth: int | None = None, withdirs: bool | None = None, **kwargs, ) -> list[S3Object]: """List the objects below a path, as described in ``find``. Args: path: S3 path to search under. maxdepth: Maximum number of levels to descend, at least 1 (None for unlimited). withdirs: Whether to include directories in the result. **kwargs: Additional arguments including ``prefix`` and ``refresh``, as described in ``find``. Returns: The objects found, and the directories if ``withdirs`` is True. Raises: ValueError: If ``maxdepth`` is less than 1 or the path is the root. """ return [f for f, _ in self._find_paths(path, maxdepth, withdirs, **kwargs)] def _find_paths( self, path: str | os.PathLike[str], maxdepth: int | None = None, withdirs: bool | None = None, **kwargs, ) -> list[tuple[S3Object, str | S3Path]]: """List the objects below a path as :meth:`_find` does, with the paths they name. A listed entry names its bucket and key (see :meth:`_entry_path`), and the path itself, also a version that its entry names after its key alone, is named by the path string as given (see :meth:`_find_under`). Args: path: S3 path to search under. maxdepth: Maximum number of levels to descend, at least 1 (None for unlimited). withdirs: Whether to include directories in the result. **kwargs: Additional arguments including ``prefix`` and ``refresh``, as described in ``find``. Returns: The objects found, and the directories if ``withdirs`` is True, with the paths that they name. Raises: ValueError: If ``maxdepth`` is less than 1 or the path is the root. """ if maxdepth is not None and maxdepth < 1: raise ValueError("maxdepth must be at least 1") path = self._strip_protocol(path) if path in ["", "/"]: raise ValueError("Cannot traverse all files in S3.") s3_path = S3Path.parse(path) prefix = kwargs.pop("prefix", "") refresh = kwargs.pop("refresh", False) if s3_path.key and s3_path.version_id: return [(f, path) for f in self._ls_version(s3_path, refresh=refresh)] return self._find_under( self._directory_object(s3_path.bucket, s3_path.key), maxdepth=maxdepth, withdirs=withdirs, prefix=prefix, refresh=refresh, name=path, ) def _find_under( self, directory: S3Object, maxdepth: int | None = None, withdirs: bool | None = None, prefix: str = "", refresh: bool = False, name: str | None = None, ) -> list[tuple[S3Object, str | S3Path]]: """List the objects below a directory, as described in ``find``, with the paths they name. The directory is listed by its key prefix (see :meth:`_directory_prefix`), never by parsing its name. A listed entry names its bucket and key (see :meth:`_entry_path`). Args: directory: The bucket, the directory or the object to search under, as a listing or :meth:`_directory_object` returns it. maxdepth: Maximum number of levels to descend, at least 1 (None for unlimited). withdirs: Whether to include directories in the result. prefix: Key prefix, relative to the directory, to filter the listed keys by. refresh: If True, bypass the cache and list from S3. name: The path string that the caller passed for the directory, which names the directory itself and which ``info()`` looks up when nothing is listed below it; None for a listed directory, which names its bucket and key. Returns: The objects found, and the directories if ``withdirs`` is True, with the paths that they name. """ bucket, key = directory.bucket, directory.key own: str | S3Path = name if name is not None else S3Path(bucket, key) if maxdepth is not None: # The entries listed with the prefix lie as many levels further # below the path as the prefix has slashes. files = self._find_levels( directory, maxdepth - prefix.count("/"), prefix=prefix, refresh=refresh ) else: files = self._ls_prefix(directory, prefix=prefix, delimiter="", refresh=refresh) # S3 doesn't return directory entries without a delimiter, so the # directories are derived from the listed keys, below the last # slash of the listed key prefix, as with maxdepth. if withdirs: listed = self._directory_prefix(directory) + prefix # Build a new list; files may be the cached listing. files = files + self._extract_parent_directories( files, bucket, listed[: listed.rfind("/") + 1] ) found: list[tuple[S3Object, str | S3Path]] = [(f, self._entry_path(f)) for f in files] if found: # Something is listed below the path, so the path is a directory, # which fsspec includes with the directories. A bucket is not # included, since cp_file() cannot copy it in a recursive copy. if withdirs and key: found = [(self._directory_object(bucket, key), own), *found] elif key: # As in fsspec, the path itself is returned if it is an object, # or with withdirs if it is a directory. try: if name is not None: found = [(self.info(name, refresh=refresh), own)] else: found = [(self._path_info(S3Path(bucket, key), refresh=refresh), own)] except FileNotFoundError: found = [] if not withdirs: found = [(f, t) for f, t in found if f.type != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY] return found def _find_levels( self, directory: S3Object, maxdepth: int, prefix: str = "", refresh: bool = False ) -> list[S3Object]: """List the entries below a directory level by level with ``Delimiter="/"``. Each listed directory is listed by its key prefix, never by parsing its name (see :meth:`_directory_prefix`). Args: directory: The bucket or directory to search under. maxdepth: Number of levels to list. prefix: Key prefix, relative to the directory, to filter the first level by. refresh: If True, bypass the cache and list from S3. Returns: The objects and directories found. """ if maxdepth < 1: return [] result: list[S3Object] = [] for item in self._ls_prefix(directory, prefix=prefix, delimiter="/", refresh=refresh): result.append(item) if item.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: result.extend(self._find_levels(item, maxdepth - 1, refresh=refresh)) return result
[docs] def find( self, path: str | os.PathLike[str], maxdepth: int | None = None, withdirs: bool | None = None, detail: bool = False, **kwargs, ) -> dict[str, S3Object] | list[str]: """Find all files below a given S3 path. Recursively searches for files under the specified path, with optional depth limiting and directory inclusion. Uses efficient S3 list operations with delimiter handling for performance. As in fsspec, the result includes the path itself if withdirs is True and objects exist below it, unless it is a bucket, or if it is an object and nothing is listed below it. A key path with a version ID names that version of the object: the result is the version, named as ``ls(versions=True)`` names it, or empty if HeadObject reports it missing. Args: path: S3 path to search under (e.g., "s3://bucket/prefix"). maxdepth: Maximum number of levels to descend, at least 1 (None for unlimited). With 1, only the entries directly under the path are listed. withdirs: Whether to include directories in results (None = default behavior). detail: If True, return dict of {path: S3Object}; if False, return list of paths. **kwargs: Additional arguments including: prefix: Key prefix, relative to the path, to filter the listed keys by. Each slash in the prefix counts as one level of maxdepth. With withdirs, the directories above the prefix, such as ``sub`` for ``sub/deep/``, are not included. refresh: If True, bypass the cache and list from S3. Returns: Dictionary mapping paths to S3Objects (if detail=True) or list of paths (if detail=False). Raises: ValueError: If ``maxdepth`` is less than 1 or the path is the root. Example: >>> fs = S3FileSystem() >>> fs.find("s3://bucket/data/", maxdepth=2) # Limit depth >>> fs.find("s3://bucket/", withdirs=True) # Include directories """ files = self._find(path=path, maxdepth=maxdepth, withdirs=withdirs, **kwargs) if detail: return {f.name: f for f in files} return [f.name for f in files]
[docs] def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): """Walk the directories below a path, as fsspec's ``walk()`` does. Unlike fsspec, a listed directory is listed by its key prefix rather than by its name (see :meth:`_directory_prefix`), so a directory whose name ends in a version ID query or in an empty segment is walked as it was listed. Its name in the listing of its parent is the part of its name after the parent's name, such as ``""`` for the directory ``bucket/dir/`` of the keys ``dir//x`` in ``bucket/dir``, and it is walked under its name as listed. Args: path: The path to walk. maxdepth: Maximum number of levels to walk, at least 1 (None for unlimited). topdown: Whether to yield a directory before the directories below it. When True, removing names from the yielded directory names skips their directories. on_error: For the listing of the path, ``"omit"`` to yield nothing if it fails, ``"raise"`` to raise its error, or a callable that receives the error. As in fsspec, a directory below the path whose listing fails is skipped. **kwargs: Additional arguments passed to ``ls()``, such as ``refresh`` and ``versions``; ``detail=True`` yields the entries by name instead of the names. Yields: Tuples of a directory path, its directories, and its files. Raises: ValueError: If ``maxdepth`` is less than 1. """ yield from self._walk_tree( self._strip_protocol(path), maxdepth=maxdepth, topdown=topdown, on_error=on_error, **kwargs, )
def _walk_tree( self, path: str, maxdepth: int | None = None, topdown: bool = True, on_error: Any = "omit", directory: S3Object | None = None, yield_failed: bool = False, **kwargs, ) -> Generator[tuple[str, Any, Any], None, None]: """Walk a path, or a listed directory by its key prefix, as :meth:`walk` describes. Args: path: The path to walk, without the protocol, or the name of ``directory``. maxdepth: Maximum number of levels to walk, at least 1. topdown: Whether to yield a directory before those below it. on_error: What to do with an error of the listing of the path, as in :meth:`walk`. directory: The listed directory to walk, or None to list ``path`` with ``ls()``. yield_failed: Whether to yield a directory whose listing failed with no entries, as fsspec's ``_walk()`` does. **kwargs: Additional arguments passed to ``ls()``, as in :meth:`walk`. Yields: Tuples of a directory path, its directories, and its files. Raises: ValueError: If ``maxdepth`` is less than 1. """ if maxdepth is not None and maxdepth < 1: raise ValueError("maxdepth must be at least 1") detail = kwargs.pop("detail", False) try: listing = self._walk_listing(path, directory, **kwargs) except (FileNotFoundError, OSError) as e: if on_error == "raise": raise if callable(on_error): on_error(e) if yield_failed: yield path, {} if detail else [], {} if detail else [] return dirs, files = self._walk_entries(path, listing) names: Any = dirs if detail else list(dirs) listed: Any = files if detail else list(files) if topdown: yield path, names, listed if maxdepth is not None: maxdepth -= 1 if maxdepth < 1: if not topdown: yield path, names, listed return for d in names: # As in fsspec, on_error applies to the path itself, and the # failed listings of the directories below it are skipped. yield from self._walk_tree( dirs[d].name, maxdepth=maxdepth, topdown=topdown, directory=dirs[d], yield_failed=yield_failed, detail=detail, **kwargs, ) if not topdown: yield path, names, listed def _walk_listing(self, path: str, directory: S3Object | None, **kwargs) -> list[S3Object]: """List a directory of a walk. Args: path: The path to list, without the protocol, when ``directory`` is None. directory: The listed directory to list by its key prefix, or None. **kwargs: Additional arguments passed to ``ls()``, of which a listed directory uses ``refresh`` and ``versions``. Returns: The entries of the directory. Raises: FileNotFoundError: If nothing exists at the path, or nothing is listed under the directory. """ if directory is None: return cast(list[S3Object], self.ls(path, detail=True, **kwargs)) return self._ls_directory( directory, refresh=kwargs.get("refresh", False), versions=kwargs.get("versions", False) ) @staticmethod def _walk_entries( path: str, listing: list[S3Object] ) -> tuple[dict[str, S3Object], dict[str, S3Object]]: """Name the directories and files of a walked directory. An entry named below the path is named by the rest of its name, which an empty segment, or the folder marker of the path (its key followed by ``/``), leaves empty. Another entry, such as the object that ``ls()`` returns for the path itself, is named as fsspec's ``walk()`` names it. Args: path: The path of the walked directory. listing: The entries of the directory. Returns: The directories and the files, by name. """ dirs: dict[str, S3Object] = {} files: dict[str, S3Object] = {} for info in listing: name = info.name is_directory = info.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY if name.startswith(path + "/"): (dirs if is_directory else files)[name[len(path) + 1 :]] = info continue pathname = name.rstrip("/") if is_directory and pathname != path: dirs[pathname.rsplit("/", 1)[-1]] = info elif pathname == path: files[""] = info else: files[pathname.rsplit("/", 1)[-1]] = info return dirs, files
[docs] def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs): """Return the sizes of the objects below a path, as fsspec's ``du()`` does. Unlike fsspec, each entry that ``find()`` returns is looked up by its bucket and key (see :meth:`_entry_path`), never by parsing its name. Args: path: The path to search under. total: Whether to return the sum of the sizes instead of the sizes by name. maxdepth: Maximum number of levels to descend, at least 1 (None for unlimited). withdirs: Whether to include the directories, and the path itself if it is a directory. **kwargs: Additional arguments passed to ``find()``. Returns: The total size, or the sizes by the names that ``info()`` gives. """ sizes: dict[str, int] = {} if withdirs and self.isdir(path): info = self.info(path) sizes[info["name"]] = info["size"] for _, target in self._find_paths(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs): info = self.info(target) sizes[info["name"]] = info["size"] return sum(sizes.values()) if total else sizes
[docs] def glob(self, path, maxdepth=None, **kwargs): """Find the paths that match a glob pattern. As fsspec's ``glob()``, except that only the keys that start with the part of the last path component before the first wildcard are listed, as with fsspec's asynchronous ``_glob()``. Args: path: S3 glob pattern, as a string or an ``os.PathLike``. maxdepth: Maximum depth of the paths below the literal part of the pattern, at least 1. **kwargs: Additional arguments passed to ``find``, such as ``detail``. Returns: The matching paths, or a dictionary of them and their objects with ``detail=True``. """ stripped = self._strip_protocol(path) wildcards = [i for i in (stripped.find(c) for c in "*?[") if i >= 0] stem = stripped[: min(wildcards)].rpartition("/")[2] if wildcards else "" if stem and "prefix" not in kwargs: kwargs["prefix"] = stem return super().glob(stringify_path(path), maxdepth=maxdepth, **kwargs)
[docs] def tree( self, path: str | os.PathLike[str] = "/", recursion_limit: int = 2, max_display: int = 25, display_size: bool = False, prefix: str = "", is_last: bool = True, first: bool = True, indent_size: int = 4, ) -> str: """Return a tree of the paths below a path, as fsspec's ``tree()`` does. Unlike fsspec, a listed directory is listed by its key prefix rather than by its name, as in :meth:`walk`. Args: path: The root path, as a string or an ``os.PathLike``. recursion_limit: Maximum depth of the tree. max_display: Maximum number of entries shown per directory. display_size: Whether to show the sizes of the files. prefix: The prefix of each line. is_last: Whether the path is the last entry of its directory. first: Whether the path is the root of the tree. indent_size: Number of spaces per level. Returns: The tree as a string. """ return _S3TreeListing(self).tree( path=stringify_path(path), recursion_limit=recursion_limit, max_display=max_display, display_size=display_size, prefix=prefix, is_last=is_last, first=first, indent_size=indent_size, )
[docs] def get_mapper( self, root: str | os.PathLike[str] = "", check: bool = False, create: bool = False, missing_exceptions=None, ): """Return a key/value store at a path, as fsspec's ``get_mapper()`` does. Args: root: The root path of the store, as a string or an ``os.PathLike``. check: Whether to check that the root is writable. create: Whether to create the root if it does not exist. missing_exceptions: The exceptions that mean a missing key. Returns: The ``fsspec.mapping.FSMap`` of the root. """ return super().get_mapper( stringify_path(root), check=check, create=create, missing_exceptions=missing_exceptions, )
[docs] def unstrip_protocol(self, name: str | os.PathLike[str]) -> str: """Add the protocol to a path, as fsspec's ``unstrip_protocol()`` does. Args: name: The path, as a string or an ``os.PathLike``. Returns: The path with the ``s3://`` protocol. """ return cast(str, super().unstrip_protocol(stringify_path(name)))
@staticmethod def _stringify_paths(path: Any) -> Any: """Convert an ``os.PathLike`` path, or those of a list, to strings. The paths are converted with fsspec's ``stringify_path``, as ``_strip_protocol()`` converts them, but their slashes are kept. Args: path: A path or a list of paths. Returns: The converted path, or a new list of the converted paths. Strings are returned unchanged. """ if isinstance(path, list): return [stringify_path(p) for p in path] return stringify_path(path)
[docs] def expand_path(self, path, recursive=False, maxdepth=None, **kwargs) -> list[str]: """Expand glob patterns and directories into the paths they match. As in fsspec, except that a path with a version ID names that version of an object: it is not a glob pattern, although its ``?`` is one in fsspec, nor is anything expanded below it. With ``recursive``, it is included only if the version exists, as fsspec includes a path that exists. Args: path: S3 path, glob pattern, or list of them. recursive: Whether to include the paths below the directories. maxdepth: Maximum depth of the expansion, at least 1. **kwargs: Additional arguments passed to ``glob`` and ``find``. Returns: The sorted matching paths. Raises: ValueError: If ``maxdepth`` is less than 1. FileNotFoundError: If nothing matches. """ # The expansion is sorted by name; a name of two paths is returned once. return list( dict.fromkeys( name for name, _ in self._expand_paths(path, recursive, maxdepth, **kwargs) ) )
def _expand_paths( self, path, recursive: bool = False, maxdepth: int | None = None, **kwargs ) -> list[tuple[str, str | S3Path]]: """Expand glob patterns and directories as :meth:`expand_path`, with the paths they name. A path that the caller passes names itself, as the string that :meth:`expand_path` returns. A path that a listing returns names the bucket and key of its entry (see :meth:`_entry_path`), also when its name ends in what looks like a version ID query, and the expansion below a matched directory lists it by its key prefix. As in fsspec, the directories are expanded below the matches of a recursive glob pattern. Args: path: S3 path, glob pattern, or list of them. recursive: Whether to include the paths below the directories. maxdepth: Maximum depth of the expansion, at least 1. **kwargs: Additional arguments passed to ``glob`` and ``find``, and ``assume_literal``, as in fsspec. Returns: The names, as :meth:`expand_path` returns them, and the paths that they name, sorted by name: the name itself for a path that the caller passed, which the caller parses as a path string, and an ``S3Path`` for a listed entry. A path that the caller passed and a listed entry that name the same object are returned once, as the first of them. Raises: ValueError: If ``maxdepth`` is less than 1. FileNotFoundError: If nothing matches. """ if maxdepth is not None and maxdepth < 1: raise ValueError("maxdepth must be at least 1") versions, others = self._split_version_paths(path) # Insertion-ordered set of the names and the paths that they name. out: dict[tuple[str, str | S3Path], None] = { (p, p): None for p in versions if not recursive or self.exists(p) } if others: try: out.update(self._expand_unversioned(others, recursive, maxdepth, **kwargs)) except FileNotFoundError: if not out: raise if not out: raise FileNotFoundError(path) # The first name and path of each object, by what the path names. expanded: dict[S3Path | str, tuple[str, str | S3Path]] = {} for name, target in out: try: identity: S3Path | str = S3Path.of(target) except ValueError: # Such as the root path, which names no bucket. identity = target expanded.setdefault(identity, (name, target)) return sorted(expanded.values(), key=lambda item: item[0]) def _expand_unversioned( self, paths: list[str], recursive: bool, maxdepth: int | None, **kwargs ) -> dict[tuple[str, str | S3Path], None]: """Expand paths without a version ID as fsspec's ``expand_path()`` does. As :meth:`_expand_paths` describes: a path that the caller passes names itself, and a listed entry names its bucket and key. As in fsspec, a recursive glob pattern that matches nothing raises ``FileNotFoundError`` when the matches are expanded below. Args: paths: The paths and glob patterns, without the protocol. recursive: Whether to include the paths below the directories. maxdepth: Maximum depth of the expansion. **kwargs: Additional arguments passed to ``glob`` and ``find``, and ``assume_literal``, as in fsspec. Returns: The insertion-ordered names and the paths that they name. Raises: FileNotFoundError: If nothing matches, or a recursive glob pattern matches nothing. """ assume_literal = kwargs.pop("assume_literal", False) # The expansion asks glob() for the entries. kwargs.pop("detail", None) out: dict[tuple[str, str | S3Path], None] = {} names: set[str] = set() def add(name: str, target: str | S3Path) -> None: """Add a name and the path that it names. Args: name: The name. target: The path. """ out[(name, target)] = None names.add(name) for p in paths: if not assume_literal and has_magic(p): matches = cast( dict[str, S3Object], self.glob(p, maxdepth=maxdepth, detail=True, **kwargs) ) for name, match in matches.items(): add(name, self._entry_path(match)) # As fsspec expands the matches, one level fewer deep. if recursive and not (maxdepth is not None and maxdepth <= 1): if not matches: # As fsspec's expansion of no paths raises. raise FileNotFoundError([]) depth = maxdepth - 1 if maxdepth is not None else None for match in matches.values(): for f, target in self._find_under( match, depth, withdirs=True, prefix=kwargs.get("prefix", ""), refresh=kwargs.get("refresh", False), ): add(f.name, target) continue if recursive: for f, target in self._find_paths(p, maxdepth=maxdepth, withdirs=True, **kwargs): add(f.name, target) if p not in names and (not recursive or self.exists(p)): add(p, p) if not out: raise FileNotFoundError(paths) return out @staticmethod def _entry_path(entry: S3Object) -> S3Path: """Return the path that an entry of a listing or a lookup names. The bucket and the key of the entry are taken as they are, never parsed from its name. A listing of the current objects, and a lookup of a path without a version, name no version, also when a ``version_aware`` lookup records the version of the object. Args: entry: An entry that ``find()``, ``glob()`` or ``info()`` of a path without a version returns. Returns: The path of the bucket, the directory or the object. """ return S3Path(entry.bucket, entry.key) def _split_version_paths(self, path) -> tuple[list[str], list[str]]: """Split paths into those with a version ID and the others. Args: path: S3 path, glob pattern, or list of them. Returns: Tuple of the paths with a version ID and the other paths, without the protocol. """ paths = [ self._strip_protocol(p) for p in ([path] if isinstance(path, (str, os.PathLike)) else path) ] versions: list[str] = [] others: list[str] = [] for p in paths: (versions if S3Path.has_version_id(p) else others).append(p) return versions, others
[docs] def exists(self, path: str | os.PathLike[str], **kwargs) -> bool: """Check if an S3 path exists. Determines whether a bucket, object, or prefix exists in S3. Uses caching and efficient head operations to minimize API calls. Args: path: S3 path to check (e.g., "s3://bucket" or "s3://bucket/key"). **kwargs: Additional arguments including: refresh: If True, bypass the cache and query S3. ExpectedBucketOwner, RequestPayer, SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5: The request parameters on which the authorization of the requests depends, as described in :meth:`info`. With them, cached listings are not used. Other request parameters are ignored. Returns: True if the path exists, False otherwise. A bucket that HeadBucket denies access to (403) exists. Raises: PermissionError: If the lookup of an object is denied (403). Without ``s3:ListBucket`` on the bucket, S3 also answers 403 for a key that does not exist. Example: >>> fs = S3FileSystem() >>> fs.exists("s3://my-bucket/file.txt") >>> fs.exists("s3://my-bucket/") """ refresh = kwargs.pop("refresh", False) path = self._strip_protocol(path) if path in ["", "/"]: # The root always exists. return True s3_path = S3Path.parse(path) lookup_kwargs = self._get_lookup_kwargs(kwargs) # The cached listings are made without lookup parameters. use_listings = not refresh and not lookup_kwargs if s3_path.key: try: if use_listings and self._ls_from_cache(s3_path): return True info = self.info(path, refresh=refresh, **lookup_kwargs) return bool(info) except FileNotFoundError: return False if use_listings and self._ls_from_cache(S3Path(s3_path.bucket)): return True try: file = self._head_bucket(s3_path.bucket, refresh=refresh, lookup_kwargs=lookup_kwargs) except PermissionError: # HeadBucket answers 403 for a bucket that exists but that the # caller may not access. return True return bool(file)
[docs] def rm_file(self, path: str | os.PathLike[str], **kwargs) -> None: """Delete an S3 object with DeleteObject. Does nothing for a bucket path. If the path has a version ID, that version is deleted. Args: path: S3 path (s3://bucket/key) of the object to delete. **kwargs: Additional parameters passed to the DeleteObject API, such as ``RequestPayer``. Those that it does not accept are ignored, for fsspec compatibility. Raises: TypeError: If kwargs has ``bucket``, ``key`` or ``version_id``, or ``Bucket``, ``Key`` or ``VersionId``, which the path gives. """ path = stringify_path(path) s3_path = S3Path.parse(path) if not s3_path.key: return for name in ("bucket", "key", "version_id", "Bucket", "Key", "VersionId"): if name in kwargs: raise TypeError(f"rm_file() got multiple values for keyword argument '{name}'") _logger.debug(f"Delete object: {s3_path.uri}") self.core.delete_object(s3_path, **self.core.operation_params("delete_object", kwargs)) self.invalidate_cache(path)
[docs] def rm(self, path, recursive=False, maxdepth=None, batch_size=None, **kwargs) -> None: """Delete objects with DeleteObjects requests. Expands the paths with ``expand_path`` and deletes the matched objects in parallel requests of up to ``S3DeleteBatch.MAX_KEYS`` keys each, one set of requests per bucket. A path with a version ID deletes that version without expansion. Args: path: S3 path (s3://bucket/key) or list of paths to delete. recursive: Whether to delete all objects below the paths. maxdepth: Maximum depth to expand when ``recursive`` is True. batch_size: Number of DeleteObjects requests to send at the same time. By default the ``batch_size`` of the filesystem; None or -1 there is ``max_workers``. ``max_workers`` is a deprecated alias. **kwargs: Additional parameters passed to the DeleteObjects API. ``Quiet`` (default True) sets the quiet mode of the requests. Raises: ValueError: If a path is a bucket, or ``batch_size`` is neither positive, -1, nor None. TypeError: If both ``batch_size`` and ``max_workers`` are given. OSError: If S3 could not delete some of the objects. """ path = self._stringify_paths(path) max_workers = self._delete_batch_limit(batch_size, kwargs) paths = self._delete_paths(path, recursive=recursive, maxdepth=maxdepth) self._delete_objects(paths, max_workers=max_workers, **kwargs)
def _delete_batch_limit(self, batch_size: int | None, kwargs: dict[str, Any]) -> int: """Return the number of DeleteObjects requests of ``rm()`` to send at once. Args: batch_size: The ``batch_size`` argument of ``rm()``. kwargs: The other arguments of ``rm()``, from which its deprecated alias ``max_workers`` is removed. Returns: The number of requests. Raises: ValueError: If the number is neither positive, -1, nor None. TypeError: If both ``batch_size`` and ``max_workers`` are given. """ if "max_workers" in kwargs: if batch_size is not None: raise TypeError("rm() got both batch_size and its alias max_workers.") warnings.warn( "The max_workers argument of rm() is deprecated; use batch_size.", DeprecationWarning, stacklevel=3, ) batch_size = kwargs.pop("max_workers") return self._batch_limit(batch_size) def _delete_paths( self, path: str | list[str], recursive: bool = False, maxdepth: int | None = None ) -> list[str | S3Path]: """Expand the paths that ``rm()`` deletes (see :meth:`S3PathPairing.delete_paths`). Args: path: S3 path or list of paths. recursive: Whether to include all objects below the paths. maxdepth: Maximum depth to expand when ``recursive`` is True. Returns: The paths with a version ID as given, followed by the paths that the expansion of the other paths names (see :meth:`_expand_paths`): the path strings that the caller passed, and the listed paths. Raises: ValueError: If a path is a bucket. """ versioned_paths, unversioned_paths = S3PathPairing.delete_paths(path) paths: list[str | S3Path] = list(versioned_paths) if not unversioned_paths: # The expansion raises FileNotFoundError for no paths. return paths return paths + [ target for _, target in self._expand_paths( unversioned_paths, recursive=recursive, maxdepth=maxdepth ) ] def _create_executor(self, max_workers: int) -> _S3PoolScope: """Create the executor of the parallel requests of a file or an operation. The files of ``S3FileSystem`` and ``AioS3FileSystem`` and the parallel operations of both take their executors from this method. The requests run in the request pool of the filesystem, which they share, at most ``max_workers`` of those of the executor at once; shutting the executor down waits for its own requests only, and leaves the pool running. Args: max_workers: The maximum number of requests of the executor in the request pool at once. Returns: An executor on the request pool of the filesystem. Raises: ValueError: If ``max_workers`` is not positive. """ return _S3PoolScope(self._request_pool, max_workers) def _map( self, fn: Callable[[ItemT], T], items: Iterable[ItemT], max_workers: int | None = None, **kwargs: Any, ) -> S3MapResult[T]: """Run the parallel requests of an operation in a scope of their own. See :meth:`S3Executor.map() <pyathena.filesystem.s3_executor.S3Executor.map>`. The scope is shut down without another wait, since the map has waited for the calls that it started, unless a second interrupt stopped it. Args: fn: The request to run for each item. items: The items. max_workers: Maximum number of parallel requests. Defaults to ``self.max_workers``. **kwargs: The options of ``S3Executor.map()``. Returns: The outcomes of the requests. """ executor = self._create_executor(max_workers=max_workers or self.max_workers) try: return executor.map(fn, items, **kwargs) finally: executor.shutdown(wait=False) @staticmethod def _check_batch_size(batch_size: int | None) -> int | None: """Check a ``batch_size`` argument. Args: batch_size: The argument. Returns: The argument. Raises: ValueError: If it is neither positive, -1, nor None. """ if batch_size is not None and batch_size != -1 and batch_size <= 0: raise ValueError("batch_size must be a positive integer, -1, or None.") return batch_size def _batch_limit(self, batch_size: int | None) -> int: """Return the number of calls of a batch operation that may run at once. Args: batch_size: The ``batch_size`` of the call. None takes that of the filesystem; None or -1 there is ``max_workers``, the size of the request pool. Returns: The number of calls. Raises: ValueError: If ``batch_size`` is neither positive, -1, nor None. """ batch_size = self._check_batch_size(batch_size) if batch_size is None: batch_size = self.batch_size if batch_size is None or batch_size == -1: return self.max_workers return batch_size def _batch_kwargs(self, concurrency: int, kwargs: dict[str, Any]) -> dict[str, Any]: """Return the arguments of the per-object calls of a batch operation. With several calls at once, each object gets ``max_workers // concurrency`` parallel requests, at least one, so that a batch keeps about ``max_workers`` requests in the request pool. A ``max_workers`` of the caller is capped at that; None or 0 takes it. Args: concurrency: The number of calls that run at once. kwargs: The arguments that the caller gave the operation. Returns: The arguments of each call. """ if concurrency <= 1: return kwargs window = max(1, self.max_workers // concurrency) given = kwargs.get("max_workers") return {**kwargs, "max_workers": min(given, window) if given else window} def _run_batch( self, fn: Callable[[ItemT], T], items: list[ItemT], concurrency: int, counts: Callable[[BaseException], bool] | None = None, ) -> S3MapResult[T]: """Run the per-object calls of a batch operation. The calls are composite: each one waits for requests of its object in the request pool. They run in threads of a pool of their own, at most ``concurrency`` at once, and never in the request pool. A single item runs in the calling thread. See :meth:`S3Executor.map() <pyathena.filesystem.s3_executor.S3Executor.map>` for the failures, interrupts, and ``counts``. Args: fn: The call to run for each item. items: The items, in the order of the outcomes. concurrency: The number of calls that run at once. counts: Whether a failure stops the batch, by default every one. Returns: The outcomes of the calls. """ if len(items) == 1: try: return S3MapResult([fn(items[0])], None) except Exception as e: if counts is None or counts(e): raise return S3MapResult([e], None) executor = S3ThreadPool(max_workers=concurrency) try: return executor.map(fn, items, counts=counts) finally: # The map has waited for the calls that it started, unless a # second interrupt stopped it. executor.shutdown(wait=False) def _delete_objects( self, paths: list[str | S3Path], max_workers: int | None = None, **kwargs ) -> None: """Delete objects with DeleteObjects requests grouped by bucket. Args: paths: Paths of the objects to delete. Bucket paths are skipped. max_workers: Maximum number of parallel requests. Defaults to ``self.max_workers``. **kwargs: Additional parameters passed to the DeleteObjects API. ``Quiet`` (default True) sets the quiet mode of the requests. Raises: TypeError: If kwargs has ``Bucket`` or ``Delete``. OSError: If S3 could not delete some of the objects. """ batches, params = self._delete_batches(paths, **kwargs) if not batches: return # Every request runs also after another failed, and an interrupt # cancels those that have not started. results = self._map( functools.partial(self._delete_batch, **params), batches, max_workers=max_workers, stop_on_failure=False, ) self._raise_delete_errors(results.outcomes) @staticmethod def _delete_batches( paths: list[str | S3Path], **kwargs ) -> tuple[list[S3DeleteBatch], dict[str, Any]]: """Group the objects that ``rm`` deletes into DeleteObjects batches. Args: paths: Paths of the objects to delete. Bucket paths are skipped. **kwargs: Additional parameters of the DeleteObjects API. ``Quiet`` (default True) sets the quiet mode of the batches. Returns: The batches, and the parameters of their requests without ``Quiet``. Raises: TypeError: If kwargs has ``Bucket`` or ``Delete``. """ for name in ("Bucket", "Delete"): if name in kwargs: raise TypeError(f"rm() got an unexpected keyword argument '{name}'") quiet = kwargs.pop("Quiet", True) s3_paths = [S3Path.parse(p) if isinstance(p, str) else p for p in paths] batches = S3DeleteBatch.from_paths([p for p in s3_paths if p.key], quiet=quiet) return batches, kwargs def _delete_batch(self, batch: S3DeleteBatch, **kwargs) -> S3DeleteResult: """Delete a batch and invalidate the cache of its objects. The cache is invalidated when the request has finished, also when it fails, because S3 may have deleted some of the objects. Args: batch: The objects to delete. **kwargs: Additional parameters passed to the DeleteObjects API. Returns: The result of the request. """ try: return self.core.delete_objects(batch, **kwargs) finally: for path in batch.objects: self._invalidate(path) @staticmethod def _raise_delete_errors(results: list[S3DeleteResult | BaseException]) -> None: """Raise an error for the batches that S3 could not delete. Args: results: The result or the exception of each batch. Raises: BaseException: The first exception of the batches, with a note that lists the errors of the results, if any. OSError: If no batch raised and a result has errors. """ exceptions: list[BaseException] = [] errors: list[str] = [] for result in results: if isinstance(result, BaseException): exceptions.append(result) else: errors.extend(str(e) for e in result.errors) message = f"Failed to delete objects: {', '.join(sorted(errors))}" if errors else None if exceptions: if message: exceptions[0].add_note(message) raise exceptions[0] if message: raise OSError(message)
[docs] def mkdir(self, path: str | os.PathLike[str], create_parents: bool = True, **kwargs) -> None: """Create an S3 bucket. S3 has no real directories below the bucket level; creating a key prefix requires no operation. This method creates the bucket when the path points at a bucket (or when ``create_parents`` is True and the bucket does not exist yet), and does nothing for key prefixes under an existing bucket. Bucket lifecycle operations are disabled by default because they are infrastructure-level changes; pass ``allow_bucket_creation=True`` to the filesystem constructor to enable bucket creation. Args: path: S3 path (e.g., "s3://bucket" or "s3://bucket/prefix"). create_parents: If True, create the bucket when it does not exist, even if the path contains a key prefix. **kwargs: Additional arguments including: acl: Canned ACL to apply to the bucket. region_name: Region to create the bucket in. Defaults to the client's region. Other parameters that the CreateBucket API accepts, such as ``ObjectLockEnabledForBucket``, are passed to it; the others are ignored, for fsspec compatibility. Raises: FileExistsError: If the path is a bucket that already exists. FileNotFoundError: If the bucket does not exist and ``create_parents`` is False. PermissionError: If the bucket would be created but bucket creation is not enabled on this filesystem instance. ValueError: If the ACL is invalid, the path is empty, or botocore rejects the parameters of the CreateBucket request. """ path = self._strip_protocol(path).rstrip("/") if not path: raise ValueError("Cannot create the root directory.") s3_path = S3Path.parse(path) if self.exists(s3_path.bucket): if not s3_path.key: # Requested to create a bucket, but the bucket already exists. raise FileExistsError(s3_path.bucket) # Do nothing as the bucket already exists. elif not s3_path.key or create_parents: if not self.allow_bucket_creation: raise PermissionError( "Bucket creation is disabled. " "Set allow_bucket_creation=True on the filesystem to enable it." ) self.core.create_bucket( s3_path.bucket, acl=kwargs.pop("acl", None), region_name=kwargs.pop("region_name", None), **self.core.operation_params("create_bucket", kwargs), ) # invalidate_cache of the bucket keeps the cached bucket # listing, so evict it directly. self._evict_cache("") self.invalidate_cache(s3_path.bucket) else: # exists() has already confirmed the bucket does not exist, # and it is not requested to be created. raise FileNotFoundError(s3_path.bucket)
[docs] def makedirs(self, path: str | os.PathLike[str], exist_ok: bool = False) -> None: """Recursively create a directory, creating the bucket if necessary. Creating the bucket requires ``allow_bucket_creation=True`` on the filesystem constructor; see :meth:`mkdir`. Args: path: S3 path (e.g., "s3://bucket" or "s3://bucket/prefix"). exist_ok: If False, raise FileExistsError when the path is a bucket that already exists. Raises: FileExistsError: If the path is a bucket that already exists and ``exist_ok`` is False. PermissionError: If the bucket would be created but bucket creation is not enabled on this filesystem instance. """ try: self.mkdir(path, create_parents=True) except FileExistsError: if not exist_ok: raise
[docs] def rmdir(self, path: str | os.PathLike[str]) -> None: """Remove an S3 bucket, which must be empty. S3 has no real directories below the bucket level, so only bucket paths can be removed. Bucket lifecycle operations are disabled by default because they are infrastructure-level changes; pass ``allow_bucket_deletion=True`` to the filesystem constructor to enable bucket deletion. Args: path: S3 bucket path (e.g., "s3://bucket"). Raises: FileExistsError: If the path contains a key that exists. The user may have meant ``rm(path, recursive=True)``. FileNotFoundError: If the path contains a key that does not exist, or the bucket does not exist. PermissionError: If bucket deletion is not enabled on this filesystem instance. OSError: If the bucket is not empty. """ path = self._strip_protocol(path).rstrip("/") s3_path = S3Path.parse(path) if s3_path.key: if self.exists(path): # The user may have meant rm(path, recursive=True). raise FileExistsError(path) raise FileNotFoundError(path) if not self.allow_bucket_deletion: raise PermissionError( "Bucket deletion is disabled. " "Set allow_bucket_deletion=True on the filesystem to enable it." ) self.core.delete_bucket(s3_path.bucket) self.invalidate_cache(s3_path.bucket) # invalidate_cache of the bucket keeps the cached bucket listing, # so evict it directly. self._evict_cache("")
[docs] def touch( self, path: str | os.PathLike[str], truncate: bool = True, **kwargs ) -> dict[str, Any]: """Create an empty object with PutObject. Args: path: S3 path (s3://bucket/key) of the object. truncate: If True, replace an existing object with an empty one; if False, raise if the object exists. A key that is only a prefix of other keys has no object. **kwargs: Additional parameters passed to the PutObject API, except ``Body``: the object is empty. Returns: The PutObject response as a dictionary (see :meth:`S3PutObject.to_dict`). Raises: TypeError: If kwargs has ``Body``. ValueError: If the path has a version ID, is a bucket, or exists while ``truncate`` is False. """ path = stringify_path(path) s3_path = S3Path.parse(path) if s3_path.version_id: raise ValueError("Cannot touch the file with the version specified.") if "Body" in kwargs: raise TypeError("touch() creates an empty object; use pipe_file() to write data.") if not truncate and self._key_exists(s3_path, self._get_lookup_kwargs(kwargs)): raise ValueError("Cannot touch the existing file without specifying truncate.") if not s3_path.key: raise ValueError("Cannot touch the bucket.") # The empty body is passed, so that a body keyword argument raises # TypeError. object_ = self.core.put_object(s3_path, None, **kwargs) self.invalidate_cache(path) return object_.to_dict()
[docs] def mv(self, path1, path2, recursive=False, maxdepth=None, batch_size=None, **kwargs) -> None: """Move files from one S3 location to another. Copies the files to the destinations that ``copy()`` gives them, and then deletes only the source objects that were copied. fsspec's ``AbstractFileSystem.mv()`` instead removes ``path1`` by expanding it again, which also deletes copies placed where ``path1`` matches them and files that ``maxdepth`` kept from being copied. A file whose destination is the file itself is left in place, and directories, which S3 does not store as objects, are not copied. A ``null`` version moved onto its key is copied and deleted only when bucket versioning is enabled. Without versioning, or with versioning suspended, that move is left in place because the copy would replace the version that the deletion removes. Comparing a ``null`` version with its unversioned key calls GetBucketVersioning once per bucket during the move's planning and requires ``s3:GetBucketVersioning``. The result is not cached across moves. Directory buckets do not support versioning and need no lookup. A source without a version whose destination conflicts with another pair is looked up with HeadObject during the planning too. The files are copied in parallel as ``copy()`` copies them, and are deleted once every copy has succeeded; nothing is deleted after a failed copy. Args: path1: Source S3 path, glob pattern, or list of paths. path2: Destination S3 path, or list of paths when ``path1`` is a list. recursive: Whether to move the directories with their contents. maxdepth: Maximum depth of a recursive move. batch_size: Number of copies to run at the same time, as in ``copy()``. **kwargs: Additional S3 copy parameters passed to ``cp_file()``. The lookups and the deletion of the sources receive ``RequestPayer`` and the source's expected bucket owner (see :meth:`S3Core.copy_source_params`); the lookups also receive the source's SSE-C parameters. Raises: ValueError: If two sources have the same destination, or a destination is another source, including one left in place, which is checked before anything is copied. A source without an object at its key, such as a directory, writes no destination and does not conflict. Also if ``batch_size`` is neither positive, -1, nor None. FileNotFoundError: If a source does not exist, once the running copies have ended; nothing is deleted. OSError: If bucket versioning cannot be read. """ path1, path2 = self._stringify_paths(path1), self._stringify_paths(path2) if path1 == path2: return limit = self._batch_limit(batch_size) source_params = self.core.copy_source_params(kwargs) pairs = self._move_pairs( path1, path2, recursive=recursive, maxdepth=maxdepth, source_params=source_params, ) if not pairs: return concurrency = min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> bool: """Copy a pair of the move. Args: pair: The source and its destination. Returns: Whether the source was copied, as ``_copy_file()`` returns it. """ return self._copy_file(*pair, **call_kwargs) # Every copy has ended, and succeeded, before anything is deleted. copied = self._run_batch(copy_pair, pairs, concurrency).outcomes self._delete_objects( [p1 for (p1, _), c in zip(pairs, copied, strict=True) if c], **self.core.operation_params("delete_objects", source_params), )
def _move_pairs( self, path1: str | list[str], path2: str | list[str], recursive: bool = False, maxdepth: int | None = None, source_params: Mapping[str, Any] | None = None, ) -> list[tuple[str | S3Path, str | S3Path]]: """Pair and check the paths of a move (see :meth:`S3PathPairing.move_pairs`). Args: path1: Source S3 path, glob pattern, or list of paths. path2: Destination S3 path, or list of paths when ``path1`` is a list. recursive: Whether to include the contents of the directories. maxdepth: Maximum depth of the expansion. source_params: The parameters of the requests that read the sources (see :meth:`S3Core.copy_source_params`), sent with the lookups of the sources. Returns: The sources and destinations that are moved. Raises: ValueError: If the move has conflicting paths. OSError: If bucket versioning cannot be read. """ pairing = S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth) pairs = self._copy_pairs(pairing) missing, versioned_buckets = self._conflict_lookups(pairing, pairs, source_params) return pairing.move_pairs( pairs, missing=missing, versioning_enabled_buckets=versioned_buckets ) def _conflict_lookups( self, pairing: S3PathPairing, pairs: list[tuple[str | S3Path, str | S3Path]], source_params: Mapping[str, Any] | None = None, ) -> tuple[set[str | S3Path], set[str]]: """Look up what the conflict checks of a move or a copy need. See :meth:`S3PathPairing.conflict_candidates`. Args: pairing: The paths of the move or the copy. pairs: The pairs of the paths. source_params: The parameters of the requests that read the sources (see :meth:`S3Core.copy_source_params`), sent with the lookups of the sources. Returns: The conflict candidates with no object at their key, and the buckets whose versioning is enabled among those with a ``null`` version and its unversioned key in the pairs. Raises: OSError: If bucket versioning cannot be read. """ paths = {S3Path.of(p) for pair in pairs for p in pair} unversioned = {p.name for p in paths if not p.version_id} buckets = { p.bucket for p in paths if p.version_id == "null" and p.name in unversioned and not p.is_directory_bucket } versioned_buckets = { bucket for bucket in buckets if self.core.get_bucket_versioning(bucket) == "Enabled" } lookup_kwargs = self._get_lookup_kwargs(source_params or {}) missing = { source for source in pairing.conflict_candidates( pairs, versioning_enabled_buckets=versioned_buckets ) if self._head_object(S3Path.of(source), lookup_kwargs=lookup_kwargs) is None } return missing, versioned_buckets def _copy_pairs( self, pairing: S3PathPairing, local: bool = False, **kwargs: Any, ) -> list[tuple[str | S3Path, str | S3Path]]: """Expand and pair the paths of a copy (see :meth:`S3PathPairing.s3_copy_pairs`). The sources are expanded as :meth:`_expand_paths` names them: a path that the caller passes as the string it was given, and a listed entry as its bucket and key. The destination is looked up only when it decides the pairing. Args: pairing: The paths of the copy. local: Whether the destinations are the local paths of ``get()``, which the local filesystem looks up, and which are named as :meth:`S3PathPairing.copy_pairs` names them. **kwargs: Passed to the expansion, as fsspec's ``get()`` passes its keyword arguments to ``expand_path()``. Returns: The sources and their destinations. """ if not pairing.expands: return list(pairing.copy_pairs()) sources = [ target for name, target in self._expand_paths( pairing.path1, recursive=pairing.recursive, maxdepth=pairing.maxdepth, **kwargs ) if not (pairing.skips_directories and (trailing_sep(name) or self.isdir(target))) ] isdir = LocalFileSystem().isdir if local else self.isdir destination_is_dir = ( # A string, as looks_up_destination checks. isdir(cast(str, pairing.path2)) if sources and pairing.looks_up_destination else None ) if local: return list(pairing.copy_pairs(sources, destination_is_dir)) return pairing.s3_copy_pairs(sources, destination_is_dir) def _checked_copy_pairs( self, path1: str | list[str], path2: str | list[str], recursive: bool = False, maxdepth: int | None = None, source_params: Mapping[str, Any] | None = None, ) -> list[tuple[str | S3Path, str | S3Path]]: """Pair and check the paths of a copy (see :meth:`S3PathPairing.check_copy_pairs`). Args: path1: Source S3 path, glob pattern, or list of them. path2: Destination S3 path, or list of paths when ``path1`` is a list. recursive: Whether to include the contents of the directories. maxdepth: Maximum depth of the expansion. source_params: The parameters of the requests that read the sources (see :meth:`S3Core.copy_source_params`), sent with the lookups of the sources. Returns: The sources and destinations that are copied. Raises: ValueError: If the copy has conflicting paths. OSError: If bucket versioning cannot be read. """ pairing = S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth) pairs = self._copy_pairs(pairing) if pairs: missing, versioned_buckets = self._conflict_lookups(pairing, pairs, source_params) pairing.check_copy_pairs( pairs, missing=missing, versioning_enabled_buckets=versioned_buckets ) return pairs
[docs] def copy( self, path1, path2, recursive=False, maxdepth=None, on_error=None, batch_size=None, **kwargs, ) -> None: """Copy files within S3. As fsspec's ``copy()``, except that a source with a version ID copies that version of the object to a destination named after its key, as :meth:`~pyathena.filesystem.s3_path_pairing.S3PathPairing.copy_pairs` pairs them, and that the files are copied in parallel, ``batch_size`` at a time. Pairs that would depend on the order of the copies raise before anything is copied (see :meth:`~pyathena.filesystem.s3_path_pairing.S3PathPairing.check_copy_pairs`). An error that ``on_error`` does not ignore starts no further copy, stops the running ones before their next request where it can (a multipart copy sends no further part and is aborted), and is raised once they have ended. Copies that ended before it stay in place. Args: path1: Source S3 path, glob pattern, or list of them. path2: Destination S3 path, or list of paths when ``path1`` is a list. recursive: Whether to copy the directories with their contents. maxdepth: Maximum depth of a recursive copy. on_error: ``"raise"`` or ``"ignore"`` for a missing source; by default ``"ignore"`` with ``recursive`` and ``"raise"`` otherwise. batch_size: Number of copies to run at the same time. By default the ``batch_size`` of the filesystem; None or -1 there is ``max_workers``. With ``n`` copies at once, the smaller of ``batch_size`` and the number of pairs, each multipart copy sends at most ``max(1, max_workers // n)`` parts at once. **kwargs: Additional S3 copy parameters passed to ``cp_file()``. Raises: ValueError: If two sources have the same destination, or a destination is another source, except for a source without an object at its key, such as a directory, which is not copied. Also if ``batch_size`` is neither positive, -1, nor None. OSError: If bucket versioning cannot be read for the checks. """ path1, path2 = self._stringify_paths(path1), self._stringify_paths(path2) if on_error is None: on_error = "ignore" if recursive else "raise" limit = self._batch_limit(batch_size) pairs = self._checked_copy_pairs( path1, path2, recursive=recursive, maxdepth=maxdepth, source_params=self.core.copy_source_params(kwargs), ) if not pairs: return concurrency = min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> None: """Copy a pair. Args: pair: The source and its destination. """ self.cp_file(*pair, **call_kwargs) self._run_batch( copy_pair, pairs, concurrency, counts=None if on_error == "raise" else lambda e: not isinstance(e, FileNotFoundError), )
[docs] def get( self, rpath, lpath, recursive=False, callback=_DEFAULT_CALLBACK, maxdepth=None, batch_size=None, **kwargs, ) -> None: """Copy files from S3 to the local filesystem. As fsspec's ``get()``, except that a source with a version ID downloads that version of the object to a local path named after its key, as :meth:`~pyathena.filesystem.s3_path_pairing.S3PathPairing.copy_pairs` pairs them, and that the files are downloaded in parallel, as with ``copy()``. The destinations named after the sources are checked to lie under ``lpath``. With ``outfile``, the files are written to it one after another, in order. A failed download starts no further one, the running ones read and write no further block, and the error is raised once they have ended. Args: rpath: Source S3 path, glob pattern, or list of them. lpath: Local destination path, or list of paths or file-like objects when ``rpath`` is a list. recursive: Whether to copy the directories with their contents. callback: Progress callback. It and its branches for the files are used from the threads that download them, one at a time. maxdepth: Maximum depth of a recursive copy. batch_size: Number of downloads to run at the same time, as in ``copy()``. **kwargs: Additional parameters passed to ``get_file()``. Raises: ValueError: If a destination named after a source lies outside ``lpath``, two downloads write the same local file, or ``batch_size`` is neither positive, -1, nor None. """ rpath = self._stringify_paths(rpath) limit = self._batch_limit(batch_size) root = make_path_posix(lpath) if isinstance(lpath, (str, os.PathLike)) else lpath pairs = self._checked_get_pairs( S3PathPairing(rpath, root, recursive=recursive, maxdepth=maxdepth), kwargs, **kwargs ) if not pairs: return # get_file() writes to an outfile instead of the local paths. outfile = kwargs.get("outfile") is not None concurrency = 1 if outfile else min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) locked = _S3LockedCallback(callback, threading.Lock()) def get_pair(pair: tuple[str | S3Path, Any]) -> None: """Download a pair with a branch of the callback. Args: pair: The S3 source and its local destination. The branch is named after the source as a string. """ rp, lp = pair with locked.branched(str(rp), lp) as child: self.get_file(rp, lp, callback=child, **call_kwargs) locked.relative_update(1) locked.set_size(len(pairs)) if outfile: # One after another, in order, in the calling thread, which may # own the stream. for pair in pairs: get_pair(pair) return self._run_batch(get_pair, pairs, concurrency)
def _checked_get_pairs( self, pairing: S3PathPairing, file_kwargs: Mapping[str, Any], **kwargs: Any ) -> list[tuple[str | S3Path, Any]]: """Expand, pair and check the paths of a ``get()``. The destinations named after the sources are checked to lie under the local destination. Without ``outfile``, the files are downloaded in parallel, so no two of them may write the same local file (see :meth:`_check_local_destinations`). Args: pairing: The paths of the copy, with the local destination. file_kwargs: The arguments of ``get_file()``, whose ``outfile`` and ``version_id`` the checks follow. **kwargs: Passed to ``expand_path()``. Returns: The S3 sources and their local destinations. Raises: ValueError: If a destination named after a source lies outside the local destination, or two downloads write the same local file. """ pairs = self._copy_pairs(pairing, local=True, **kwargs) if isinstance(pairing.path2, str): check_contained(pairing.path2, [lp for _, lp in pairs]) if pairs and file_kwargs.get("outfile") is None: self._check_local_destinations(pairs, file_kwargs.get("version_id")) return pairs def _check_local_destinations( self, pairs: list[tuple[str | S3Path, Any]], version_id: str | None = None ) -> None: """Check that no two downloads of a batch write the same local file. Destinations are compared by the object for file-like ones, by the file for existing paths, such as hard links to one file, and by the resolved path otherwise. A source that ``get_file()`` creates as a local directory writes no file, so several of them may share a local path; the sources are looked up only when their local path is shared. Args: pairs: The S3 sources and their local paths or file-like objects. version_id: The version ID that ``get_file()`` receives. Raises: ValueError: If two downloads write the same file, or one creates a directory where another writes a file. """ sources: dict[tuple[Any, ...], list[str | S3Path]] = {} for rpath, destination in pairs: key: tuple[Any, ...] if isfilelike(destination): key = ("object", id(destination)) else: path = os.fspath(destination) try: stat = os.stat(path) except OSError: key = ("path", os.path.normcase(os.path.realpath(path))) else: if os.path.isdir(path): # Directories are created, not written. continue key = ("file", stat.st_dev, stat.st_ino) sources.setdefault(key, []).append(rpath) for key, rpaths in sources.items(): # get_file() writes into a file-like object whatever the source. if len(rpaths) > 1 and ( key[0] == "object" or not all(self._is_directory_source(rp, version_id) for rp in rpaths) ): raise ValueError("Cannot get several paths to the same destination.") def _is_directory_source( self, rpath: str | os.PathLike[str] | S3Path, version_id: str | None = None ) -> bool: """Return whether ``get_file()`` creates a local directory for a source. A requested version always names an object, while ``isdir()`` would look up the latest version, or the prefix of the same name when the version does not exist. Args: rpath: The S3 source, or an ``S3Path``, as for ``get_file()``. version_id: The version ID that ``get_file()`` receives. Returns: True if the source has no version and is a directory. """ path = rpath if isinstance(rpath, S3Path) else S3Path.parse(self._strip_protocol(rpath)) if version_id or path.version_id: return False return bool(self.isdir(rpath))
[docs] def cp_file( self, path1: str | os.PathLike[str] | S3Path, path2: str | os.PathLike[str] | S3Path, recursive=False, maxdepth=None, on_error=None, **kwargs, ): """Copy an S3 object to another S3 location. Performs server-side copy of S3 objects, which is more efficient than downloading and re-uploading. Automatically chooses between simple copy and multipart copy based on object size. Args: path1: Source S3 path (s3://bucket/key), or an ``S3Path``, whose bucket, key and version are used as they are, such as a key that a listing returns. path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. recursive: Unused parameter for fsspec compatibility. maxdepth: Unused parameter for fsspec compatibility. on_error: Unused parameter for fsspec compatibility. **kwargs: Additional S3 copy parameters (e.g., metadata, storage class). The ``block_size`` and ``max_workers`` parameters control a multipart copy and are not sent to S3. The lookup of the source receives ``RequestPayer``, and the source's expected bucket owner and SSE-C parameters under the names of the requests that read it (see :meth:`S3Core.copy_source_params`). Raises: ValueError: If trying to copy to a versioned file or copy buckets, or if a directive of a multipart copy has an invalid value. Note: Uses multipart copy for objects larger than the maximum part size to optimize performance for large files. The copy operation is performed entirely on the S3 service without data transfer. The multipart copy applies the metadata, tagging and annotation directives as CopyObject does (default ``COPY``). A directory ``path1``, which recursive ``copy()`` passes along with the files under it, is skipped. """ self._copy_file( path1 if isinstance(path1, S3Path) else stringify_path(path1), path2 if isinstance(path2, S3Path) else stringify_path(path2), **kwargs, )
def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`cp_file` does. Args: path1: Source S3 path (s3://bucket/key), or an ``S3Path``. path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. **kwargs: Additional S3 copy parameters, as for :meth:`cp_file`. Returns: False if ``path1`` is a directory, which is skipped; True if the object was copied. Raises: ValueError: If trying to copy to a versioned file or copy buckets. """ # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) source = S3Path.of(path1) destination = S3Path.of(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: raise ValueError("Cannot copy buckets.") # Looked up with the parameters that authorize the access to the # source, such as its customer-provided key. info1 = self.info(path1, **self.core.copy_source_params(kwargs)) if info1.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # Recursive copy() passes the directories too; S3 has no object # to copy for them. return False size1 = info1.get("size", 0) try: if size1 <= self.core.MULTIPART_UPLOAD_MAX_PART_SIZE: raise_if_stopped(f"copy {source.uri} to {destination.uri}") self.core.copy_object(source, destination, **kwargs) else: self._copy_object_with_multipart_upload( source, destination, max_workers=max_workers, block_size=block_size, **kwargs, ) finally: # A multipart copy that fails to copy an annotation has already # written the destination. self._invalidate(destination) return True def _copy_object_with_multipart_upload( self, source: S3Path, destination: S3Path, max_workers: int | None = None, block_size: int | None = None, **kwargs, ) -> None: """Copy an object with a multipart upload of its byte ranges. Runs the plan of :meth:`S3Core.plan_multipart_copy`, which reads the source and lists its annotations before anything is written. The parts are copied in parallel with UploadPartCopy, ``max_workers`` at a time, each one sent once an earlier one has finished, and the annotations are copied onto the destination after the upload completes. A failed part or completion aborts the upload, and so does an interrupt, after the creation of the upload and the running part copies, which then make no further attempt, have finished. Further interrupts while the creation is waited for are suppressed; one while the part copies are waited for or the upload is aborted stops the cleanup, and the upload ID is logged. A failed annotation copy is raised and leaves the destination in place. The annotations are copied, retries included, also once an enclosing stop scope, such as that of a batch operation, has been stopped, since the destination is committed; until then, a stop sends no further part or completion and the upload is aborted. If HeadObject reports a size that fits in a single CopyObject request, the reported version is copied with CopyObject instead. Args: source: Source S3 path, with the version ID to copy, if any. destination: Destination S3 path. max_workers: Maximum number of parallel requests. block_size: Size in bytes of the copied ranges. **kwargs: The CopyObject parameters of the copy; each request receives those that it accepts. Raises: ValueError: If ``block_size`` is out of the part size limits, a directive has an invalid value, or HeadObject reports no size. """ max_workers = max_workers if max_workers else self.max_workers plan = self.core.plan_multipart_copy(source, destination, block_size, **kwargs) if plan.fits_single_request: # The size that the caller found, which may come from a cached # listing, was larger than the copied version, which fits in a # single CopyObject request. raise_if_stopped(f"copy {plan.source.uri} to {plan.destination.uri}") self.core.copy_object(plan.source, plan.destination, **kwargs) return with self._create_executor(max_workers=max_workers) as executor: # The requests run on the executor, so that an interrupt lets the # running ones finish and the upload be aborted. session = _S3WriteSession( self.core, plan.destination, executor, block_size=block_size if block_size else self.core.MULTIPART_UPLOAD_MAX_PART_SIZE, max_workers=max_workers, prepared_params={ "create_multipart_upload": plan.create_params, "upload_part_copy": plan.part_params, "complete_multipart_upload": plan.complete_params, "abort_multipart_upload": plan.abort_params, }, ) try: session.start() for part_number, range_ in enumerate(plan.ranges, 1): session.wait_ready() session.submit_copy(part_number, plan.source, range_) completed = session.complete() except BaseException: try: session.abort() except KeyboardInterrupt: if (upload := session.upload) is not None: _logger.warning( f"Interrupted the abort of multipart upload {upload.upload_id} " f"to s3://{upload.bucket}/{upload.key}." ) raise except Exception: # Logged instead of raised, so that the error that made # the copy fail is raised. upload = cast(S3MultipartUpload, session.upload) _logger.exception( f"Failed to abort multipart upload {upload.upload_id} " f"to s3://{upload.bucket}/{upload.key}." ) raise # The destination is committed, so its annotations are copied to the # end also when an enclosing batch has stopped. with unstoppable_retries(): for name in plan.annotations: self.core.copy_object_annotation( name, plan.source, plan.destination, completed.version_id, completed.etag, **kwargs, )
[docs] def pipe(self, path, value=None, batch_size=None, **kwargs) -> None: """Write bytes into one or more paths. As fsspec's ``pipe()``, except that the values of a dict are written in parallel with ``pipe_file()``, as ``copy()`` copies its files. Args: path: S3 path to write ``value`` to, or a dict of the values to write by path. value: The bytes to write to a single path; ignored with a dict. batch_size: Number of writes to run at the same time, as in ``copy()``. **kwargs: Additional parameters passed to ``pipe_file()``. Raises: ValueError: If ``path`` is neither a string nor a dict, two paths of a dict name the same object, or ``batch_size`` is neither positive, -1, nor None. """ if not isinstance(path, dict): path = stringify_path(path) limit = self._batch_limit(batch_size) if isinstance(path, str): self.pipe_file(self._strip_protocol(path), value, **kwargs) return if not isinstance(path, dict): raise ValueError("path must be str or dict") items = self._pipe_items(path) if not items: return concurrency = min(limit, len(items)) call_kwargs = self._batch_kwargs(concurrency, kwargs) def pipe_item(item: tuple[str, Any]) -> None: """Write a value. Args: item: The path and its value. """ self.pipe_file(*item, **call_kwargs) self._run_batch(pipe_item, items, concurrency)
def _pipe_items(self, values: Mapping[Any, Any]) -> list[tuple[str, Any]]: """Return the paths and values of a ``pipe()`` of a dict, which are written in parallel. Args: values: The values to write by path. Returns: The paths without the protocol, and their values. Raises: ValueError: If two paths name the same object. """ items = [(self._strip_protocol(k), v) for k, v in values.items()] if len({k for k, _ in items}) != len(items): raise ValueError("Cannot pipe several values to the same destination.") return items
[docs] def pipe_file( self, path: str | os.PathLike[str], value: bytes | bytearray | memoryview, mode: str = "overwrite", **kwargs, ) -> None: """Write bytes into the path. Writes data up to the block size with a single PutObject request instead of the inherited ``open()`` + ``write()`` path. Larger data and writes inside an fsspec transaction go through the buffered path, which uploads the data as a parallel multipart upload and keeps the deferred-commit semantics of transactions. A write that fails on that path leaves the existing object unchanged. Both paths write to the path without a trailing slash, as ``open()`` does. Args: path: S3 path (s3://bucket/key) to write to. value: The bytes to write. mode: "overwrite" (default) or "create". With "create", raise FileExistsError when the object already exists, including one created during the write, which is not replaced. **kwargs: Additional parameters passed to the PutObject API (e.g., ContentType, StorageClass) on the single-request path. The ``block_size``, ``max_workers``, and ``s3_additional_kwargs`` parameters of the ``open()`` path are also accepted, and so is ``compression``: the codec of ``open()`` to compress the value with before it is uploaded, or ``"infer"`` to take it from the extension of the path. Raises: FileExistsError: If the mode is "create" and the path already exists, or an object is created at it before the write is committed. ValueError: If the path does not contain a key or specifies a version, if the compression is not supported, or if the data takes more than ``S3Core.MULTIPART_UPLOAD_MAX_PARTS`` blocks. """ # Normalized as open() normalizes it, so that the key written, and # the codec that "infer" takes from its extension, do not depend on # the path that the size of the value selects. path = self._strip_protocol(path) compression = get_compression(path, kwargs.pop("compression", None)) if compression is not None: # Compressed up front, so that every path uploads the compressed # bytes, and open() returns the file instead of a wrapper. value = CompressedBuffer.compress(value, compression) block_size = kwargs.get("block_size") or self.default_block_size # The size in bytes; the length of a memoryview counts its items. size = memoryview(value).nbytes self.core.check_multipart_upload_size(size, block_size) if self._intrans or size > min(block_size, self.core.MULTIPART_UPLOAD_MAX_PART_SIZE): # Defer to the buffered open() path, which keeps the # deferred-commit semantics of fsspec transactions and uploads # large data as a parallel multipart upload. self.open(path, "xb" if mode == "create" else "wb", **kwargs)._write_and_close(value) return s3_path = S3Path.parse(path) if s3_path.version_id: raise ValueError("Cannot write to the file with the version specified.") if not s3_path.key: raise ValueError("Cannot write to a bucket.") kwargs.pop("block_size", None) kwargs.pop("max_workers", None) request_kwargs = { **self.core.operation_params("put_object", self.s3_additional_kwargs), **kwargs.pop("s3_additional_kwargs", {}), **kwargs, } if mode == "create": # Checked up front, as open() does in "xb" mode, and with # IfNoneMatch for an object created since. if self._object_exists(path, self._get_lookup_kwargs(request_kwargs)): raise FileExistsError(path) request_kwargs["IfNoneMatch"] = "*" if not isinstance(value, bytes): # Accept bytes-like values (bytearray, memoryview) as the # buffered path does. value = bytes(value) raise_if_stopped(f"put {s3_path.uri}") self.core.put_object(s3_path, value, **request_kwargs) self.invalidate_cache(path)
[docs] def cat(self, path, recursive=False, on_error="raise", batch_size=None, **kwargs): """Read the contents of one or more paths. As fsspec's ``cat()``, except that the objects are read in parallel with ``cat_file()``, as ``copy()`` copies its files. With ``on_error="raise"``, an error starts no further read and is raised once the running reads have ended. Args: path: S3 path, glob pattern, or list of them. recursive: Whether to read the files below the directories. on_error: ``"raise"`` to raise an error of a read, ``"omit"`` to leave out the paths whose read failed, or ``"return"`` to return the error in place of their contents. batch_size: Number of reads to run at the same time, as in ``copy()``. **kwargs: Additional parameters passed to ``expand_path()`` and ``cat_file()``. Returns: The contents of the path, or a dict of the contents by path if there are several paths or the path was expanded. Raises: ValueError: If ``batch_size`` is neither positive, -1, nor None. """ limit = self._batch_limit(batch_size) targets = self._first_targets(self._expand_paths(path, recursive=recursive, **kwargs)) paths = list(targets) if not (len(paths) > 1 or isinstance(path, list) or paths[0] != self._strip_protocol(path)): return self.cat_file(targets[paths[0]], **kwargs) outcomes: list[Any] | None = None contents: dict[str, Any] | None = None outcome = None try: outcomes = self._run_batch( functools.partial(self.cat_file, **kwargs), list(targets.values()), min(limit, len(paths)), counts=None if on_error == "raise" else lambda e: False, ).outcomes contents = {} for p, outcome in zip(paths, outcomes, strict=True): if on_error != "omit" or not isinstance(outcome, Exception): contents[p] = outcome return contents except BaseException: # Not kept by this frame, which the traceback of an interrupt # keeps (F5). contents = outcomes = outcome = None raise
@staticmethod def _first_targets(expanded: list[tuple[str, str | S3Path]]) -> dict[str, str | S3Path]: """Return the first path that each name of an expansion names. An explicit version and a listed key that ends in the same query share a name. The contents are returned by name, so a name is read once, as the first path that it names, which is the path that the caller passed. Args: expanded: The names and the paths that they name, as :meth:`_expand_paths` returns them. Returns: The paths by name, in the order of the names. """ targets: dict[str, str | S3Path] = {} for name, target in expanded: targets.setdefault(name, target) return targets
[docs] def cat_ranges( self, paths, starts, ends, max_gap=None, on_error="return", batch_size=None, **kwargs ) -> list[bytes | Exception]: """Read byte ranges of one or more paths. As fsspec's ``cat_ranges()``, except that the ranges are read in parallel with ``cat_file()``, as ``copy()`` copies its files. Unless ``on_error`` is ``"return"``, an error starts no further read and is raised once the running reads have ended. Args: paths: The S3 paths, one per range. starts: The start offset of each range, or one for all of them. ends: The end offset of each range, or one for all of them. max_gap: Not supported; must be None. on_error: ``"return"`` to return the error of a read in place of its bytes; otherwise the error is raised. batch_size: Number of reads to run at the same time, as in ``copy()``. **kwargs: Additional parameters passed to ``cat_file()``. Returns: The bytes of each range, or the error of its read. Raises: NotImplementedError: If ``max_gap`` is given. TypeError: If ``paths`` is not a list. ValueError: If the numbers of paths, starts, and ends differ, or if ``batch_size`` is neither positive, -1, nor None. """ if max_gap is not None: raise NotImplementedError if not isinstance(paths, list): raise TypeError if not isinstance(starts, list): starts = [starts] * len(paths) if not isinstance(ends, list): ends = [ends] * len(paths) if len(starts) != len(paths) or len(ends) != len(paths): raise ValueError limit = self._batch_limit(batch_size) if not paths: return [] def cat_range(item: tuple[str, int | None, int | None]) -> bytes: """Read a range. Args: item: The path, start, and end. Returns: The bytes of the range. """ return self.cat_file(*item, **kwargs) return cast( list[bytes | Exception], self._run_batch( cat_range, list(zip(paths, starts, ends, strict=True)), min(limit, len(paths)), counts=None if on_error != "return" else lambda e: False, ).outcomes, )
[docs] def cat_file( self, path: str | os.PathLike[str] | S3Path, start: int | None = None, end: int | None = None, **kwargs, ) -> bytes: """Read the contents of an S3 object with GetObject. ``start`` and ``end`` select bytes like a slice of the object: an empty range, or one that starts at or past the end of the object, returns ``b""``, and an end past the object reads up to its end. Non-negative offsets are sent to S3 as they are, and so is a negative ``start`` without an ``end``, as a suffix range of the last bytes. Other negative offsets are resolved against the size from :meth:`info`, which also checks that the object exists for an empty range and receives the lookup parameters among ``kwargs``. Args: path: S3 path (s3://bucket/key) of the object, or an ``S3Path``, whose bucket, key and version are read as they are, such as a key that a listing returns. start: Byte offset to start reading at. A negative value counts from the end of the object. end: Byte offset to stop reading at (exclusive). A negative value counts from the end of the object. **kwargs: Additional parameters passed to the GetObject API, except ``version_id``: the version ID to read when the path has none. Returns: The bytes read from the object. Raises: FileNotFoundError: If the path has no key or the key does not exist. """ if isinstance(path, S3Path): return self._cat_object(path, start=start, end=end, **kwargs) path = stringify_path(path) return self._cat_object(S3Path.parse(path), start=start, end=end, name=path, **kwargs)
def _cat_object( self, path: S3Path, start: int | None = None, end: int | None = None, name: str | None = None, **kwargs, ) -> bytes: """Read the contents of the object of a path, as :meth:`cat_file` describes. Args: path: The path of the object, whose bucket, key and version are used as they are. start: Byte offset to start reading at, as in :meth:`cat_file`. end: Byte offset to stop reading at, as in :meth:`cat_file`. name: The path string that the caller passed, which ``info()`` looks up for the size of the object of a negative offset or an empty range, and which a missing object is reported by; None to look up ``path`` itself. **kwargs: Additional parameters passed to the GetObject API, and ``version_id``, as in :meth:`cat_file`. Returns: The bytes read from the object. Raises: FileNotFoundError: If the path has no key or the key does not exist. """ if not path.key: raise FileNotFoundError(name or str(path)) version_id = kwargs.pop("version_id", None) if path.version_id: version_id = path.version_id ranges: tuple[int, int | None] | None = None if start is not None and start < 0 and end is None: # S3 returns the last bytes, or the whole object when it is # shorter, without the size of the object. ranges = (start, None) else: if (start is not None and start < 0) or ( end is not None and (end < 0 or (start or 0) >= end) ): # A negative offset needs the size of the object, and an # empty range sends no GetObject request that would report a # missing object. lookup_kwargs = self._get_lookup_kwargs(kwargs) info = self.info( name if name is not None else path, version_id=version_id, **lookup_kwargs ) if ( info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY or info.key != path.key ): # There is no object to read, as GetObject reports for # the other ranges, or info() describes the key without # the trailing slash of this one. raise FileNotFoundError(name or str(path)) start, end, _ = slice(start, end).indices(info.get("size", 0)) if start is not None or end is not None: start = start or 0 if end is not None and start >= end: # S3 would return the whole object for an empty range. return b"" ranges = (start, end) try: return self.core.get_object(path.with_version_id(version_id), ranges, **kwargs) except OSError as e: if ranges and S3ClientError.code_of(e) == "InvalidRange": # The range starts at or past the end of the object. return b"" raise
[docs] def put( self, lpath, rpath, recursive=False, callback=_DEFAULT_CALLBACK, maxdepth=None, batch_size=None, **kwargs, ) -> None: """Copy files from the local filesystem to S3. As fsspec's ``put()``, except that the files are uploaded in parallel with ``put_file()``, as ``copy()`` copies its files. A failed upload starts no further one, the running ones send no further part and do not commit, and the error is raised once they have ended. Args: lpath: Local source path, glob pattern, or list of them. rpath: S3 destination path, or list of paths when ``lpath`` is a list. recursive: Whether to copy the directories with their contents. callback: Progress callback. It and its branches for the files are used from the threads that upload them, one at a time. maxdepth: Maximum depth of a recursive copy. batch_size: Number of uploads to run at the same time, as in ``copy()``. **kwargs: Additional parameters passed to ``put_file()``. Raises: ValueError: If two files have the same destination, or ``batch_size`` is neither positive, -1, nor None. """ lpath, rpath = self._stringify_paths(lpath), self._stringify_paths(rpath) limit = self._batch_limit(batch_size) pairs = self._put_pairs(lpath, rpath, recursive=recursive, maxdepth=maxdepth, **kwargs) if not pairs: return concurrency = min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) locked = _S3LockedCallback(callback, threading.Lock()) def put_pair(pair: tuple[str, str]) -> None: """Upload a pair with a branch of the callback. Args: pair: The local source and its S3 destination. """ lp, rp = pair with locked.branched(lp, rp) as child: self.put_file(lp, rp, callback=child, **call_kwargs) locked.relative_update(1) locked.set_size(len(pairs)) self._run_batch(put_pair, pairs, concurrency)
def _put_pairs( self, lpath, rpath, recursive=False, maxdepth=None, **kwargs ) -> list[tuple[str, str]]: """Pair the local sources of ``put()`` with their S3 destinations as fsspec does. The files are uploaded in parallel, so no two files may have the same destination; a local directory uploads nothing. Args: lpath: Local source path, glob pattern, or list of them. rpath: S3 destination path, or list of paths when ``lpath`` is a list. recursive: Whether to include the directories with their contents. maxdepth: Maximum depth of a recursive copy. **kwargs: Additional arguments passed to the ``expand_path()`` of the local filesystem. Returns: The local paths, including the directories of a recursive copy, with their destinations without the protocol. Raises: ValueError: If two files have the same destination. """ if isinstance(lpath, list) and isinstance(rpath, list): pairs = [(lp, self._strip_protocol(rp)) for lp, rp in zip(lpath, rpath, strict=False)] else: source_is_str = isinstance(lpath, str) if source_is_str: lpath = make_path_posix(lpath) local = LocalFileSystem() lpaths = local.expand_path(lpath, recursive=recursive, maxdepth=maxdepth, **kwargs) if source_is_str and (not recursive or maxdepth is not None): # A string source without recursion does not copy directories. lpaths = [p for p in lpaths if not (trailing_sep(p) or local.isdir(p))] if not lpaths: return [] dest_is_dir = isinstance(rpath, str) and (trailing_sep(rpath) or self.isdir(rpath)) # A list of destinations or one path, also path-like, as in fsspec. rpath = self._strip_protocol(rpath) exists = source_is_str and ( (has_magic(lpath) and len(lpaths) == 1) or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) ) rpaths = other_paths(lpaths, rpath, exists=exists, flatten=not source_is_str) pairs = list(zip(lpaths, rpaths, strict=True)) # put_file() uploads nothing for a local directory. files = [rp for lp, rp in pairs if not os.path.isdir(lp)] if len(set(files)) != len(files): raise ValueError("Cannot put several paths to the same destination.") return pairs
[docs] def put_file( self, lpath: str | os.PathLike[str], rpath: str | os.PathLike[str], callback=_DEFAULT_CALLBACK, mode: str = "overwrite", **kwargs, ): """Upload a local file to S3. Uploads a file from the local filesystem to an S3 location. Supports automatic content type detection based on file extension and provides progress callback functionality. An upload that fails before it is completed, including one of a local file that cannot be read, leaves the existing object unchanged. Args: lpath: Local file path to upload. rpath: S3 destination path (s3://bucket/key). callback: Progress callback for tracking upload progress. mode: "overwrite" (default) or "create". With "create", the file is written as with ``open()`` in ``xb`` mode: raise FileExistsError when the object already exists, including one created during the upload, which is not replaced. **kwargs: Additional S3 parameters (e.g., ContentType, StorageClass). The ``block_size``, ``max_workers``, and ``s3_additional_kwargs`` parameters of ``open()`` are also accepted. Raises: FileExistsError: If the mode is "create" and the path already exists, or an object is created at it before the upload is committed. ValueError: If the file takes more than ``S3Core.MULTIPART_UPLOAD_MAX_PARTS`` blocks. Note: Directories are not supported for upload. If lpath is a directory, the method returns without performing any operation. Bucket-only destinations (without key) are also not supported. """ rpath = stringify_path(rpath) if os.path.isdir(lpath): # No support for directory uploads. return if not S3Path.parse(rpath).key: # No support for bucket copy. return size = os.path.getsize(lpath) block_size = kwargs.pop("block_size", None) or self.default_block_size max_workers = kwargs.pop("max_workers", self.max_workers) # The other parameters are S3 request parameters, as in pipe_file(). s3_additional_kwargs = {**kwargs.pop("s3_additional_kwargs", {}), **kwargs} self.core.check_multipart_upload_size(size, block_size) callback.set_size(size) if "ContentType" not in {**self.s3_additional_kwargs, **s3_additional_kwargs}: content_type, _ = mimetypes.guess_type(lpath) if content_type is not None: s3_additional_kwargs["ContentType"] = content_type # The local file is opened first, so that an unreadable one fails # before the remote file is opened. with open(lpath, "rb") as local: self.open( rpath, "xb" if mode == "create" else "wb", block_size=block_size, max_workers=max_workers, s3_additional_kwargs=s3_additional_kwargs, )._write_file_and_close(local, callback) self.invalidate_cache(rpath)
[docs] def get_file( self, rpath: str | os.PathLike[str] | S3Path, lpath=None, callback=_DEFAULT_CALLBACK, outfile=None, **kwargs, ): """Download an S3 file to local filesystem. The object is read in waves of parallel range requests of the block size, at most ``max_workers`` of them at once, but no more than the request pool of the filesystem runs, and the bodies of each wave are written in order before the next wave is read, so that the download holds at most one wave of bodies; a thread of the pool that has just read a range drops its body when its call returns (see :meth:`S3File._download`). The reads are conditional on the version and the ETag of the object looked up when the download starts, as with ``open()``, so that an object replaced during the download fails it instead of mixing two objects, unless the download reads a version other than ``null``, which a replacement does not change. A failed download leaves the part written so far in a local file ``lpath``, as fsspec's ``AbstractFileSystem.get_file()`` does. As with fsspec's ``AbstractFileSystem.get_file()``, a directory ``rpath`` creates the local directory ``lpath``, and the parent directories of a local file ``lpath`` are created as needed. Args: rpath: S3 source path (s3://bucket/key), or an ``S3Path``, whose bucket, key and version are used as they are, such as a key that a listing returns. lpath: Local destination path, or a file-like object to write to. Not needed when ``outfile`` is given. callback: Progress callback for tracking download progress, with the size of the object. outfile: A file-like object to write to instead of ``lpath``. **kwargs: The arguments of ``open()``, such as ``block_size``, ``max_workers``, ``version_id``, and ``s3_additional_kwargs``, and additional S3 parameters. ``cache_type`` and ``cache_options`` are not used. With ``compression``, the object is decompressed into the destination, read in order through ``open()`` with a block cache, whose block size, if not positive, is that of the filesystem, or ``DEFAULT_BLOCK_SIZE`` if that is not positive either, and the size of the callback is unknown. """ s3_path = S3Path.of(rpath if isinstance(rpath, S3Path) else self._strip_protocol(rpath)) if outfile is None and isfilelike(lpath): outfile = lpath elif outfile is None and self._is_directory_source(rpath, kwargs.get("version_id")): os.makedirs(lpath, exist_ok=True) return # The codec, if any, as fsspec's open() resolves it, with "infer" # from the key without the version. codec = get_compression(s3_path.key or "", kwargs.pop("compression", None)) # Not the caller's cache: one such as "all" would read the whole # object when the file is opened, and "background" reads outside # the stop scopes of the caller. Without a codec, the object is read # in ranges without a cache; with one, the decompressor's small reads # go through a block cache, which needs a positive block size. kwargs.pop("cache_options", None) kwargs["cache_type"] = "none" if codec is None else "bytes" if codec is not None and not ( isinstance(kwargs.get("block_size"), int) and kwargs["block_size"] > 0 ): kwargs["block_size"] = ( self.default_block_size if self.default_block_size > 0 else self.DEFAULT_BLOCK_SIZE ) # The remote file is opened first so that no local file is created # when open() finds no object at the path. with contextlib.ExitStack() as stack: remote = stack.enter_context( self._open(rpath, "rb", **kwargs) if isinstance(rpath, S3Path) else self.open(rpath, "rb", **kwargs) ) reader: Any = None if codec is not None: # As fsspec's open() wraps the file, but closed with it. reader = stack.enter_context(compr[codec](remote, mode="r")) if outfile is None: # Not abspath(), which would resolve ".." before symlinks. if parent := os.path.dirname(lpath): os.makedirs(parent, exist_ok=True) outfile = stack.enter_context(open(lpath, "wb")) if reader is None: callback.set_size(remote.size) remote._download(outfile, callback) return # The size of the decompressed data is unknown. callback.set_size(None) while data := reader.read(remote.blocksize): # A block read when the stop came is not written. raise_if_stopped(f"write the contents of {rpath}") outfile.write(data) callback.relative_update(len(data))
[docs] def checksum(self, path: str | os.PathLike[str], **kwargs): """Get checksum for S3 object or directory. Computes a checksum for the specified S3 path. For individual objects, returns the ETag converted to an integer. For directories, returns a checksum based on the directory's tokenized representation. Args: path: S3 path (s3://bucket/key) to get checksum for. **kwargs: Additional arguments including: refresh: If True, refresh cached info before computing checksum. Returns: Integer checksum value derived from S3 ETag or directory token. Note: For multipart uploads, ETag format is different and only the first part before the dash is used for checksum calculation. """ refresh = kwargs.pop("refresh", False) info = self.info(path, refresh=refresh) if info.get("type") != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: return int(info.get("etag").strip('"').split("-")[0], 16) return int(tokenize(info), 16)
[docs] def sign(self, path: str | os.PathLike[str], expiration: int = 3600, **kwargs): """Generate a presigned URL for S3 object access. Creates a presigned URL that allows temporary access to an S3 object without requiring AWS credentials. Useful for sharing files or providing time-limited access to resources. Args: path: S3 path (s3://bucket/key) to generate URL for. expiration: URL expiration time in seconds. Defaults to 3600 (1 hour). **kwargs: Additional parameters including: client_method: S3 operation ('get_object', 'put_object', etc.). Defaults to 'get_object'. Additional parameters passed to the S3 operation. Returns: Presigned URL string that provides temporary access to the S3 object. Example: >>> fs = S3FileSystem() >>> url = fs.sign("s3://my-bucket/file.txt", expiration=7200) >>> # URL valid for 2 hours >>> >>> # Generate upload URL >>> upload_url = fs.sign( ... "s3://my-bucket/upload.txt", ... client_method="put_object" ... ) """ path = stringify_path(path) client_method = kwargs.pop("client_method", "get_object") return self.core.generate_presigned_url( S3Path.parse(path), client_method, expiration, **kwargs )
[docs] def metadata(self, path: str | os.PathLike[str], **kwargs) -> S3Metadata: """Return the metadata of the path. Args: path: S3 path (s3://bucket/key) to get metadata for. **kwargs: Additional parameters passed to the HeadObject API. Returns: S3Metadata, which behaves as a read-only mapping of the user-defined metadata (``x-amz-meta-*``) and exposes the system-defined metadata (content type, encryption settings, etc.) as typed properties. """ path = stringify_path(path) s3_path = S3Path.parse(path) if not s3_path.key: raise ValueError("Cannot get metadata of a bucket.") _logger.debug(f"Head object metadata: {s3_path.uri}") return self.core.head_object(s3_path, **kwargs)
[docs] def getxattr(self, path: str | os.PathLike[str], attr_name: str, **kwargs) -> str | None: """Get an attribute from the user-defined metadata of the path. Args: path: S3 path (s3://bucket/key) to get the attribute for. attr_name: The name of the attribute. **kwargs: Additional parameters passed to :meth:`metadata`. Returns: The value of the attribute, or None if the attribute is not set. """ return self.metadata(path, **kwargs).get(attr_name)
[docs] def setxattr( self, path: str | os.PathLike[str], copy_kwargs: dict[str, Any] | None = None, **kw_args ) -> None: """Set the user-defined metadata of the path. S3 does not allow updating the metadata of an existing object in place, so the object is copied onto itself with the REPLACE metadata directive. Note that this rewrites the object and updates its last-modified time. The system-defined metadata (e.g., ``ContentType`` and ``CacheControl``), the storage class, and the server-side encryption algorithm and KMS key of the object are kept. HeadObject does not return the KMS encryption context, and an ``Expires`` value that botocore cannot parse as a date is not kept. Args: path: S3 path (s3://bucket/key) to set metadata for. A path with a version ID is rejected, since copying a version onto the key would replace the current object with it. copy_kwargs: Additional parameters to use for the underlying CopyObject API call. They take precedence over the kept system-defined metadata and storage class. Any encryption parameter replaces all kept encryption settings. The lookup of the object receives ``RequestPayer``, and the source's expected bucket owner and SSE-C parameters (e.g., ``CopySourceSSECustomerKey``) under the names of HeadObject (see :meth:`S3Core.copy_source_params`). **kw_args: Key-value pairs to set, where the values must be strings. The keys are used as-is; names that are not valid Python identifiers (e.g., containing hyphens) can be passed by unpacking a dictionary. Does not alter existing fields, unless the field appears here - if the value is None, delete the field. Example: >>> fs = S3FileSystem() >>> fs.setxattr("s3://bucket/key", attribute1="value1") >>> fs.setxattr("s3://bucket/key", **{"attribute-2": "value2"}) Raises: ValueError: If the path is a bucket or has a version ID. """ path = stringify_path(path) s3_path = S3Path.parse(path) if not s3_path.key: raise ValueError("Cannot set metadata of a bucket.") if s3_path.version_id: raise ValueError("Cannot set metadata of a version.") copy_kwargs = copy_kwargs or {} # Looked up with the parameters that authorize the access to the # object as the source of the copy, such as its customer-provided key. head = self.metadata(path, **self.core.copy_source_params(copy_kwargs)) metadata = dict(head) for k, v in kw_args.items(): if v is None: metadata.pop(k, None) else: metadata[k] = v self.core.replace_object_metadata(s3_path, head, metadata, **copy_kwargs) self.invalidate_cache(path)
[docs] def get_tags(self, path: str | os.PathLike[str]) -> dict[str, str]: """Retrieve the tag key/values for the given path. Args: path: S3 path (s3://bucket/key) to get tags for. Returns: Dictionary mapping tag keys to tag values. """ path = stringify_path(path) s3_path = S3Path.parse(path) if not s3_path.key: raise ValueError("Cannot get tags of a bucket.") return self.core.get_object_tagging(s3_path)
[docs] def put_tags(self, path: str | os.PathLike[str], tags: dict[str, str], mode: str = "o") -> None: """Set the tags for the given existing key. Tags are a str:str mapping that can be attached to any key, distinct from the user-defined metadata, which is usually set at key creation time. See https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html Args: path: S3 path (s3://bucket/key) of the existing key to attach tags to. tags: Tags to apply. mode: One of 'o' or 'm'. 'o' will over-write any existing tags. 'm' will merge in new tags with existing tags, which incurs two remote calls. """ path = stringify_path(path) s3_path = S3Path.parse(path) if not s3_path.key: raise ValueError("Cannot put tags of a bucket.") if mode == "m": new_tags = {**self.core.get_object_tagging(s3_path), **tags} elif mode == "o": new_tags = tags else: raise ValueError(f"Mode must be {{'o', 'm'}}, not {mode}.") self.core.put_object_tagging(s3_path, new_tags)
[docs] def chmod( self, path: str | os.PathLike[str], acl: str, recursive: bool = False, **kwargs ) -> None: """Set the Access Control on a bucket/key. See https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl Args: path: S3 path (s3://bucket or s3://bucket/key) to set the ACL on. acl: The value of the canned ACL to apply. recursive: Whether to apply the ACL to all keys below the given path too, and to the object at the key, if any. The keys are updated in parallel; once one fails, the updates that have not started are cancelled, and the running ones make no further attempt. A path with a version ID updates that version only. **kwargs: Additional parameters passed to the PutObjectAcl or PutBucketAcl API. """ path = stringify_path(path) s3_path = S3Path.parse(path) # Validate before any ACL is applied so that a recursive call cannot # partially apply object ACLs and then fail on the bucket ACL. if not s3_path.key and acl not in self.core.BUCKET_ACLS: raise ValueError(f"ACL not in {self.core.BUCKET_ACLS}.") if s3_path.key and acl not in self.core.OBJECT_ACLS: raise ValueError(f"ACL not in {self.core.OBJECT_ACLS}.") if recursive and not (s3_path.key and s3_path.version_id): found = self._find_paths(path, withdirs=False) items: list[tuple[str | S3Path, bool]] = [(target, False) for _, target in found] name = self._strip_protocol(path) # The key of the path as find() names it ("bucket//" is the bucket). if S3Path.parse(name).key and found and name not in {f.name for f, _ in found}: # Objects lie below the key, so find() has not looked the key # itself up; it may be an object too, which is skipped if missing. items.append((name, True)) self._map(lambda item: self._chmod_object(*item, acl, **kwargs), items) if s3_path.key: return if s3_path.key: self.core.put_object_acl(s3_path, acl, **kwargs) else: self.core.put_bucket_acl(s3_path.bucket, acl, **kwargs)
def _chmod_object(self, path: str | S3Path, missing_ok: bool, acl: str, **kwargs) -> None: """Apply a canned ACL to an object in a recursive ``chmod``. Args: path: S3 path of the object. missing_ok: Whether to skip the object if it does not exist. acl: The canned ACL, one of ``OBJECT_ACLS``. **kwargs: Additional parameters passed to the PutObjectAcl API. Raises: FileNotFoundError: If the object does not exist and ``missing_ok`` is False. """ try: self.core.put_object_acl( S3Path.parse(path) if isinstance(path, str) else path, acl, **kwargs ) except FileNotFoundError: if not missing_ok: raise
[docs] def list_multipart_uploads(self, path: str | os.PathLike[str]) -> list[S3MultipartUpload]: """List in-progress (incomplete) multipart uploads in a bucket. Incomplete multipart uploads continue to accrue storage costs until they are completed or aborted. Use :meth:`clear_multipart_uploads` to abort all of them. Args: path: S3 bucket or key path (e.g., "bucket", "s3://bucket" or "s3://bucket/prefix"). If the path contains a key, only the uploads to that key and to the keys under ``key/`` are listed, not those to sibling keys that merely start with the same characters (e.g., ``prefix2/a``). Returns: List of S3MultipartUpload instances describing the in-progress multipart uploads. """ path = stringify_path(path) s3_path = S3Path.parse(path) # S3 matches Prefix as a plain string, so the uploads are filtered to # the key itself and the keys under it. prefix = f"{s3_path.key.rstrip('/')}/" if s3_path.key else "" _logger.debug(f"List multipart uploads: {s3_path.uri}") return [ upload for page in self.core.list_multipart_uploads(s3_path.bucket, prefix=s3_path.key or None) for upload in page.uploads if upload.key is not None and (upload.key == s3_path.key or upload.key.startswith(prefix)) ]
[docs] def object_version_info( self, path: str | os.PathLike[str], delete_markers: bool = False, **kwargs ) -> list[S3ObjectVersion]: """List the versions of the object or of the objects under the path. A key path without a trailing slash selects that key if it has any versions or delete markers, and otherwise the keys under ``key/``. The choice does not depend on ``delete_markers``, so a key that has only delete markers yields no versions without them. A key path with a trailing slash selects the keys under it, and a bucket path selects all the keys in the bucket. Sibling keys that merely start with the same characters (e.g., ``key.bak``) are never included. Args: path: S3 path (s3://bucket/key or a key prefix) to list the versions for. delete_markers: Whether to include delete markers in the result. **kwargs: Additional parameters passed to the ListObjectVersions API. Returns: List of S3ObjectVersion instances describing the versions. """ path = stringify_path(path) s3_path = S3Path.parse(path) # S3 matches Prefix as a plain string, so the versions are filtered to # the key itself or the keys under it. prefix = f"{s3_path.key.rstrip('/')}/" if s3_path.key else "" _logger.debug(f"List object versions: {s3_path.uri}") versions: list[S3ObjectVersion] = [] # Explicit markers start the listing, which the pages then advance. key_marker = kwargs.pop("KeyMarker", None) version_id_marker = kwargs.pop("VersionIdMarker", None) for page in self.core.list_object_versions( s3_path.bucket, prefix=s3_path.key or "", key_marker=key_marker, version_id_marker=version_id_marker, **kwargs, ): versions.extend(page.versions) # Delete markers are kept until the key is chosen, so that the # choice is the same with and without them. versions.extend(page.delete_markers) # botocore decodes the keys only when it sets EncodingType itself, so # the keys of an explicit EncodingType="url" are decoded for matching. url_encoded = kwargs.get("EncodingType") == "url" keys = [unquote_plus(v.key) if url_encoded else v.key for v in versions] if s3_path.key and not s3_path.key.endswith("/") and s3_path.key in keys: selected = [v for v, k in zip(versions, keys, strict=True) if k == s3_path.key] else: selected = [v for v, k in zip(versions, keys, strict=True) if k.startswith(prefix)] return [v for v in selected if delete_markers or not v.is_delete_marker]
[docs] def clear_multipart_uploads(self, path: str | os.PathLike[str]) -> None: """Abort any incomplete multipart uploads in the bucket. Uploads completed or aborted after listing are already cleared. All abort results are checked before another abort error is raised. Args: path: S3 bucket or key path (e.g., "bucket", "s3://bucket" or "s3://bucket/prefix"). If the path contains a key, only the uploads to that key and to the keys under ``key/`` are aborted, as listed by :meth:`list_multipart_uploads`. """ uploads = self.list_multipart_uploads(path) if not uploads: return # Every abort runs also after another failed, and an interrupt cancels # those that have not started. results = self._map( self.core.abort_multipart_upload, uploads, stop_on_failure=False, counts=self._is_not_no_such_upload, ) if results.failure is not None: raise results.failure
@staticmethod def _is_not_no_such_upload(error: BaseException) -> bool: """Return whether an error of an abort is other than an upload that no longer exists. Args: error: The error of ``abort_multipart_upload``. Returns: False for the ``FileNotFoundError`` of a ``NoSuchUpload`` error. """ return not ( isinstance(error, FileNotFoundError) and S3ClientError.code_of(error) == "NoSuchUpload" )
[docs] def created(self, path: str | os.PathLike[str]) -> datetime: """Return the creation time of the path. Returns the same value as :meth:`modified`. Args: path: S3 path (s3://bucket/key). Returns: The last-modified time of the object. """ return self.modified(path)
[docs] def modified(self, path: str | os.PathLike[str]) -> datetime: """Return the last-modified time of the path. Args: path: S3 path (s3://bucket/key). Returns: The ``last_modified`` field from :meth:`info`, which is None for buckets and directories. """ info = self.info(path) return cast(datetime, info.get("last_modified"))
[docs] def invalidate_cache(self, path: str | os.PathLike[str] | None = None) -> None: """Remove the cached entries of the path and its parent paths. The cached bucket listing is removed only by the root path (``""``, ``"/"`` or ``"s3://"``), not by the paths of buckets or keys. A version-qualified path invalidates the lookup of the version, and also the object path without the version, because deleting or copying a version can change the current version of the object. It also invalidates the keys that end in its query, in each spelling that names the version, such as the keys ``a/x?versionId=v1`` and ``a/x?version_id=v1`` of ``bucket/a/x?versionId=v1``, which a listing names as the paths. Only the path itself is parsed for a version: its parent paths are directories, named as they are listed, such as ``bucket/a/dir?versionId=v1`` for the key ``a/dir?versionId=v1/x``. A path that ends in slashes is invalidated both with and without them, since lookups strip them while a key such as ``dir//`` is listed in the directory ``bucket/dir/``. Args: path: The path to invalidate. If None, clear the whole cache. """ if path is None: self.dircache.clear() return raw = stringify_path(path) stripped = self._strip_protocol(raw) if not stripped: self._evict_cache("") return trailing = len(raw) - len(raw.rstrip("/")) for spelling in (stripped, stripped + "/" * trailing) if trailing else (stripped,): base, version_id = S3Path.split_version_id(spelling) if not version_id: # Also the directory of a trailing slash. self._invalidate_name(spelling) continue # The lookup of the version, as _lookup_cache_key() keys it, the # keys that end in the query in each spelling that names the # version, and the object path without the version. self._evict_lookup((base, _VERSION_CACHE_KEY, version_id)) for query in ("versionId", "versionID", "versionid", "version_id"): self._invalidate_name(f"{base}?{query}={version_id}") self._invalidate_name(base)
def _invalidate(self, path: S3Path) -> None: """Remove the cached entries of a path and of the directories above it. A path with a version removes the lookup of that version (see :meth:`_lookup_cache_key`), and then the entries of its key. The names of the directories above it are cut at their last slash without parsing them, so that a directory named with a version ID query or an empty segment is invalidated as it is listed. Args: path: The path of the bucket or the object. """ if path.version_id: self._evict_lookup(self._lookup_cache_key(path)) self._invalidate_name(path.name) def _invalidate_name(self, name: str) -> None: """Remove the cached entries of a name and of the directories above it. The names of the directories are cut at their last slash without parsing them, so that a directory named with a version ID query or an empty segment is invalidated as it is listed. Args: name: The name of the bucket, the directory or the object, without the protocol. """ while name: # _ls_prefix caches listings under (name, delimiter). for cache_key in ((name, "/"), (name, "")): self._evict_cache(cache_key) self._evict_lookup(name) name = name.rpartition("/")[0] def _evict_lookup(self, key: str | tuple[str, str, str]) -> None: """Remove the cached lookups under a lookup cache key, with and without parameters. Args: key: The lookup cache key (see :meth:`_lookup_cache_key`). """ self._evict_cache(key) self._evict_cache((key, _LOOKUPS_CACHE_KEY)) def _evict_cache(self, key: str | tuple[Any, ...]) -> None: """Remove a dircache entry if it exists. ``DirCache.pop()`` reads and then deletes the entry, so it raises KeyError when another thread removes the same entry in between, such as the request threads of ``rm()`` invalidating a shared parent at once. A single ``del`` raises KeyError only when the entry is already gone, which this ignores. Args: key: The dircache key, a path, a ``(path, delimiter)`` listing key, or a ``(path, _LOOKUPS_CACHE_KEY)`` key. """ with contextlib.suppress(KeyError): del self.dircache[key] def _key_exists(self, path: S3Path, lookup_kwargs: Mapping[str, Any]) -> bool: """Check whether an object exists at the key of the path as written. Unlike :meth:`_object_exists`, a key that ends in a slash, such as a folder marker, is looked up as it is, for a write of that key. Args: path: The path of the object. lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``) to send with the requests. Returns: True if an object exists at the key. """ if not (path.key or "").endswith("/"): return self._object_exists(path, lookup_kwargs) try: self.core.head_object(path, **lookup_kwargs) except FileNotFoundError: return False return True def _object_exists(self, path: str | S3Path, lookup_kwargs: Mapping[str, Any]) -> bool: """Check whether an object exists at the path, for a write that must not replace one. A key that is only a prefix of other keys has no object, as when reading. Args: path: S3 path (s3://bucket/key). lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``) to send with the requests. Returns: True if an object exists at the path. """ try: info = self.info(path, **lookup_kwargs) except FileNotFoundError: return False return bool(info.get("type") != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY) @staticmethod def _get_lookup_kwargs(kwargs: Mapping[str, Any]) -> dict[str, Any]: """Select the request parameters that lookups send. Lookups (``info()`` and ``exists()``) send only the parameters on which their authorization depends, which also select their cached results. Other parameters that HeadObject accepts, such as ``IfMatch``, ``PartNumber`` or ``ResponseContentType``, are not sent, because the result would depend on them. Args: kwargs: The parameters to select from. Returns: The lookup parameters. """ return {k: v for k, v in kwargs.items() if k in _LOOKUP_REQUEST_PARAMETERS} def _get_cached_lookup( self, path: str | tuple[str, str, str], lookup_kwargs: Mapping[str, Any] ) -> S3Object | None: """Get the cached HeadObject or HeadBucket result of a lookup. A lookup without lookup parameters uses the entry under the path. A lookup with them uses only the result of a lookup with the same values, cached under ``(path, _LOOKUPS_CACHE_KEY)``, because the authorization of the request depends on them. Each of these results expires on its own after the ``listings_expiry_time`` of the dircache. Args: path: The lookup cache key of the object (see :meth:`_lookup_cache_key`), or the bucket name. lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``). Returns: The cached object, or None if there is none. """ if not lookup_kwargs: return cast("S3Object | None", self.dircache.get(path)) lookups = self.dircache.get((path, _LOOKUPS_CACHE_KEY)) cached = lookups.get(self._get_lookup_cache_id(lookup_kwargs)) if lookups else None if cached is None: return None cached_at, file = cached expiry_time = self.dircache.listings_expiry_time if expiry_time and time.time() - cached_at > expiry_time: # Caching the result of any parameters renews the dircache entry # of the path, so each result expires on its own. return None return cast(S3Object, file) def _cache_lookup( self, path: str | tuple[str, str, str], lookup_kwargs: Mapping[str, Any], file: S3Object ) -> None: """Cache the HeadObject or HeadBucket result of a lookup. Args: path: The lookup cache key of the object (see :meth:`_lookup_cache_key`), or the bucket name. lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``) that the request was made with. file: The object to cache. """ if not lookup_kwargs: self.dircache[path] = file return key = (path, _LOOKUPS_CACHE_KEY) lookups = self.dircache.get(key) if lookups is None: lookups = {} # Updated in place: a copy written back could replace a result that # another thread has cached since for other parameters. lookups[self._get_lookup_cache_id(lookup_kwargs)] = (time.time(), file) # Set again so that the expiry time of the entry is renewed. self.dircache[key] = lookups @staticmethod def _get_lookup_cache_id(lookup_kwargs: Mapping[str, Any]) -> tuple[tuple[str, Any], ...]: """Identify a set of lookup parameters in the cache. Args: lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``). Returns: The sorted parameters, with ``SSECustomerKey`` replaced by its SHA-256 digest so that the cache does not keep the key. """ return tuple( sorted( ( k, hashlib.sha256(v if isinstance(v, bytes) else str(v).encode()).hexdigest() if k == "SSECustomerKey" else v, ) for k, v in lookup_kwargs.items() ) ) def _ls_from_cache(self, path: S3Path) -> list[S3Object] | S3Object | None: """Check the dircache for a cached entry of the path. fsspec's implementation looks up listings under the path itself, but S3FileSystem caches a single S3Object under the lookup cache key of an object or a bucket (HeadObject/HeadBucket results, see :meth:`_lookup_cache_key`), the bucket listing under ``""``, and the other listings under ``(name, delimiter)`` (see ``_ls_prefix``). Args: path: The path of the bucket or the object. Returns: The cached entry of the path, the entries of a cached parent listing named as the path, or None if no cached entry describes the path. A listing of the path itself is not used, because it cannot tell whether an object of the same name exists. A path with a version uses only its own entry, because listings describe the current versions. Raises: FileNotFoundError: If a cached listing of the parent directory of a key path does not contain the path. """ cache = self.dircache.get(self._lookup_cache_key(path)) if cache is not None: return cast("list[S3Object] | S3Object", cache) if path.version_id: return None if path.key: parent_cache = self.dircache.get((self._listing_name(path), "/")) else: parent_cache = self.dircache.get("") if parent_cache is not None: files = [f for f in parent_cache if f["name"] == path.name] if files: return files if path.key: raise FileNotFoundError(str(path)) # The bucket listing holds only the buckets that the caller owns, # so a bucket missing from it is looked up with HeadBucket. return None def _open( self, path: str | S3Path, mode: str = "rb", block_size: int | None = None, cache_type: str | None = None, autocommit: bool = True, cache_options: dict[Any, Any] | None = None, **kwargs, ) -> S3File: if block_size is None: block_size = self.default_block_size if cache_type is None: cache_type = self.default_cache_type max_workers = kwargs.pop("max_workers", None) if max_workers is None: # None takes the default of the filesystem. max_workers = self.max_workers # The parameters of the call take precedence over those of the # filesystem; the caller's dictionary is not modified. s3_additional_kwargs = { **self.s3_additional_kwargs, **kwargs.pop("s3_additional_kwargs", {}), } return S3File( self, path, mode, max_workers=max_workers, executor=self._create_executor(max_workers=max_workers), block_size=block_size, cache_type=cache_type, autocommit=autocommit, cache_options=cache_options, s3_additional_kwargs=s3_additional_kwargs, **kwargs, ) def _call(self, method: str | Callable[..., Any], **kwargs) -> dict[str, Any]: """Send a request with the core (see :meth:`S3Core.call`). Args: method: The name of the client method, or the method itself. **kwargs: The request parameters. Returns: The response. """ return self._core.call(method, **kwargs)
# The parameters of the encryption of a written object, kept from an appended # object as a whole. _ENCRYPTION_PARAMS = frozenset( [ "ServerSideEncryption", "SSEKMSKeyId", "SSEKMSEncryptionContext", "BucketKeyEnabled", "SSECustomerAlgorithm", "SSECustomerKey", "SSECustomerKeyMD5", ] ) # The object lock of an appended object, which the rewritten object does not # keep. _OBJECT_LOCK_PARAMS = frozenset( ["ObjectLockMode", "ObjectLockRetainUntilDate", "ObjectLockLegalHoldStatus"] )
[docs] @dataclass class S3FileTarget: """The object that a file reads or writes, resolved when it is opened. In read mode, the object is looked up, and the reads are made conditional on its ETag (``IfMatch``). In append mode, an existing object is looked up; one smaller than ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` is also read, so that the file rewrites it from its buffer, and a larger one is copied by the file as the first parts of a multipart upload. In exclusive-create mode, the object must not exist, and the upload is committed with ``IfNoneMatch="*"``. Attributes: path: The path of the file. A version that is read is carried in the path as a ``?versionId=`` suffix, so that a reopened file reads the same version. bucket: The bucket of the object. key: The key of the object. version_id: The version that is read, if any. request_kwargs: The parameters of the S3 requests of the file, with the conditions above and, in append mode, the attributes of the existing object that the request parameters (those of the call and of the filesystem) do not give, which the rewritten object keeps: its system-defined and user-defined metadata, storage class, and encryption, unless they give any encryption parameter. Its object lock is not kept. details: The object that is read or appended, or an empty dictionary. size: The size of the object that is read, if known. appended: The existing object in append mode, if any. append_data: The data of an appended object that the file rewrites from its buffer, if any. """ path: str bucket: str key: str version_id: str | None request_kwargs: dict[str, Any] details: S3Object | dict[str, Any] = field(default_factory=dict) size: int | None = None appended: S3Object | None = None append_data: bytes | None = None
[docs] @classmethod def resolve( cls, fs: S3FileSystem, path: str | S3Path, mode: str, *, block_size: int, version_id: str | None = None, size: int | None = None, request_kwargs: Mapping[str, Any] | None = None, ) -> S3FileTarget: """Validate the arguments of a file and look up its object. Sends the lookup requests of the mode, which block. Args: fs: The filesystem that the file belongs to. path: S3 path (s3://bucket/key) of the file, or an ``S3Path``, whose bucket, key and version are used as they are, such as a key that a listing returns, and which is looked up by them. The path of the file is then the string of the ``S3Path``. mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. block_size: The block size of the file. version_id: The version ID to read. Must match the version ID in the path if both are given. A version cannot be given, in either form, for writing or appending. size: The size of the object to read, if known. request_kwargs: The parameters of the S3 requests of the file. A copy is kept. Returns: The resolved target. Raises: FileExistsError: If an object exists at the path in exclusive-create mode. FileNotFoundError: If no object exists at the path when reading, including when the path is a prefix. ValueError: If the path has no key, the version IDs do not match, a version is given for writing, or the block size is not between ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` and ``S3Core.MULTIPART_UPLOAD_MAX_PART_SIZE`` for writing. """ # A new dictionary, so that the caller's is not modified. kwargs: dict[str, Any] = dict(request_kwargs or {}) # The path as it is looked up: a path string as a string, and an # S3Path by its bucket, key and version. target = path s3_path = S3Path.of(path) path = str(path) path_version_id = s3_path.version_id if not s3_path.key: raise ValueError("The path does not contain a key.") if version_id and path_version_id: if version_id != path_version_id: raise ValueError( f"The version_id: {version_id} specified in the argument and " f"the version_id: {path_version_id} specified in the path do not match." ) elif path_version_id: version_id = path_version_id if version_id and "r" not in mode: raise ValueError("Cannot write to the file with the version specified.") if version_id and not path_version_id: # Carry the version in the path, as with the ?versionId= suffix, # so that a reopened (e.g., unpickled) file reads the same version. path = f"{path}?versionId={version_id}" target = s3_path.with_version_id(version_id) if isinstance(target, S3Path) else path if "r" not in mode and not ( fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE <= block_size <= fs.core.MULTIPART_UPLOAD_MAX_PART_SIZE ): # When writing, every full block is uploaded as a part of a # multipart upload. raise ValueError( "Block size for writing must be between " f"5 MiB ({fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE} bytes) and " f"5 GiB ({fs.core.MULTIPART_UPLOAD_MAX_PART_SIZE} bytes), inclusive: {block_size}." ) details: S3Object | dict[str, Any] = {} appended: S3Object | None = None append_data: bytes | None = None # The lookups need the parameters of the file on which their # authorization depends, such as the customer-provided key. lookup_kwargs = fs._get_lookup_kwargs(kwargs) if "r" in mode: # Looked up here; the base class initializer of a file would # otherwise take the size from the latest version of the object. info = fs.info(target, version_id=version_id, **lookup_kwargs) if info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # A prefix has no object to read. raise FileNotFoundError(path) if fs.version_aware and not version_id: # Pin the version observed at open time so that reads are # consistent even if the object is overwritten. info() heads # the object when the cached entry carries no version. version_id = info.get("version_id") if version_id: # Carried in the path as an explicit version is, so that # the methods of the file use it too. path = f"{path}?versionId={version_id}" if etag := info.get("etag"): # A caller's condition is sent as given, so that a read of # another object fails as S3 answers it (412). kwargs.setdefault("IfMatch", etag) details = info if size is None: size = info.get("size") elif "a" in mode: # The rewritten object keeps the metadata of the existing one, # which a cached listing entry lacks, so look up the object. with contextlib.suppress(FileNotFoundError): appended = fs.info(target, refresh=True, **lookup_kwargs) if ( appended is not None and appended.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY ): # A prefix has no object to append to, as when reading. appended = None if appended is not None: etag = appended.get("etag") if appended.get("size", 0) < fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE: # Too small to be a part of a multipart upload: rewritten # from the buffer. Only the lookup parameters, and the # condition on the object looked up, are sent, so that # the whole object is read, and an object replaced since # is not. condition = {"IfMatch": etag} if etag else {} append_data = fs.cat_file(target, **condition, **lookup_kwargs) # The attributes of the object looked up are kept where the # caller gives none: the encryption as a whole, so that its # fields are not combined with another encryption, and # without the object lock, which the caller sets. inherited = appended.to_api_repr() if _ENCRYPTION_PARAMS.intersection(kwargs): inherited = {k: v for k, v in inherited.items() if k not in _ENCRYPTION_PARAMS} for k, v in inherited.items(): if k not in _OBJECT_LOCK_PARAMS: kwargs.setdefault(k, v) # The part copies read the object looked up, with the # parameters that authorize the access to it under the names # of the copy source; a caller's values are kept. copy_source = fs.core.copy_from_source_params(lookup_kwargs) if etag: copy_source["CopySourceIfMatch"] = etag for k, v in copy_source.items(): kwargs.setdefault(k, v) details = appended elif "x" in mode: # Checked up front so that no data is uploaded for an existing # object, and on commit with IfNoneMatch for one created since. if fs._object_exists(target, lookup_kwargs): raise FileExistsError(path) kwargs.update({"IfNoneMatch": "*"}) return cls( path=path, bucket=s3_path.bucket, key=s3_path.key, version_id=version_id, request_kwargs=kwargs, details=details, size=size, appended=appended, append_data=append_data, )
[docs] class S3File(AbstractBufferedFile): """A buffered file object for reading and writing an S3 object. Instances are returned by ``S3FileSystem.open()``. """ fs: S3FileSystem buffer: BytesIO | None
[docs] def __init__( self, fs: S3FileSystem, path: str | S3Path, mode: str = "rb", version_id: str | None = None, max_workers: int | None = None, executor: S3Executor | None = None, block_size: int = S3FileSystem.DEFAULT_BLOCK_SIZE, cache_type: str = "bytes", autocommit: bool = True, cache_options: dict[Any, Any] | None = None, size: int | None = None, s3_additional_kwargs: dict[str, Any] | None = None, **kwargs, ) -> None: """Initialize the file for the path and mode. In read mode, the object is looked up with ``info()`` and the reads are made conditional on its ETag (``IfMatch``). In append mode, an existing object smaller than ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` is read into the write buffer; a larger one is copied with ``UploadPartCopy`` as the first parts of a multipart upload, whatever the block size. In exclusive-create mode, the object must not exist when the file is opened, and the upload is committed with ``IfNoneMatch="*"`` so that it does not replace an object created in the meantime. Args: fs: The filesystem that the file belongs to. path: S3 path (s3://bucket/key) of the file, or an ``S3Path`` (see :meth:`S3FileTarget.resolve`). mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. version_id: The version ID to read. Must match the version ID in the path if both are given. A version cannot be given, in either form, for writing or appending. max_workers: The number of parallel requests of the file: range reads, part copies, and part uploads. A write keeps at most ``max_workers`` parts, including the part copies of an append, in flight, waiting for one of them to finish before it submits another. If None, the ``max_workers`` of the filesystem. executor: The executor for parallel operations. If None, one on the request pool of the filesystem (``fs._create_executor()``). block_size: The block size for reads and writes. Must be between ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` and ``S3Core.MULTIPART_UPLOAD_MAX_PART_SIZE``, inclusive, unless reading. cache_type: The fsspec cache type for reads; see ``S3FileSystem.__init__()``. autocommit: Whether to commit the written data when the file is closed. If False, :meth:`commit` must be called. cache_options: Options for the fsspec cache. size: The size of the object, if known. Passed to ``fsspec.spec.AbstractBufferedFile``. s3_additional_kwargs: Additional parameters for the S3 requests of the file, such as ``ContentType`` or ``RequestPayer``. Each request receives those that its operation accepts. **kwargs: Additional parameters for the S3 requests of the file, which take precedence over ``s3_additional_kwargs``. Raises: FileExistsError: If an object exists at the path in exclusive-create mode. FileNotFoundError: If no object exists at the path when reading, including when the path is a prefix. ValueError: If the path has no key, the version IDs do not match, a version is given for writing, or the block size is not between ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` and ``S3Core.MULTIPART_UPLOAD_MAX_PART_SIZE`` for writing, or ``max_workers`` is not positive. """ if max_workers is None: max_workers = fs.max_workers if max_workers <= 0: raise ValueError("max_workers must be greater than 0") # The process that opened the file; a forked child rejects it. self._pid = os.getpid() self.max_workers = max_workers # The arguments are validated, and the objects looked up, before the # base class initializer: a file that fails here is never opened, so # its garbage collection does not close (flush and commit) it. target = S3FileTarget.resolve( fs, path, mode, block_size=block_size, version_id=version_id, size=size, request_kwargs={**(s3_additional_kwargs or {}), **kwargs}, ) self.bucket = target.bucket self.key = target.key self.version_id: str | None = target.version_id self.s3_additional_kwargs: dict[str, Any] = target.request_kwargs self._details: S3Object | dict[str, Any] = target.details self._executor: S3Executor = executor or fs._create_executor(max_workers=max_workers) # The requests and the state of the write; None when reading. self._session: _S3WriteSession | None = None if mode != "rb": self._session = _S3WriteSession( fs.core, S3Path(target.bucket, target.key), self._executor, block_size=block_size, max_workers=max_workers, # The mapping itself, which the requests read when they are # sent, as before; a mapping assigned later is not read. request_kwargs=self.s3_additional_kwargs, ) super().__init__( fs=fs, path=target.path, mode=mode, block_size=block_size, autocommit=autocommit, cache_type=cache_type, cache_options=cache_options, size=target.size, # Kept as ``self.kwargs``, which fsspec passes to open() when it # reopens the file (e.g., when it is unpickled): the parameters # of the requests, including the condition on the object looked # up, so that the reopened file reads the same object or fails. **(target.request_kwargs if mode == "rb" else {}), ) self.append_block = False # The size of the appended object, as looked up when opening. self._append_size = 0 if target.appended is not None: if target.append_data is not None: self.write(target.append_data) else: # Copied with UploadPartCopy as the leading part(s). self.append_block = True self._append_size = self.loc = target.appended.get("size", 0)
def _get_request_kwargs(self, method: str) -> dict[str, Any]: """Select the parameters of the file that an S3 operation accepts. Args: method: The name of the client method, such as ``get_object``. Returns: The parameters in ``s3_additional_kwargs`` that the operation accepts. """ return self.fs.core.operation_params(method, self.s3_additional_kwargs)
[docs] def close(self) -> None: """Close the file, flushing any written data, and shut down its executor. In a forked child process, a file that the parent opened is only marked closed: its data is neither flushed nor committed, and its upload is not aborted, since they belong to the parent. Called by another finalizer while the cyclic garbage collector runs in this thread, such as that of a ``TextIOWrapper``, the file is closed in a new thread once the collection has ended (see :meth:`__del__`), and this returns before that. A close handed over to another thread is waited for, except during a collection or in a thread of an :class:`S3ThreadPool`, where this returns without waiting. """ if self._inherited(): self.closed = True return if not self.closed and S3File._in_collection(): self._close_later() return if self._wait_for_close_later(): self._close()
def _close(self) -> None: """Close the file in this thread, flushing any written data, and shut down its executor.""" try: super().close() finally: # The executor is shut down even if the final flush fails. self._executor.shutdown()
[docs] def __del__(self) -> None: """Close the file when it is finalized without having been closed. A file whose last reference is dropped in a user thread is closed here. A file that the cyclic garbage collector finalizes, or that is finalized in a thread of an :class:`S3ThreadPool`, is closed in a new thread instead, since a wait here could last forever (T1): the requests of the file may be queued behind this thread or include the one that runs the collection, and this thread may hold a lock that the close takes. During a collection, the thread starts once the collection has ended, after the other finalizers of the collection, such as that of a ``TextIOWrapper``, have written to the file. """ if getattr(self, "closed", True) or self._inherited(): # The upload of a file inherited across a fork is the parent's. return if S3ThreadPool.in_pool_thread(): self._close_later() else: # Hands the close over itself during a collection. self.close()
@staticmethod def _track_collection(phase: str, info: dict[str, int]) -> None: """Record whether the cyclic garbage collector runs in this thread. Registered in ``gc.callbacks``, which the collector calls in the thread that runs it. When the collection stops, the closes that it handed over start. Args: phase: ``"start"`` or ``"stop"``. info: The details of the collection, which are not used. """ _collecting.active = phase == "start" pending = getattr(_collecting, "pending", None) if not _collecting.active and pending: _collecting.pending = [] for file, complete in pending: file._start_close(complete) @staticmethod def _in_collection() -> bool: """Whether the cyclic garbage collector runs in this thread. Returns: True while a collection runs in the calling thread. """ return getattr(_collecting, "active", False) def _close_later(self) -> None: """Close the file in a new thread, once. The written data is copied first: the collector finalizes every object that it found unreachable before it checks whether a finalizer has made it reachable again, and finalizing the buffer closes it. If the buffer is already closed, the data is incomplete, so the write is discarded instead of committed. During a collection, the thread starts once the collection has ended, so that the other finalizers of the collection can still write to the file. """ if getattr(self, "_closing_later", False): return self._closing_later = True self._closed_later = threading.Event() complete = True if self.mode != "rb" and self.buffer is not None: if self.buffer.closed: complete = False else: buffer = BytesIO(self.buffer.getvalue()) buffer.seek(self.buffer.tell()) self.buffer = buffer _handed_over_closes.add(self._closed_later) if S3File._in_collection(): if not hasattr(_collecting, "pending"): _collecting.pending = [] _collecting.pending.append((self, complete)) else: self._start_close(complete) def _start_close(self, complete: bool) -> None: """Start the close that :meth:`_close_later` handed over. The file is closed in a thread of the ``_thread`` module, since ``Thread.start()`` takes locks of the ``threading`` module, which this thread, in which a finalizer or the collector runs, may hold, and waits for the new thread, which may take another. The interpreter does not join such a thread, so an exit waits for the close instead (see :meth:`_wait_for_handed_over_closes`). The thread runs with an empty context, so that the stop scopes of the finalizing thread do not stop the requests of the file. During interpreter finalization, and once the exit has waited for the handed-over closes, the file is closed in this thread without starting one, since no thread started during finalization runs, and none started afterwards is waited for. No other thread runs requests then, and the pools take no more calls, so the close does not wait for a part queued behind it. If no thread can be started otherwise, the file is left unclosed rather than closed in this thread, where it could wait forever, until a later :meth:`close`, :meth:`commit`, or :meth:`discard` closes it (see :meth:`_wait_for_close_later`). Args: complete: Whether the written data is complete. """ if self._inherited(): # Handed over in the parent, which forked before the collection # ended. Its event, whose lock the fork may have copied held, is # left alone. self.closed = True return if sys.is_finalizing() or _handed_over_closes_waited: self._close_collected(complete) return try: _thread.start_new_thread(contextvars.Context().run, (self._close_collected, complete)) except Exception: # Such as RuntimeError, when no more threads can be started. _logger.warning( f"s3://{self.bucket}/{self.key} was not closed, and no thread could be " "started to close it when it was finalized; its data was not committed." ) self._unstarted_close: bool | None = complete _handed_over_closes.discard(self._closed_later) # Wakes the callers that wait for it, which then close the file. self._closed_later.set() def _close_collected(self, complete: bool) -> None: """Close a finalized file, logging a failure instead of raising it. Args: complete: Whether the written data is complete. If not, the write is discarded instead of committed. """ self._closer_ident = threading.get_ident() try: if complete: self._close() else: _logger.warning( f"s3://{self.bucket}/{self.key} was not closed, and its buffer was " "finalized before it; its data was not committed." ) self._close_without_commit() if getattr(self, "_discard_later", False): self.discard() except Exception: _logger.exception(f"Failed to close s3://{self.bucket}/{self.key}.") finally: self._end_close_later() def _end_close_later(self) -> None: """Mark the close that :meth:`_close_later` handed over as ended.""" _handed_over_closes.discard(self._closed_later) self._closed_later.set() @staticmethod def _wait_for_handed_over_closes() -> None: """Wait for the closes that run in threads of the ``_thread`` module. Registered with :mod:`atexit`, since the interpreter does not join these threads when it exits. Also waits for the closes that are handed over while it waits; later ones, such as in an exit handler that runs after it, are closed in the finalizing thread. """ global _handed_over_closes_waited while _handed_over_closes: for closed in list(_handed_over_closes): closed.wait() _handed_over_closes_waited = True def _wait_for_close_later(self) -> bool: """Wait for a close that :meth:`_close_later` handed over to end. It is not waited for in the thread that runs it, during a collection, whose end starts it, or in a thread of an :class:`S3ThreadPool`, since it may wait for a request queued behind that thread. A close for which no thread could be started is run here instead, where it can be waited for. Returns: False if a handed-over close has not ended and cannot be waited for in this thread; otherwise True. """ closed_later = getattr(self, "_closed_later", None) if closed_later is None or getattr(self, "_closer_ident", None) == threading.get_ident(): return True unsafe = S3File._in_collection() or S3ThreadPool.in_pool_thread() if not closed_later.is_set(): if unsafe: return False closed_later.wait() if getattr(self, "_unstarted_close", None) is not None: if unsafe: return False unstarted, self._unstarted_close = self._unstarted_close, None self._close_collected(cast(bool, unstarted)) return True def _raise_unless_closed_later(self) -> None: """Wait for a handed-over close to end, or raise if it cannot be waited for here. Raises: RuntimeError: If a close of the file was handed over to another thread and runs, and this is called during a collection or in a thread of an :class:`S3ThreadPool`. """ if not self._wait_for_close_later(): raise RuntimeError( f"s3://{self.bucket}/{self.key} is being closed in another thread, which " "cannot be waited for during a garbage collection or in a thread of an " "S3ThreadPool." )
[docs] def read(self, length: int = -1) -> bytes: """Read bytes from the file. Args: length: The number of bytes to read; -1 reads to the end. Returns: The bytes read. Raises: RuntimeError: In a forked child process of the process that opened the file. ValueError: If the file is closed or not open for reading. """ self._check_process() data: bytes = super().read(length) return data
[docs] def flush(self, force: bool = False) -> None: """Upload the buffered data as a part once a block is full, or all of it with ``force``. Args: force: Whether to upload the remaining data and commit the file, after which it cannot be written. Raises: RuntimeError: In a forked child process of the process that opened the file. ValueError: If ``force`` is given again after a forced flush. """ self._check_process() super().flush(force=force)
def _inherited(self) -> bool: """Whether this process is a forked child of the process that opened the file. Returns: True in a forked child process of the process that opened the file. """ # Compared with the process ID, not a count that an at-fork hook # updates, since hooks registered before it may close the file. return getattr(self, "_pid", os.getpid()) != os.getpid() def _check_process(self) -> None: """Reject the use of a file that the parent of a forked process opened. Raises: RuntimeError: In a forked child process of the process that opened the file. """ if self._inherited(): raise RuntimeError( f"s3://{self.bucket}/{self.key} was opened by the parent of this forked " "process and cannot be used here; open it again in this process." )
[docs] def write(self, data: bytes | bytearray | memoryview) -> int: """Write data to the buffer, uploading each full block as a part. Args: data: The bytes to write. Returns: The number of bytes written. Raises: BaseException: The error of a request of the write that failed since the last write, after which the file is closed without committing it. RuntimeError: In a forked child process of the process that opened the file. S3MultipartStateError: If the write was discarded. ValueError: If the file is not open for writing. """ self._check_process() if self._session is not None and not self.closed: try: self._session.check() except BaseException: self._close_without_commit() raise written: int = super().write(data) return written
def _close_without_commit(self) -> None: """Close the file without uploading the written data. Drops the buffered data, and discards the write (see :meth:`discard`), so that neither close() nor a deferred commit() uploads the data. An abort failure is logged instead of raised, so it does not mask the error that the caller is handling; the upload is then kept, so that :meth:`discard` can abort it. The executor is shut down here, as fsspec does not close a closed file again when it is garbage collected. """ if self.buffer is None and getattr(self, "closed", False): return self.buffer = None self.closed = True try: if self._wait_for_close_later(): self.discard() else: # The handed-over close, which cannot be waited for here, # discards the write once it ends. self._discard_later = True except Exception: upload_id = cast(S3MultipartUpload, self.multipart_upload).upload_id _logger.exception( f"Failed to abort multipart upload {upload_id} to s3://{self.bucket}/{self.key}." ) finally: self._executor.shutdown() def _write_and_close(self, value: bytes | bytearray | memoryview) -> None: """Write the whole value and close the file. Unlike a ``with`` block, a failed write closes the file without committing it, so the existing object is left unchanged. Args: value: The bytes to write. """ try: if isinstance(value, memoryview) and not value.c_contiguous: # The buffer of the file cannot write a non-contiguous memoryview. value = value.tobytes() self.write(value) except BaseException: self._close_without_commit() raise self.close() def _write_file_and_close(self, local: BinaryIO, callback: Callback) -> None: """Write the rest of a local file and close the file. Unlike a ``with`` block, a failed read, write, or progress update closes the file without committing it, so the existing object is left unchanged. Args: local: The local file to read from. callback: Progress callback, updated with the size of each block. """ try: while data := local.read(self.blocksize): self.write(data) callback.relative_update(len(data)) except BaseException: self._close_without_commit() raise self.close() @property def multipart_upload(self) -> S3MultipartUpload | None: """The multipart upload of the write, from its creation until it is aborted.""" return self._session.upload if self._session is not None else None @property def completed_parts(self) -> tuple[S3MultipartUploadPart, ...]: """The uploaded or copied parts in part-number order; empty after an abort.""" return self._session.completed_parts if self._session is not None else () def _initiate_upload(self) -> None: if not self.append_block and self.tell() < self.blocksize: return session = cast(_S3WriteSession, self._session) try: session.start() session.wait_ready() # Raises the failure of the creation. session.check() if self.append_block: # The size of the object looked up when opening, which the # part copies are conditional on. size = self._append_size whole_source = size <= self.fs.core.MULTIPART_UPLOAD_MAX_PART_SIZE source = S3Path(self.bucket, self.key) for part_number, range_ in session.iter_copy_parts(size): # The part copies count toward max_workers, as the parts do. session.wait_ready() session.submit_copy(part_number, source, None if whole_source else range_) except BaseException: # A creation that has started may still create an upload, which # the abort waits for. self._close_without_commit() raise def _upload_chunk(self, final: bool = False) -> bool: """Upload the buffered data as parts of the multipart upload. A file smaller than the block size, unless an append copied the existing object as parts, is left in the buffer for :meth:`commit`, which uploads it with PutObject. Otherwise, the buffered data is uploaded as parts, and a part is submitted only while fewer than ``max_workers`` parts, including the part copies of an append, are in flight; the write waits for one of them to finish first. A request found failed before a part is submitted stops the write with its error; one that fails afterwards is reported by the next write or by :meth:`commit`. If the upload of the chunk fails or is interrupted, the file is closed without committing it. Args: final: Whether this is the last flush of the file, after which the upload is committed if ``autocommit`` is set. Returns: Whether fsspec's ``flush()`` resets the buffer afterwards, which it does for any value other than False. Raises: BaseException: The error of a request of the write that failed before a part is submitted. ValueError: If the data needs more parts than a multipart upload allows. """ # Returning ``not final`` keeps the buffer intact on the final flush so a # deferred commit() (autocommit=False, i.e. inside an fsspec transaction) # can still read the bytes; resetting it there would upload an empty # object for small files. Mid-stream chunks (final=False) return True so # fsspec clears the already-uploaded buffer between parts. if not self.append_block and self.tell() < self.blocksize: # Files smaller than block size in size cannot be multipart uploaded. if self.autocommit and final: self.commit() return not final session = cast(_S3WriteSession, self._session) # fsspec's flush() never calls this on a closed file, whose buffer # may have been dropped. buffer = cast(BytesIO, self.buffer) buffer.seek(0) try: for part_number, data in session.iter_parts(buffer): # Otherwise the parts waiting for a worker pile up in memory. session.wait_ready() session.submit_part(part_number, data) except BaseException: self._close_without_commit() raise if self.autocommit and final: self.commit() return not final
[docs] def commit(self) -> None: """Commit the written data. Creates an empty object if nothing was written, uploads the buffered data with PutObject if no multipart upload was started, and otherwise completes the multipart upload once its parts have been uploaded. A failed part is raised as soon as it fails, without waiting for the parts before it. If a part or the completion fails or is interrupted, the file is closed without committing it, and its upload is aborted; if the abort also fails, the upload is kept so that :meth:`discard` can abort it. Committing again after a successful commit sends no request. Invalidates the cache of the path afterwards. Does nothing for a file opened for reading. Waits first for a close of the file that was handed over to another thread (see :meth:`close`). Raises: FileExistsError: If an object was created at the path after the file was opened in exclusive-create mode. RuntimeError: In a forked child process of the process that opened the file, or if a close of the file that was handed over to another thread runs and this is called during a garbage collection or in a thread of an :class:`S3ThreadPool`. S3MultipartStateError: If the write was discarded, or the write or an earlier commit failed. """ self._check_process() self._raise_unless_closed_later() if self._session is None: # A file opened for reading has nothing to commit. return session = self._session if not session.multipart: body = None if self.buffer is not None and self.tell(): self.buffer.seek(0) body = self.buffer.read() # None writes an empty object. session.put(body) else: try: session.complete() except BaseException: # Closed without the buffer, which holds only the data of the # last part. self._close_without_commit() raise self.fs._invalidate(S3Path(self.bucket, self.key))
[docs] def discard(self) -> None: """Discard the written data, and abort the multipart upload, if any. The part requests that have not started are cancelled, and the running ones make no further attempt and are waited for before the abort, whose own retries are not stopped. If the abort fails or is interrupted, the upload is kept, and calling this again retries the abort. Afterwards, :meth:`commit` raises :class:`~pyathena.filesystem.s3_writer.S3MultipartStateError`. Does nothing for a committed file or a file opened for reading. Waits first for a close of the file that was handed over to another thread (see :meth:`close`). Raises: RuntimeError: In a forked child process of the process that opened the file, or if a close of the file that was handed over to another thread runs and this is called during a garbage collection or in a thread of an :class:`S3ThreadPool`. """ self._check_process() self._raise_unless_closed_later() if self._session is not None: self._session.abort()
def _lookup_kwargs(self) -> dict[str, Any]: """Select the parameters of the file on which the access to its object depends. Returns: The lookup parameters of the file (see ``S3FileSystem._get_lookup_kwargs``), such as ``RequestPayer`` or the ``SSECustomer*`` parameters of an object encrypted with a customer-provided key. """ return self.fs._get_lookup_kwargs(self.s3_additional_kwargs)
[docs] def url(self, expiration: int = 3600, **kwargs) -> str: """Generate a presigned HTTP URL to read this file (if it already exists). The URL is signed with the lookup parameters of the file that the signed operation accepts, such as ``RequestPayer`` or the ``SSECustomer*`` parameters, whose headers the request with the URL must then send. Args: expiration: URL expiration time in seconds. Defaults to 3600 (1 hour). **kwargs: Additional parameters passed to :meth:`S3FileSystem.sign`, which take precedence over those of the file. Returns: Presigned URL string that provides temporary access to the S3 object. """ # Only the parameters of the file that the signed operation accepts. params = self.fs.core.operation_params( kwargs.get("client_method", "get_object"), self._lookup_kwargs() ) return cast(str, self.fs.sign(self.path, expiration=expiration, **{**params, **kwargs}))
[docs] def metadata(self, **kwargs) -> S3Metadata: """Return the metadata of the file. See :meth:`S3FileSystem.metadata`. The HeadObject request receives the lookup parameters of the file, such as ``RequestPayer`` or the ``SSECustomer*`` parameters. Args: **kwargs: Additional parameters passed to the HeadObject API, which take precedence over those of the file. Returns: S3Metadata, which behaves as a read-only mapping of the user-defined metadata and exposes the system-defined metadata as typed properties. """ return self.fs.metadata(self.path, **{**self._lookup_kwargs(), **kwargs})
[docs] def getxattr(self, xattr_name: str, **kwargs) -> str | None: """Get an attribute from the user-defined metadata of the file. See :meth:`S3FileSystem.getxattr`, and :meth:`metadata` for the parameters of the request. Args: xattr_name: The name of the attribute. **kwargs: Additional parameters passed to the HeadObject API, which take precedence over those of the file. Returns: The value of the attribute, or None if the attribute is not set. """ return self.fs.getxattr(self.path, xattr_name, **{**self._lookup_kwargs(), **kwargs})
[docs] def setxattr(self, copy_kwargs: dict[str, Any] | None = None, **kwargs) -> None: """Set the user-defined metadata of the file. See :meth:`S3FileSystem.setxattr`. The copy of the object onto itself receives the lookup parameters of the file both for its source and for its destination: ``RequestPayer``, the expected bucket owner as ``ExpectedSourceBucketOwner`` and ``ExpectedBucketOwner``, and the ``SSECustomer*`` parameters also as ``CopySourceSSECustomer*``, so that an object encrypted with a customer-provided key keeps it. A ``ServerSideEncryption`` in ``copy_kwargs`` replaces the ``SSECustomer*`` parameters of the file for the destination; those in the ``s3_additional_kwargs`` of the filesystem are still sent. Args: copy_kwargs: Additional parameters to use for the underlying CopyObject API call, which take precedence over those of the file. **kwargs: Key-value pairs of metadata to set. """ if self.writable(): raise NotImplementedError("Cannot update metadata while the file is open for writing.") copy_kwargs = copy_kwargs or {} lookup_kwargs = self._lookup_kwargs() destination = lookup_kwargs if "ServerSideEncryption" in copy_kwargs: # Another encryption of the copy replaces the customer-provided key. destination = { k: v for k, v in lookup_kwargs.items() if not k.startswith("SSECustomer") } self.fs.setxattr( self.path, copy_kwargs={ **destination, **self.fs.core.copy_from_source_params(lookup_kwargs), **copy_kwargs, }, **kwargs, )
def _fetch_range(self, start: int, end: int) -> bytes: """Read a byte range of the object for the fsspec cache. The range is clamped to the size of the object, since fsspec caches may request a range that is empty or reaches past the end of the object. S3 would answer the former with the whole object and a range starting past the end with an ``InvalidRange`` error. A range larger than the block size is read with parallel requests (see :meth:`S3Executor.map() <pyathena.filesystem.s3_executor.S3Executor.map>`); when one fails, those that have not started are cancelled, the running ones make no further attempt, and its error is raised once they have ended. Once an enclosing stop scope is stopped, such as that of a batch operation after another object failed, no further request is sent (see :func:`~pyathena.util.raise_if_stopped`). Args: start: The offset of the first byte to read. end: The offset to stop reading at (exclusive). Returns: The bytes read, empty if the clamped range is empty. Raises: concurrent.futures.CancelledError: If an enclosing stop scope has been stopped. """ end = min(end, self.size) if start >= end: return b"" ranges = self._get_ranges( start, end, max_workers=self.max_workers, worker_block_size=self.blocksize ) path = S3Path(self.bucket, self.key, self.version_id) request_kwargs = self._get_request_kwargs("get_object") if len(ranges) > 1: # Not kept by a variable of this frame, which the traceback of an # interrupt keeps (F5). return b"".join( cast( list[bytes], self._executor.map( functools.partial(self._get_range, path, **request_kwargs), ranges, ).outcomes, ) ) return self._get_range(path, ranges[0], **request_kwargs) def _get_range(self, path: S3Path, range_: tuple[int, int], **kwargs: Any) -> bytes: """Read a byte range with GetObject unless the read has been stopped. A range that waited behind other requests checks the stop scopes of its context when it starts, since the retry stop does not cover the first attempt. Args: path: The object, with its version ID, if any. range_: The range, with an exclusive end. **kwargs: The GetObject parameters. Returns: The bytes of the range. Raises: concurrent.futures.CancelledError: If an enclosing stop scope has been stopped. """ raise_if_stopped(f"read {path.uri}") return self.fs.core.get_object(path, range_, **kwargs) def _download(self, outfile: Any, callback: Callback) -> None: """Write the whole object into a writable file with parallel range reads. The object is read in waves of ranges of the block size, or in one range if the block size is not positive, one wave of at most ``max_workers`` ranges at a time, but no more than the request pool of the filesystem runs at once, since the others would only wait with their bodies held. Each wave is read with :meth:`S3Executor.map() <pyathena.filesystem.s3_executor.S3Executor.map>`, and its bodies are written in order, each dropped once written, before the next wave is read, so that the download holds at most one wave of bodies; a thread of the pool that has just returned a body drops it when its call returns. The reads carry the version and the conditions of the file, and each checks the stop scopes when it starts; a body read when a stop came is not written. A failure or an interrupt while a wave is read is raised as :meth:`S3Executor.map()` raises it. One while its bodies are written, including a failure of the write or of the callback, drops the bodies first, also from the completed frames of its traceback (see :meth:`S3MapResult.release() <pyathena.filesystem.s3_executor.S3MapResult.release>`). Args: outfile: The writable file-like object. callback: Progress callback, updated with the size of each body once it is written. Raises: concurrent.futures.CancelledError: If an enclosing stop scope has been stopped. """ size = cast(int, self.size) # A block size that is not positive reads the rest of the object, # as read() does. block = self.blocksize if self.blocksize > 0 else max(size, 1) window = min(self.max_workers, self.fs._request_pool.max_workers) path = S3Path(self.bucket, self.key, self.version_id) read = functools.partial(self._get_range, path, **self._get_request_kwargs("get_object")) wave = window * block result: S3MapResult[bytes] | None = None try: for wave_start in range(0, size, wave): wave_end = min(size, wave_start + wave) # Assigned within the try block, so that an interrupt right # after the wave was read still drops its bodies. result = self._executor.map( read, [ (start, min(start + block, wave_end)) for start in range(wave_start, wave_end, block) ], ) bodies = result.outcomes for i in range(len(bodies)): # A body read when the stop came is not written. raise_if_stopped(f"write the contents of {self.path}") length = len(cast(bytes, bodies[i])) outfile.write(bodies[i]) # Not kept while the next one is written or read. bodies[i] = b"" callback.relative_update(length) except BaseException as e: if result is not None: result.release(e) raise @staticmethod def _get_ranges( start: int, end: int, max_workers: int, worker_block_size: int ) -> list[tuple[int, int]]: ranges = [] range_size = end - start if max_workers > 1 and range_size > worker_block_size: range_start = start while True: range_end = range_start + worker_block_size if range_end >= end: # Also when the size is an exact multiple of the block # size, so that no empty trailing range is generated. ranges.append((range_start, end)) break ranges.append((range_start, range_end)) range_start += worker_block_size else: ranges.append((start, end)) return ranges
gc.callbacks.append(S3File._track_collection) atexit.register(S3File._wait_for_handed_over_closes) if hasattr(os, "register_at_fork"): # The closes of the parent do not run in a child. os.register_at_fork(after_in_child=_handed_over_closes.clear)