"""fsspec filesystem and file implementations for Amazon S3."""
from __future__ import annotations
import _thread
import atexit
import contextlib
import contextvars
import functools
import gc
import hashlib
import logging
import mimetypes
import os.path
import sys
import threading
import time
import warnings
from collections.abc import Callable, Generator, Iterable, Mapping
from copy import deepcopy
from dataclasses import dataclass, field
from datetime import datetime
from glob import has_magic
from io import BytesIO
from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, cast
from urllib.parse import unquote_plus
import fsspec
from boto3 import Session
from botocore import UNSIGNED
from botocore.client import BaseClient, Config
from fsspec import AbstractFileSystem
from fsspec.callbacks import _DEFAULT_CALLBACK, Callback
from fsspec.compression import compr
from fsspec.config import apply_config
from fsspec.core import get_compression
from fsspec.implementations.local import LocalFileSystem, make_path_posix, trailing_sep
from fsspec.spec import AbstractBufferedFile
from fsspec.utils import check_contained, isfilelike, other_paths, stringify_path, tokenize
import pyathena
from pyathena.filesystem.s3_core import S3Core, S3DeleteBatch, S3DeleteResult
from pyathena.filesystem.s3_errors import S3ClientError
from pyathena.filesystem.s3_executor import (
ItemT,
S3Executor,
S3MapResult,
S3ThreadPool,
T,
_S3PoolScope,
)
from pyathena.filesystem.s3_object import (
S3Metadata,
S3MultipartUpload,
S3MultipartUploadPart,
S3Object,
S3ObjectType,
S3ObjectVersion,
S3StorageClass,
)
from pyathena.filesystem.s3_path import S3Path
from pyathena.filesystem.s3_path_pairing import S3PathPairing
from pyathena.filesystem.s3_writer import _S3WriteSession
from pyathena.util import RetryConfig, override, raise_if_stopped, unstoppable_retries
if TYPE_CHECKING:
from pyathena.connection import Connection
_logger = logging.getLogger(__name__)
# The message of the TypeError that a filesystem with an injected client, and
# the files that it opens for reading, raise when they are pickled.
_UNPICKLABLE_CLIENT_MESSAGE = (
"{cls} built with client= or connection= cannot be pickled, as its boto3 client "
"cannot be. To pickle it, for example to send it to Dask, Ray, or multiprocessing "
"workers, build it from s3fs-style arguments such as profile=, region_name=, or "
"key=, secret=, and token=."
)
# The DeprecationWarning of the connection argument of the filesystems.
_CONNECTION_DEPRECATION_MESSAGE = (
"The connection argument of {cls} is deprecated and will be removed in the next "
"major version. Use pyathena.filesystem.s3_filesystem(connection), or pass "
"client=connection.s3_client and retry_config=connection.retry_config."
)
# Whether the cyclic garbage collector runs in a thread, and the closes of
# files that it handed over (S3File._track_collection).
_collecting = threading.local()
# The ends of the closes that run in threads of the _thread module, which an
# exit waits for (S3File._wait_for_handed_over_closes), and whether it has.
_handed_over_closes: set[threading.Event] = set()
_handed_over_closes_waited = False
# The request parameters on which the authorization of a lookup (HeadObject,
# HeadBucket, or ListObjectsV2) depends.
_LOOKUP_REQUEST_PARAMETERS = frozenset(
{
"ExpectedBucketOwner",
"RequestPayer",
"SSECustomerAlgorithm",
"SSECustomerKey",
"SSECustomerKeyMD5",
}
)
# The second element of the dircache key, ``(path, _LOOKUPS_CACHE_KEY)``, of
# the lookup results of a path made with lookup request parameters.
_LOOKUPS_CACHE_KEY = "lookups"
# The second element of the dircache key,
# ``(name, _VERSION_CACHE_KEY, version_id)``, of the lookup of an explicit
# version, which no key of an unversioned path, a name string, can equal.
_VERSION_CACHE_KEY = "versionId"
class CompressedBuffer(BytesIO):
"""An in-memory buffer of data compressed with a codec of fsspec.
The buffer keeps its data when it is closed, as some codecs, such as
the ``zstandard`` stream writer, close the file that they write to.
"""
@override
def close(self) -> None:
"""Do nothing, so that the data can still be read after a codec closes the buffer."""
@classmethod
def compress(cls, value: bytes | bytearray | memoryview, compression: str) -> bytes:
"""Compress a value with a codec of fsspec.
Args:
value: The bytes to compress.
compression: Name of a codec in ``fsspec.compression.compr``.
Returns:
The compressed bytes.
Raises:
ValueError: If the codec is not supported.
"""
if compression not in compr:
raise ValueError(f"Compression type {compression} not supported")
if isinstance(value, memoryview) and not value.c_contiguous:
# Codecs cannot compress a non-contiguous memoryview.
value = value.tobytes()
buffer = cls()
with compr[compression](buffer, mode="w") as f:
f.write(value)
return buffer.getvalue()
class _S3LockedCallback(Callback):
"""A progress callback whose uses, and those of its branches, are serialized.
The per-object calls of a batch operation run in parallel threads and
update the callback of the operation and the branches that it creates.
fsspec does not require a callback to be thread-safe, nor its branches to
be independent of it, so every use goes through one lock.
Args:
callback: The callback of the operation, or a branch of it.
lock: The lock of the operation.
"""
def __init__(self, callback: Callback, lock: threading.Lock) -> None:
super().__init__()
self._callback = callback
self._lock = lock
def __enter__(self) -> _S3LockedCallback:
"""Enter the callback.
Returns:
This callback, or a proxy of the callback that the wrapped one
returned from ``__enter__()``, if that is another one.
"""
with self._lock:
entered = self._callback.__enter__()
if entered is self._callback:
return self
return _S3LockedCallback(entered, self._lock)
def __exit__(self, *exc_args: Any) -> Any:
"""Exit the callback.
Args:
*exc_args: The exception of the block, if any.
Returns:
What the callback's ``__exit__()`` returns.
"""
with self._lock:
return self._callback.__exit__(*exc_args)
def close(self) -> None:
"""Close the callback."""
with self._lock:
self._callback.close()
def branched(self, path_1, path_2, **kwargs) -> Callback:
"""Return a branch of the callback for the transfer of one file.
Args:
path_1: The source of the transfer.
path_2: The destination of the transfer.
**kwargs: Passed to the callback's ``branched()``.
Returns:
The branch, whose uses take the lock of this callback.
"""
with self._lock:
return _S3LockedCallback(self._callback.branched(path_1, path_2, **kwargs), self._lock)
def set_size(self, size) -> None:
"""Set the size of the callback.
Args:
size: The total.
"""
with self._lock:
self._callback.set_size(size)
def absolute_update(self, value) -> None:
"""Set the progress of the callback.
Args:
value: The progress.
"""
with self._lock:
self._callback.absolute_update(value)
def relative_update(self, inc=1) -> None:
"""Advance the progress of the callback.
Args:
inc: The increment.
"""
with self._lock:
self._callback.relative_update(inc)
def call(self, hook_name=None, **kwargs) -> Any:
"""Run the hooks of the callback.
Args:
hook_name: The hook to run, or None for all of them.
**kwargs: Passed to the hooks.
Returns:
What the callback's ``call()`` returns.
"""
with self._lock:
return self._callback.call(hook_name, **kwargs)
class _S3TreeListing:
"""The listings of one ``tree()`` of an ``S3FileSystem``.
fsspec's ``tree()`` lists each directory with ``ls()`` of its name, which
a version ID query or an empty segment at the end of the name changes.
This object runs fsspec's ``tree()`` in place of the filesystem: a
directory that a listing of the same ``tree()`` returned is listed by
its key prefix (see :meth:`S3FileSystem._directory_prefix`), and any
other path with ``ls()``.
"""
def __init__(self, fs: S3FileSystem) -> None:
"""Initialize the listings with no listed directory.
Args:
fs: The filesystem to list.
"""
self._fs = fs
self._directories: dict[str, S3Object] = {}
def ls(self, path: str, detail: bool = True) -> list[S3Object]:
"""List a path as ``S3FileSystem.ls(detail=True)`` does.
Args:
path: The path, or the name of a directory listed before.
detail: Unused; the entries are always returned, as fsspec's
``tree()`` asks for them.
Returns:
A new list of the entries, which fsspec's ``tree()`` sorts in place.
"""
directory = self._directories.get(path)
if directory is None:
files = cast(list[S3Object], self._fs.ls(path, detail=True))
else:
files = self._fs._ls_directory(directory)
for f in files:
if f.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY:
self._directories[f.name] = f
return files
def tree(self, **kwargs: Any) -> str:
"""Run fsspec's ``tree()`` with these listings.
Args:
**kwargs: The arguments of ``tree()``.
Returns:
The tree as a string.
"""
return cast(str, AbstractFileSystem.tree(self, **kwargs))
[docs]
class S3FileSystem(AbstractFileSystem):
"""A filesystem interface for Amazon S3 that implements the fsspec protocol.
This class provides a file-system like interface to Amazon S3, allowing you to
use familiar file operations (ls, open, cp, rm, etc.) with S3 objects. It's
designed to be compatible with s3fs while offering PyAthena-specific optimizations.
The filesystem supports standard S3 operations including:
- Listing objects and directories
- Reading and writing files
- Copying and moving objects
- Reading and writing object metadata, tags, and canned ACLs
- Multipart uploads for large files, including management of
incomplete uploads
- Version-aware reads and object version listing (see ``version_aware``)
- Creating and removing buckets (disabled by default; see
``allow_bucket_creation`` / ``allow_bucket_deletion``)
- Various S3 storage classes and encryption options
- Translating S3 error responses into standard Python exceptions
(e.g., ``404`` -> ``FileNotFoundError``, ``403`` -> ``PermissionError``)
Attributes:
allow_bucket_creation: Whether mkdir/makedirs may create buckets.
Defaults to False.
allow_bucket_deletion: Whether rmdir may delete buckets.
Defaults to False.
version_aware: Whether reads pin the object version observed at
open time and ls may list all versions. Requires the
s3:GetObjectVersion / s3:ListBucketVersions permissions.
Defaults to False.
Example:
>>> from pyathena.filesystem.s3 import S3FileSystem
>>> fs = S3FileSystem()
>>>
>>> # List objects in a bucket
>>> files = fs.ls('s3://my-bucket/data/')
>>>
>>> # Read a file
>>> with fs.open('s3://my-bucket/data/file.csv', 'r') as f:
... content = f.read()
>>>
>>> # Write a file
>>> with fs.open('s3://my-bucket/output/result.txt', 'w') as f:
... f.write('Hello, S3!')
>>>
>>> # Copy files
>>> fs.cp('s3://source-bucket/file.txt', 's3://dest-bucket/file.txt')
Note:
This filesystem is used internally by PyAthena for handling query results
stored in S3, but can also be used independently for S3 file operations.
"""
DEFAULT_BLOCK_SIZE: int = 5 * 2**20 # 5MiB
protocol = ("s3", "s3a")
_extra_tokenize_attributes = ("default_block_size",)
# The boto3 session arguments of the client that the filesystem builds.
_SESSION_PASSING_ARGS: ClassVar[frozenset[str]] = frozenset(
{
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
"region_name",
"botocore_session",
"profile_name",
}
)
# The arguments of ``Session.client()`` taken from ``client_kwargs``.
_CLIENT_PASSING_ARGS: ClassVar[frozenset[str]] = frozenset(
{
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
"api_version",
"use_ssl",
"verify",
"endpoint_url",
"region_name",
"config",
}
)
# The arguments from which the filesystem builds its client, which an
# injected client (``client`` or ``connection``) does not take.
_CLIENT_BUILDING_ARGS: ClassVar[frozenset[str]] = _SESSION_PASSING_ARGS | {
"key",
"username",
"secret",
"password",
"token",
"profile",
"anon",
"use_ssl",
"endpoint_url",
"connect_timeout",
"read_timeout",
"client_kwargs",
"config_kwargs",
}
[docs]
def __init__(
self,
connection: Connection[Any] | None = None,
default_block_size: int | None = None,
default_cache_type: str | None = None,
max_workers: int | None = None,
s3_additional_kwargs=None,
allow_bucket_creation: bool = False,
allow_bucket_deletion: bool = False,
version_aware: bool = False,
*args,
batch_size: int | None = None,
client: BaseClient | None = None,
retry_config: RetryConfig | None = None,
**kwargs,
) -> None:
"""Create a filesystem for Amazon S3.
The filesystem sends its requests with the boto3 S3 client given as
``client``, or with a client that it builds from s3fs-compatible
arguments in ``kwargs``. :func:`pyathena.filesystem.s3_filesystem`
creates one with the S3 client and retry policy of a PyAthena
connection.
Args:
connection: Deprecated; use ``client`` and ``retry_config``, or
:func:`pyathena.filesystem.s3_filesystem`. A PyAthena connection
whose S3 client (``Connection.s3_client``) and retry policy the
filesystem uses, with a ``DeprecationWarning``.
default_block_size: The block size for reads and writes; defaults to
``DEFAULT_BLOCK_SIZE``.
default_cache_type: The fsspec cache type for reads, the name of a
cache in ``fsspec.caching.caches``; defaults to ``"bytes"``. The
available names depend on the installed fsspec version. Common
ones are ``"none"``, which keeps nothing and requests the bytes
of each read; ``"bytes"``, which reads one block beyond each
read and keeps one contiguous range; ``"readahead"``, which
reads one block beyond each read and keeps only the last range
read; ``"blockcache"``, which requests whole blocks and keeps
the most recently used ones; and ``"background"``, which is like
``"blockcache"`` and also reads the next block in a thread.
max_workers: The number of threads of the request pool of the
filesystem, which its files and parallel operations share,
and the default number of parallel requests of a file. If
None, the ``max_pool_connections`` of the S3 client (10
unless configured). A client that the filesystem builds gets
``max_pool_connections`` from ``max_workers`` unless
``config_kwargs`` sets it; the connection pool of a given
``client`` keeps its size.
s3_additional_kwargs: Extra arguments for the object requests of
``open()``, ``pipe_file()`` and ``put_file()``. Their
authorization parameters (``RequestPayer``,
``ExpectedBucketOwner`` and ``SSECustomer*``) are sent with
every request, listings and copies included; a copy also
sends them for its source (``ExpectedSourceBucketOwner`` and
``CopySourceSSECustomer*``). Each request receives those that
its operation accepts, and the parameters of a call take
precedence.
allow_bucket_creation: Whether ``mkdir``/``makedirs`` may create a
bucket.
allow_bucket_deletion: Whether ``rmdir`` may delete a bucket.
version_aware: Whether reads pin the object version observed at open
time.
*args: Passed to ``fsspec.AbstractFileSystem``.
batch_size: The number of per-object calls of a batch operation
(``cat``, ``get``, ``put``, ``copy``, and the like) that run at
once, and of DeleteObjects requests of ``rm``, unless the call
gives its own. None or -1 is ``max_workers``.
client: The boto3 S3 client that the filesystem sends its requests
with. The filesystem does not close it. A filesystem with a
given client cannot be pickled.
retry_config: The retry policy of the requests; defaults to
``RetryConfig()``. A policy with the default ``exceptions``
also retries the transient S3 error codes
(:attr:`~pyathena.filesystem.s3_core.S3Core.RETRY_ERROR_CODES`).
**kwargs: Passed to ``fsspec.AbstractFileSystem``; without
``client`` or ``connection``, also s3fs-compatible client
arguments. ``requester_pays=True`` sends requester-pays
requests with the operations that accept ``RequestPayer``.
Raises:
ValueError: If ``batch_size`` is neither positive, -1, nor None.
TypeError: If ``connection`` is given with ``client`` or
``retry_config``, or ``client`` or ``connection`` with an
argument from which the filesystem builds a client, such as
``key``, ``profile``, ``anon``, ``endpoint_url``,
``client_kwargs``, or ``region_name``, other than one that
fsspec's configuration supplies.
"""
super().__init__(*args, **kwargs)
self.batch_size = self._check_batch_size(batch_size)
self._client_injected = client is not None or connection is not None
client, retry_config = self._resolve_injected_client(
type(self), connection, client, retry_config, kwargs
)
if client is None:
client = self._get_client_compatible_with_s3fs(max_workers=max_workers, **kwargs)
self.default_block_size = (
default_block_size if default_block_size else self.DEFAULT_BLOCK_SIZE
)
self.default_cache_type = default_cache_type if default_cache_type else "bytes"
# The requests of the files and parallel operations of the
# filesystem run in this pool, at most max_workers at once.
self._request_pool = S3ThreadPool(
max_workers if max_workers is not None else client.meta.config.max_pool_connections
)
self.max_workers: int = self._request_pool.max_workers
self.s3_additional_kwargs = s3_additional_kwargs if s3_additional_kwargs else {}
self.allow_bucket_creation = allow_bucket_creation
self.allow_bucket_deletion = allow_bucket_deletion
self.version_aware = version_aware
requester_pays = kwargs.pop("requester_pays", False)
# The parameters that authorize the access to the objects, sent with
# every request that accepts them, and with a copy for its source.
request_kwargs = self._get_lookup_kwargs(self.s3_additional_kwargs)
if requester_pays:
request_kwargs.setdefault("RequestPayer", "requester")
self._core = S3Core(
client,
retry_config=retry_config if retry_config is not None else RetryConfig(),
request_kwargs={**request_kwargs, **S3Core.copy_from_source_params(request_kwargs)},
)
@classmethod
def _resolve_injected_client(
cls,
filesystem_class: type[AbstractFileSystem],
connection: Connection[Any] | None,
client: BaseClient | None,
retry_config: RetryConfig | None,
kwargs: Mapping[str, Any],
) -> tuple[BaseClient | None, RetryConfig | None]:
"""Check the injected client of a filesystem and resolve it from a connection.
Warns that ``connection`` is deprecated, and rejects the arguments that
conflict with an injected client before the client of the connection is
used. ``AioS3FileSystem`` resolves its arguments with it too.
Args:
filesystem_class: The class of the filesystem being created, whose
name the warning shows and whose fsspec configuration applies.
connection: The deprecated ``connection`` argument.
client: The ``client`` argument.
retry_config: The ``retry_config`` argument.
kwargs: The other keyword arguments of the filesystem, merged with
fsspec's configuration for the protocol.
Returns:
The injected client and retry policy, or the given ones if
``connection`` is None.
Raises:
TypeError: If the arguments conflict with an injected client.
"""
if connection is not None:
message = _CONNECTION_DEPRECATION_MESSAGE.format(cls=filesystem_class.__name__)
if sys.version_info >= (3, 12):
# Attributed to the first caller outside fsspec and the filesystems,
# such as through fsspec.filesystem() or a subclass.
warnings.warn(
message,
DeprecationWarning,
skip_file_prefixes=(
os.path.dirname(fsspec.__file__) + os.sep,
os.path.dirname(__file__) + os.sep,
),
)
else:
# Past this method, __init__, and fsspec's metaclass.
warnings.warn(message, DeprecationWarning, stacklevel=4)
if client is not None or retry_config is not None:
raise TypeError("connection cannot be combined with client or retry_config.")
if client is not None or connection is not None:
# The settings of fsspec's configuration for the protocol, such as
# the FSSPEC_S3_* environment variables, apply to every filesystem
# and are ignored instead.
configured = apply_config(filesystem_class, {})
building_args = sorted(
k
for k in cls._CLIENT_BUILDING_ARGS
if kwargs.get(k) is not None and (k not in configured or kwargs[k] != configured[k])
)
if building_args:
raise TypeError(
f"{', '.join(building_args)} cannot be combined with client or "
"connection, whose client the filesystem uses as it is."
)
if connection is not None:
return connection.s3_client, connection.retry_config
return client, retry_config
[docs]
@override
def __reduce__(self) -> tuple[Any, ...]:
"""Reduce the filesystem for pickling, as fsspec does.
Returns:
The reduction of ``fsspec.AbstractFileSystem``, which creates the
filesystem again from its arguments.
Raises:
TypeError: If the filesystem was built with ``client`` or
``connection``, whose boto3 client cannot be pickled.
"""
if self._client_injected:
raise TypeError(_UNPICKLABLE_CLIENT_MESSAGE.format(cls=type(self).__name__))
return cast(tuple[Any, ...], super().__reduce__())
@property
def core(self) -> S3Core:
"""The typed S3 operations that the filesystem sends its requests with."""
return self._core
@property
def _client(self) -> BaseClient:
"""The boto3 S3 client of the filesystem (``core.client``)."""
return self._core.client
@property
def _retry_config(self) -> RetryConfig:
"""The retry policy of the requests (``core.retry_config``)."""
return self._core.retry_config
@property
def request_kwargs(self) -> dict[str, Any]:
"""The parameters sent with every operation that accepts them.
``{"RequestPayer": "requester"}`` with ``requester_pays=True``, and
the authorization parameters of ``s3_additional_kwargs``, also under
the names of a copy source; see ``core.request_kwargs``.
"""
return self._core.request_kwargs
[docs]
def close(self) -> None:
"""Release the threads of the request pool, which stays usable, once they are idle.
See :meth:`S3ThreadPool.release()
<pyathena.filesystem.s3_executor.S3ThreadPool.release>`. The calls in
progress, including the requests of open files, run to their end, and
a call after the release creates new threads.
"""
self._request_pool.release()
[docs]
def __enter__(self) -> S3FileSystem:
"""Return the filesystem, whose threads the end of the block releases."""
return self
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
self.close()
def _get_client_compatible_with_s3fs(
self, max_workers: int | None = None, **kwargs
) -> BaseClient:
"""Build a boto3 S3 client from s3fs-compatible constructor arguments.
Accepts the constructor arguments that s3fs users pass through fsspec
storage options — ``key``/``username``, ``secret``/``password``,
``token``, ``profile``, ``anon``, ``use_ssl``, ``endpoint_url``,
``connect_timeout``/``read_timeout``, and the ``client_kwargs`` /
``config_kwargs`` dictionaries — in addition to boto3 session
arguments such as ``region_name`` and ``profile_name``. ``profile``
is used as ``profile_name`` when ``profile_name`` is not given.
Args:
max_workers: The size of the request pool of the filesystem, which
becomes the ``max_pool_connections`` of the client unless
``config_kwargs`` sets it; None keeps botocore's default.
**kwargs: The filesystem constructor arguments.
Returns:
A boto3 S3 client configured from the arguments.
"""
config_kwargs = deepcopy(kwargs.pop("config_kwargs", {}))
client_kwargs = deepcopy(kwargs.pop("client_kwargs", {}))
if max_workers is not None:
config_kwargs.setdefault("max_pool_connections", max_workers)
user_agent_extra = config_kwargs.pop("user_agent_extra", None)
if user_agent_extra and pyathena.user_agent_extra not in user_agent_extra:
user_agent_extra = f"{pyathena.user_agent_extra} {user_agent_extra}"
config_kwargs.update({"user_agent_extra": user_agent_extra or pyathena.user_agent_extra})
if connect_timeout := kwargs.pop("connect_timeout", None):
config_kwargs.update({"connect_timeout": connect_timeout})
if read_timeout := kwargs.pop("read_timeout", None):
config_kwargs.update({"read_timeout": read_timeout})
use_ssl = kwargs.pop("use_ssl", None)
if use_ssl is not None:
client_kwargs.update({"use_ssl": use_ssl})
if endpoint_url := kwargs.pop("endpoint_url", None):
client_kwargs.update({"endpoint_url": endpoint_url})
if kwargs.pop("anon", False):
config_kwargs.update({"signature_version": UNSIGNED})
else:
creds = {
key: value
for key, value in {
"aws_access_key_id": kwargs.pop("key", kwargs.pop("username", None)),
"aws_secret_access_key": kwargs.pop("secret", kwargs.pop("password", None)),
"aws_session_token": kwargs.pop("token", None),
}.items()
if value is not None
}
kwargs.update(creds)
client_kwargs.update(creds)
if profile := kwargs.pop("profile", None):
kwargs.setdefault("profile_name", profile)
session = Session(**{k: v for k, v in kwargs.items() if k in self._SESSION_PASSING_ARGS})
return session.client(
"s3",
config=Config(**config_kwargs),
**{k: v for k, v in client_kwargs.items() if k in self._CLIENT_PASSING_ARGS},
)
@staticmethod
def _directory_object(bucket: str, key: str | None, version_id: str | None = None) -> S3Object:
"""Build an S3Object representing a directory entry."""
return S3Object(
init={
"ContentLength": 0,
"ContentType": None,
"StorageClass": S3StorageClass.S3_STORAGE_CLASS_DIRECTORY,
"ETag": None,
"LastModified": None,
},
type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY,
bucket=bucket,
key=key,
version_id=version_id,
)
def _head_bucket(
self,
bucket,
refresh: bool = False,
lookup_kwargs: Mapping[str, Any] | None = None,
) -> S3Object | None:
"""Get the bucket as a directory object with HeadBucket.
The result is cached under the bucket name, apart for each set of
lookup parameters (see ``_get_cached_lookup``). A missing bucket
evicts its entries and the cached bucket listing that still lists it.
Args:
bucket: The bucket name.
refresh: If True, bypass the cache and call HeadBucket.
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``)
to send with the request.
Returns:
The bucket object, or None if the bucket does not exist.
"""
lookup_kwargs = lookup_kwargs or {}
file = None if refresh else self._get_cached_lookup(bucket, lookup_kwargs)
if file is None:
try:
bucket_entry = self.core.head_bucket(
bucket, **self.core.operation_params("head_bucket", lookup_kwargs)
)
except FileNotFoundError:
self._evict_cache(bucket)
self._evict_cache((bucket, _LOOKUPS_CACHE_KEY))
# Evict the cached bucket listing only if it still lists the bucket.
buckets = self.dircache.get("")
if buckets and any(b.name == bucket for b in buckets):
self._evict_cache("")
return None
file = S3Object.from_bucket(bucket_entry)
self._cache_lookup(bucket, lookup_kwargs, file)
return file
def _head_object(
self,
path: S3Path,
refresh: bool = False,
lookup_kwargs: Mapping[str, Any] | None = None,
) -> S3Object | None:
"""Get the object with HeadObject.
The result is cached under the lookup cache key of the path (see
:meth:`_lookup_cache_key`), which keeps an explicit version apart
from the key and from other versions, apart for each set of lookup
parameters (see ``_get_cached_lookup``). An explicitly requested
``"null"`` version is not cached. A missing object evicts its entries
and, unless a version was requested, the cached listing of its parent
that still lists it.
Args:
path: The path of the object, optionally with a version ID.
refresh: If True, bypass the cache and call HeadObject.
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``)
to send with the request.
Returns:
The object, or None if it does not exist.
"""
version_id = path.version_id
# Chosen before a version_aware lookup pins the version, so that the
# lookup of the key stays cached under the key.
cache_key = self._lookup_cache_key(path)
# Writes invalidate only the path without the version, and an
# overwrite replaces the "null" version of a bucket without
# versioning, so that version is looked up every time.
cacheable = version_id != "null"
lookup_kwargs = lookup_kwargs or {}
file = None if refresh else self._get_cached_lookup(cache_key, lookup_kwargs)
if file is None:
try:
metadata = self.core.head_object(
path, **self.core.operation_params("head_object", lookup_kwargs)
)
except FileNotFoundError:
self._evict_cache(cache_key)
self._evict_cache((cache_key, _LOOKUPS_CACHE_KEY))
if not version_id:
# Evict the cached listing of the parent only if it still
# lists the path.
parent_key = (self._listing_name(path), "/")
files = self.dircache.get(parent_key)
if files and any(f.name == path.name for f in files):
self._evict_cache(parent_key)
return None
if self.version_aware and not version_id:
# Pin the version of the object so that subsequent reads see
# the version observed here even if the object is overwritten.
version_id = metadata.version_id
file = S3Object.from_metadata(metadata, version_id=version_id)
if cacheable:
self._cache_lookup(cache_key, lookup_kwargs, file)
return file
@staticmethod
def _lookup_cache_key(path: S3Path) -> str | tuple[str, str, str]:
"""Return the dircache key of the lookup of a path.
A path without a version is cached under its name, so that a key that
ends in a query such as ``?versionId=v1`` is cached under its own
name. An explicit version is cached under
``(name, _VERSION_CACHE_KEY, version_id)``, which no name equals.
Args:
path: The path of the bucket or the object.
Returns:
The dircache key.
"""
if path.version_id:
return path.name, _VERSION_CACHE_KEY, path.version_id
return path.name
@staticmethod
def _listing_name(path: S3Path) -> str:
"""Return the name of the directory whose listing holds the key of a path.
The name of the key is cut at its last slash, without parsing it, so
that the directory is named as its listing names it: ``bucket/a`` for
the key ``a/x``, ``bucket/a/`` for ``a//x``, and
``bucket/a/dir?versionId=v1`` for ``a/dir?versionId=v1/x``.
Args:
path: The path of an object.
Returns:
The name of the directory, or the bucket for a key without a
slash.
"""
return path.name.rpartition("/")[0]
def _ls_buckets(self, refresh: bool = False) -> list[S3Object]:
"""List the buckets with ListBuckets, following its pages.
The listing is cached under ``""``.
Args:
refresh: If True, bypass the cache and call ListBuckets.
Returns:
The buckets as directory objects.
"""
buckets = None if refresh else self.dircache.get("")
if buckets is None:
buckets = [
S3Object.from_bucket(b) for page in self.core.list_buckets() for b in page.buckets
]
self.dircache[""] = buckets
return buckets
def _ls_dirs(
self,
path: str,
prefix: str = "",
delimiter: str = "/",
refresh: bool = False,
) -> list[S3Object]:
"""List the objects and common prefixes under a bucket or key path.
Lists the directory that the path names, as :meth:`_ls_prefix`
describes.
Args:
path: The bucket or key path to list, without the protocol or a
trailing slash.
prefix: Key prefix to filter by, relative to the path. A prefixed
listing is neither read from nor written to the cache.
delimiter: Delimiter to group keys by; ``""`` lists recursively.
refresh: If True, bypass the cache and list from S3.
Returns:
The listed directories and files.
"""
s3_path = S3Path.parse(path)
return self._ls_prefix(
self._directory_object(s3_path.bucket, s3_path.key),
prefix=prefix,
delimiter=delimiter,
refresh=refresh,
)
@staticmethod
def _directory_prefix(directory: S3Object) -> str:
"""Return the key prefix that a listed directory or bucket lists.
A directory is named after its common prefix without the final
delimiter, which may leave the name with a version ID query or with
trailing slashes. The prefix is taken from its key instead, so that
the directory is listed as it was listed.
Args:
directory: A directory or bucket entry of a listing.
Returns:
The key prefix: ``""`` for a bucket, and otherwise the key of the
directory followed by ``/``.
"""
return "" if directory.key is None else f"{directory.key}/"
def _ls_prefix(
self,
directory: S3Object,
prefix: str = "",
delimiter: str = "/",
refresh: bool = False,
) -> list[S3Object]:
"""List the objects and common prefixes under the key prefix of a directory.
A complete, non-empty listing of the directory is cached under
``(name, delimiter)``, with the name of the directory, and an empty
one, or a listing that raises FileNotFoundError because the bucket
does not exist, evicts it. ``invalidate_cache`` drops it when the
path or a path under it is invalidated.
A common prefix is named after its prefix without the final
delimiter, so the common prefix ``dir//`` of a key with an empty
segment, such as ``dir//x``, is the directory ``bucket/dir/``, and
``/`` at the top of the bucket is ``bucket/``. Its key is the prefix
without the final delimiter (see :meth:`_directory_prefix`).
Args:
directory: The bucket or directory to list, as a listing or
:meth:`_directory_object` returns it.
prefix: Key prefix to filter by, relative to the directory. A
prefixed listing is neither read from nor written to the
cache.
delimiter: Delimiter to group keys by; ``""`` lists recursively.
refresh: If True, bypass the cache and list from S3.
Returns:
The listed directories and files.
"""
bucket = directory.bucket
use_cache = not prefix
cache_key = (directory.name, delimiter)
cached = self.dircache.get(cache_key) if use_cache and not refresh else None
if cached is not None:
return cast(list[S3Object], cached)
files: list[S3Object] = []
try:
for page in self.core.list_objects(
bucket,
prefix=self._directory_prefix(directory) + prefix,
delimiter=delimiter,
):
files.extend(
self._directory_object(bucket, c.prefix[:-1]) for c in page.common_prefixes
)
files.extend(S3Object.from_summary(o) for o in page.objects)
except FileNotFoundError:
# The bucket does not exist, so the cached listing is stale.
self._evict_cache(cache_key)
raise
if use_cache:
if files:
self.dircache[cache_key] = files
else:
self._evict_cache(cache_key)
return files
[docs]
def ls(
self, path: str | os.PathLike[str], detail: bool = False, refresh: bool = False, **kwargs
) -> list[S3Object] | list[str]:
"""List contents of an S3 path.
Lists buckets (when path is root) or objects within a bucket/prefix.
Compatible with fsspec interface for filesystem operations.
Args:
path: S3 path to list (e.g., "s3://bucket" or "s3://bucket/prefix").
detail: If True, return S3Object instances; if False, return paths as strings.
refresh: If True, bypass cache and fetch fresh results from S3.
**kwargs: Additional arguments including:
versions: If True, list all versions of the objects, named
``bucket/key?versionId=<id>`` except for the ``null``
version. Requires the filesystem to be constructed with
``version_aware=True``.
Returns:
List of S3Object instances (if detail=True) or paths as strings (if detail=False).
Raises:
FileNotFoundError: If the bucket does not exist, or if a key path
is neither an object nor a directory, a path that some key
starts with followed by ``/``, or if HeadObject reports the
version that a key path with a version ID names missing.
Example:
>>> fs = S3FileSystem()
>>> fs.ls("s3://my-bucket") # List objects in bucket
>>> fs.ls("s3://my-bucket/", detail=True) # Get detailed object info
"""
versions = kwargs.pop("versions", False)
if versions and not self.version_aware:
raise ValueError(
"Cannot list the object versions unless the filesystem is version aware."
)
path = self._strip_protocol(path).rstrip("/")
if path in ["", "/"]:
files = self._ls_buckets(refresh)
else:
s3_path = S3Path.parse(path)
key = s3_path.key
if key and s3_path.version_id:
files = self._ls_version(s3_path, refresh=refresh)
elif versions:
files = self._ls_object_versions(path)
else:
files = self._ls_dirs(path, refresh=refresh)
if not files and key:
file = self._head_object(s3_path, refresh=refresh)
if file:
files = [file]
# An empty bucket lists nothing, while a missing bucket raises
# from the listing.
if not files and key:
raise FileNotFoundError(path)
return list(files) if detail else [f.name for f in files]
def _ls_version(self, s3_path: S3Path, refresh: bool = False) -> list[S3Object]:
"""List the object version that a key path with a version ID names.
A version names one object, which has nothing below it, so no
directory is listed. The entry is named as ``ls(versions=True)``
names a version.
Args:
s3_path: The key path with a version ID.
refresh: If True, bypass the cache and call HeadObject.
Returns:
The entry of the version, or an empty list if it does not exist.
"""
file = self._head_object(s3_path, refresh=refresh)
if not file:
return []
if s3_path.version_id != "null":
# The cached entry of the lookup is named without the version.
file = file.copy()
file.name = str(s3_path)
return [file]
def _ls_object_versions(self, path: str) -> list[S3Object]:
"""List a prefix including all versions of the objects.
The listing is always fetched from S3 and is not cached, because the
dircache stores the current view of a path. A key path with nothing
under it lists the versions of its object.
Args:
path: The bucket or key path, without the protocol.
Returns:
The listed directories and versions.
"""
s3_path = S3Path.parse(path)
files = self._ls_prefix_versions(self._directory_object(s3_path.bucket, s3_path.key))
if not files and s3_path.key:
# The path may point at an object rather than a key prefix.
files = [
S3Object.from_version(v)
for page in self.core.list_object_versions(
s3_path.bucket, prefix=s3_path.key, delimiter="/"
)
for v in page.versions
if v.key == s3_path.key
]
return files
def _ls_prefix_versions(self, directory: S3Object) -> list[S3Object]:
"""List the common prefixes and the versions of the objects under a directory.
Args:
directory: The bucket or directory to list (see
:meth:`_directory_prefix`).
Returns:
The listed directories and versions, not cached.
"""
bucket = directory.bucket
files: list[S3Object] = []
for page in self.core.list_object_versions(
bucket, prefix=self._directory_prefix(directory), delimiter="/"
):
files.extend(
self._directory_object(bucket, c.prefix[:-1]) for c in page.common_prefixes
)
files.extend(S3Object.from_version(v) for v in page.versions)
return files
def _ls_directory(
self, directory: S3Object, refresh: bool = False, versions: bool = False
) -> list[S3Object]:
"""List a listed directory or bucket by its key prefix, as ``ls()`` lists a path.
Args:
directory: A directory or bucket entry of a listing (see
:meth:`_directory_prefix`).
refresh: If True, bypass the cache and list from S3.
versions: Whether to list all versions of the objects, as
``ls(versions=True)`` does.
Returns:
A new list of the listed directories and files.
Raises:
FileNotFoundError: If nothing is listed under a directory, such as
one whose keys have been deleted since it was listed.
"""
if versions:
files = self._ls_prefix_versions(directory)
else:
files = self._ls_prefix(directory, refresh=refresh)
if not files and directory.key is not None:
raise FileNotFoundError(directory.name)
return list(files)
[docs]
def info(self, path: str | os.PathLike[str] | S3Path, **kwargs) -> S3Object:
"""Return information about an S3 path.
Uses the directory cache first: a cached entry for the path is
returned, the entry of the path in a cached listing of its parent is
returned, preferring an object to a key prefix of the same name as
HeadObject does, and a cached listing of its parent without it means
it does not exist.
The cached bucket listing holds only the buckets that the caller owns,
so a bucket missing from it is looked up with HeadBucket.
Otherwise, a key path is looked up with HeadObject and, if no object
exists, with a ListObjectsV2 request (``Delimiter="/"``,
``MaxKeys=1``) that checks whether it is a key prefix; a bucket path
is looked up with HeadBucket. If these requests find a listed object
missing, or find a key prefix, the cached listing of the parent is
removed. With ``version_aware``, a cached file
entry without a version ID is looked up again. With an explicit
version, the cached entries of the path are skipped, the
HeadObject result is cached under the name and the version ID, apart
from the key and other versions, except for the ``null`` version, which an
overwrite replaces, and a missing version is not looked up as a key
prefix, since a version names an object. With request parameters on which the
authorization of the requests depends (``ExpectedBucketOwner``,
``RequestPayer``, and the ``SSECustomer*`` parameters of an object
encrypted with a customer-provided key), each request receives those
that its operation accepts, and only the cached HeadObject or
HeadBucket results of lookups with the same values of these
parameters are used.
Args:
path: S3 path (e.g., "s3://bucket" or "s3://bucket/key"), or an
``S3Path``, whose bucket, key and version are looked up as
they are, such as a key that a listing returns.
**kwargs: Additional arguments including:
refresh: If True, bypass the cache and query S3.
version_id: The version ID to look up when the path has none.
ExpectedBucketOwner, RequestPayer, SSECustomerAlgorithm,
SSECustomerKey, SSECustomerKeyMD5: The request parameters
described above. Other request parameters are ignored.
Returns:
S3Object describing the bucket, directory, or file. The root path
(``""``, ``"/"`` or ``"s3://"``) is a directory.
Raises:
FileNotFoundError: If the path does not exist.
"""
refresh = kwargs.pop("refresh", False)
version_id = kwargs.pop("version_id", None)
if isinstance(path, S3Path):
s3_path = path
else:
name = self._strip_protocol(path)
if name in ["/", ""]:
# S3Path.parse rejects the root path.
return S3Object(
init={
"ContentLength": 0,
"ContentType": None,
"StorageClass": S3StorageClass.S3_STORAGE_CLASS_BUCKET,
"ETag": None,
"LastModified": None,
},
type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY,
bucket="",
key=None,
version_id=None,
)
s3_path = S3Path.parse(name)
if version_id and not s3_path.version_id:
s3_path = s3_path.with_version_id(version_id)
return self._path_info(
s3_path, refresh=refresh, lookup_kwargs=self._get_lookup_kwargs(kwargs)
)
def _path_info(
self,
path: S3Path,
refresh: bool = False,
lookup_kwargs: Mapping[str, Any] | None = None,
) -> S3Object:
"""Return information about the bucket, object or directory of a path.
As :meth:`info` describes, for a path whose bucket, key and version
are given as they are, such as a key listed with a version ID query
in its name.
Args:
path: The path, with the version to look up, if any.
refresh: If True, bypass the cache and query S3.
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``).
Returns:
S3Object describing the bucket, directory, or file.
Raises:
FileNotFoundError: If the path does not exist.
"""
version_id = path.version_id
lookup_kwargs = lookup_kwargs or {}
name = path.name
# Cached entries describe the current version of a path as looked up
# without lookup parameters, so an explicit version or lookup
# parameters use only the HeadObject cache of that version and those
# parameters.
if not refresh and not version_id and not lookup_kwargs:
caches: list[S3Object] | S3Object | None = self._ls_from_cache(path)
if caches is not None:
if isinstance(caches, list):
matches = [c for c in caches if c.name == name]
# A key can be both an object and a key prefix.
cache = next(
(c for c in matches if c.type == S3ObjectType.S3_OBJECT_TYPE_FILE),
next(iter(matches), None),
)
elif caches.name == name:
cache = caches
else:
cache = None
if cache:
if (
self.version_aware
and cache.get("type") == S3ObjectType.S3_OBJECT_TYPE_FILE
and not cache.get("version_id")
):
# A version-aware lookup needs the version to pin;
# treat a version-less cached entry (e.g., populated
# by a listing) as stale and head the object again.
refresh = True
else:
return cache
else:
return self._directory_object(
path.bucket, path.key.rstrip("/") if path.key else None, version_id
)
if path.key:
object_info = self._head_object(path, refresh=refresh, lookup_kwargs=lookup_kwargs)
if object_info:
return object_info
if version_id:
raise FileNotFoundError(str(path))
else:
bucket_info = self._head_bucket(
path.bucket, refresh=refresh, lookup_kwargs=lookup_kwargs
)
if bucket_info:
return bucket_info
raise FileNotFoundError(str(path))
page = self.core.list_objects_page(
path.bucket,
prefix=f"{path.key.rstrip('/')}/" if path.key else "",
delimiter="/",
max_keys=1,
**self.core.operation_params("list_objects_v2", lookup_kwargs),
)
if (page.key_count or 0) > 0 or page.objects or page.common_prefixes:
# Nothing caches the key prefix, and the cached listing of the
# parent may predate it.
self._evict_cache((self._listing_name(path), "/"))
return self._directory_object(
path.bucket, path.key.rstrip("/") if path.key else None, version_id
)
raise FileNotFoundError(str(path))
def _extract_parent_directories(
self, files: list[S3Object], bucket: str, base_prefix: str
) -> list[S3Object]:
"""Extract parent directory objects from file paths.
When listing files without delimiter, S3 doesn't return directory entries.
This method creates directory objects by analyzing file paths.
Args:
files: List of S3Object instances representing files.
bucket: S3 bucket name.
base_prefix: The key prefix below which the directories are
derived: ``""``, or a prefix that ends in ``/``, such as
``a//`` for the keys ``a//x/y``.
Returns:
List of S3Object instances representing directories.
"""
dirs = set()
for f in files:
if (
f.key
and f.type == S3ObjectType.S3_OBJECT_TYPE_FILE
and f.key.startswith(base_prefix)
):
parts = f.key[len(base_prefix) :].split("/")
for i in range(1, len(parts)):
dirs.add(base_prefix + "/".join(parts[:i]))
return [self._directory_object(bucket, dir_path) for dir_path in dirs]
def _find(
self,
path: str | os.PathLike[str],
maxdepth: int | None = None,
withdirs: bool | None = None,
**kwargs,
) -> list[S3Object]:
"""List the objects below a path, as described in ``find``.
Args:
path: S3 path to search under.
maxdepth: Maximum number of levels to descend, at least 1
(None for unlimited).
withdirs: Whether to include directories in the result.
**kwargs: Additional arguments including ``prefix`` and
``refresh``, as described in ``find``.
Returns:
The objects found, and the directories if ``withdirs`` is True.
Raises:
ValueError: If ``maxdepth`` is less than 1 or the path is the root.
"""
return [f for f, _ in self._find_paths(path, maxdepth, withdirs, **kwargs)]
def _find_paths(
self,
path: str | os.PathLike[str],
maxdepth: int | None = None,
withdirs: bool | None = None,
**kwargs,
) -> list[tuple[S3Object, str | S3Path]]:
"""List the objects below a path as :meth:`_find` does, with the paths they name.
A listed entry names its bucket and key (see :meth:`_entry_path`),
and the path itself, also a version that its entry names after its
key alone, is named by the path string as given (see
:meth:`_find_under`).
Args:
path: S3 path to search under.
maxdepth: Maximum number of levels to descend, at least 1
(None for unlimited).
withdirs: Whether to include directories in the result.
**kwargs: Additional arguments including ``prefix`` and
``refresh``, as described in ``find``.
Returns:
The objects found, and the directories if ``withdirs`` is True,
with the paths that they name.
Raises:
ValueError: If ``maxdepth`` is less than 1 or the path is the root.
"""
if maxdepth is not None and maxdepth < 1:
raise ValueError("maxdepth must be at least 1")
path = self._strip_protocol(path)
if path in ["", "/"]:
raise ValueError("Cannot traverse all files in S3.")
s3_path = S3Path.parse(path)
prefix = kwargs.pop("prefix", "")
refresh = kwargs.pop("refresh", False)
if s3_path.key and s3_path.version_id:
return [(f, path) for f in self._ls_version(s3_path, refresh=refresh)]
return self._find_under(
self._directory_object(s3_path.bucket, s3_path.key),
maxdepth=maxdepth,
withdirs=withdirs,
prefix=prefix,
refresh=refresh,
name=path,
)
def _find_under(
self,
directory: S3Object,
maxdepth: int | None = None,
withdirs: bool | None = None,
prefix: str = "",
refresh: bool = False,
name: str | None = None,
) -> list[tuple[S3Object, str | S3Path]]:
"""List the objects below a directory, as described in ``find``, with the paths they name.
The directory is listed by its key prefix (see
:meth:`_directory_prefix`), never by parsing its name. A listed entry
names its bucket and key (see :meth:`_entry_path`).
Args:
directory: The bucket, the directory or the object to search
under, as a listing or :meth:`_directory_object` returns it.
maxdepth: Maximum number of levels to descend, at least 1
(None for unlimited).
withdirs: Whether to include directories in the result.
prefix: Key prefix, relative to the directory, to filter the
listed keys by.
refresh: If True, bypass the cache and list from S3.
name: The path string that the caller passed for the directory,
which names the directory itself and which ``info()`` looks
up when nothing is listed below it; None for a listed
directory, which names its bucket and key.
Returns:
The objects found, and the directories if ``withdirs`` is True,
with the paths that they name.
"""
bucket, key = directory.bucket, directory.key
own: str | S3Path = name if name is not None else S3Path(bucket, key)
if maxdepth is not None:
# The entries listed with the prefix lie as many levels further
# below the path as the prefix has slashes.
files = self._find_levels(
directory, maxdepth - prefix.count("/"), prefix=prefix, refresh=refresh
)
else:
files = self._ls_prefix(directory, prefix=prefix, delimiter="", refresh=refresh)
# S3 doesn't return directory entries without a delimiter, so the
# directories are derived from the listed keys, below the last
# slash of the listed key prefix, as with maxdepth.
if withdirs:
listed = self._directory_prefix(directory) + prefix
# Build a new list; files may be the cached listing.
files = files + self._extract_parent_directories(
files, bucket, listed[: listed.rfind("/") + 1]
)
found: list[tuple[S3Object, str | S3Path]] = [(f, self._entry_path(f)) for f in files]
if found:
# Something is listed below the path, so the path is a directory,
# which fsspec includes with the directories. A bucket is not
# included, since cp_file() cannot copy it in a recursive copy.
if withdirs and key:
found = [(self._directory_object(bucket, key), own), *found]
elif key:
# As in fsspec, the path itself is returned if it is an object,
# or with withdirs if it is a directory.
try:
if name is not None:
found = [(self.info(name, refresh=refresh), own)]
else:
found = [(self._path_info(S3Path(bucket, key), refresh=refresh), own)]
except FileNotFoundError:
found = []
if not withdirs:
found = [(f, t) for f, t in found if f.type != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY]
return found
def _find_levels(
self, directory: S3Object, maxdepth: int, prefix: str = "", refresh: bool = False
) -> list[S3Object]:
"""List the entries below a directory level by level with ``Delimiter="/"``.
Each listed directory is listed by its key prefix, never by parsing
its name (see :meth:`_directory_prefix`).
Args:
directory: The bucket or directory to search under.
maxdepth: Number of levels to list.
prefix: Key prefix, relative to the directory, to filter the
first level by.
refresh: If True, bypass the cache and list from S3.
Returns:
The objects and directories found.
"""
if maxdepth < 1:
return []
result: list[S3Object] = []
for item in self._ls_prefix(directory, prefix=prefix, delimiter="/", refresh=refresh):
result.append(item)
if item.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY:
result.extend(self._find_levels(item, maxdepth - 1, refresh=refresh))
return result
[docs]
def find(
self,
path: str | os.PathLike[str],
maxdepth: int | None = None,
withdirs: bool | None = None,
detail: bool = False,
**kwargs,
) -> dict[str, S3Object] | list[str]:
"""Find all files below a given S3 path.
Recursively searches for files under the specified path, with optional
depth limiting and directory inclusion. Uses efficient S3 list operations
with delimiter handling for performance. As in fsspec, the result
includes the path itself if withdirs is True and objects exist below
it, unless it is a bucket, or if it is an object and nothing is listed
below it. A key path with a version ID names that version of the
object: the result is the version, named as ``ls(versions=True)``
names it, or empty if HeadObject reports it missing.
Args:
path: S3 path to search under (e.g., "s3://bucket/prefix").
maxdepth: Maximum number of levels to descend, at least 1
(None for unlimited). With 1, only the entries directly under
the path are listed.
withdirs: Whether to include directories in results (None = default behavior).
detail: If True, return dict of {path: S3Object}; if False, return list of paths.
**kwargs: Additional arguments including:
prefix: Key prefix, relative to the path, to filter the listed keys
by. Each slash in the prefix counts as one level of maxdepth.
With withdirs, the directories above the prefix, such as
``sub`` for ``sub/deep/``, are not included.
refresh: If True, bypass the cache and list from S3.
Returns:
Dictionary mapping paths to S3Objects (if detail=True) or
list of paths (if detail=False).
Raises:
ValueError: If ``maxdepth`` is less than 1 or the path is the root.
Example:
>>> fs = S3FileSystem()
>>> fs.find("s3://bucket/data/", maxdepth=2) # Limit depth
>>> fs.find("s3://bucket/", withdirs=True) # Include directories
"""
files = self._find(path=path, maxdepth=maxdepth, withdirs=withdirs, **kwargs)
if detail:
return {f.name: f for f in files}
return [f.name for f in files]
[docs]
def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs):
"""Walk the directories below a path, as fsspec's ``walk()`` does.
Unlike fsspec, a listed directory is listed by its key prefix rather
than by its name (see :meth:`_directory_prefix`), so a directory whose
name ends in a version ID query or in an empty segment is walked as
it was listed. Its name in the listing of its parent is the part of
its name after the parent's name, such as ``""`` for the directory
``bucket/dir/`` of the keys ``dir//x`` in ``bucket/dir``, and it is
walked under its name as listed.
Args:
path: The path to walk.
maxdepth: Maximum number of levels to walk, at least 1 (None for
unlimited).
topdown: Whether to yield a directory before the directories
below it. When True, removing names from the yielded
directory names skips their directories.
on_error: For the listing of the path, ``"omit"`` to yield
nothing if it fails, ``"raise"`` to raise its error, or a
callable that receives the error. As in fsspec, a directory
below the path whose listing fails is skipped.
**kwargs: Additional arguments passed to ``ls()``, such as
``refresh`` and ``versions``; ``detail=True`` yields the
entries by name instead of the names.
Yields:
Tuples of a directory path, its directories, and its files.
Raises:
ValueError: If ``maxdepth`` is less than 1.
"""
yield from self._walk_tree(
self._strip_protocol(path),
maxdepth=maxdepth,
topdown=topdown,
on_error=on_error,
**kwargs,
)
def _walk_tree(
self,
path: str,
maxdepth: int | None = None,
topdown: bool = True,
on_error: Any = "omit",
directory: S3Object | None = None,
yield_failed: bool = False,
**kwargs,
) -> Generator[tuple[str, Any, Any], None, None]:
"""Walk a path, or a listed directory by its key prefix, as :meth:`walk` describes.
Args:
path: The path to walk, without the protocol, or the name of
``directory``.
maxdepth: Maximum number of levels to walk, at least 1.
topdown: Whether to yield a directory before those below it.
on_error: What to do with an error of the listing of the path,
as in :meth:`walk`.
directory: The listed directory to walk, or None to list
``path`` with ``ls()``.
yield_failed: Whether to yield a directory whose listing failed
with no entries, as fsspec's ``_walk()`` does.
**kwargs: Additional arguments passed to ``ls()``, as in
:meth:`walk`.
Yields:
Tuples of a directory path, its directories, and its files.
Raises:
ValueError: If ``maxdepth`` is less than 1.
"""
if maxdepth is not None and maxdepth < 1:
raise ValueError("maxdepth must be at least 1")
detail = kwargs.pop("detail", False)
try:
listing = self._walk_listing(path, directory, **kwargs)
except (FileNotFoundError, OSError) as e:
if on_error == "raise":
raise
if callable(on_error):
on_error(e)
if yield_failed:
yield path, {} if detail else [], {} if detail else []
return
dirs, files = self._walk_entries(path, listing)
names: Any = dirs if detail else list(dirs)
listed: Any = files if detail else list(files)
if topdown:
yield path, names, listed
if maxdepth is not None:
maxdepth -= 1
if maxdepth < 1:
if not topdown:
yield path, names, listed
return
for d in names:
# As in fsspec, on_error applies to the path itself, and the
# failed listings of the directories below it are skipped.
yield from self._walk_tree(
dirs[d].name,
maxdepth=maxdepth,
topdown=topdown,
directory=dirs[d],
yield_failed=yield_failed,
detail=detail,
**kwargs,
)
if not topdown:
yield path, names, listed
def _walk_listing(self, path: str, directory: S3Object | None, **kwargs) -> list[S3Object]:
"""List a directory of a walk.
Args:
path: The path to list, without the protocol, when ``directory``
is None.
directory: The listed directory to list by its key prefix, or
None.
**kwargs: Additional arguments passed to ``ls()``, of which a
listed directory uses ``refresh`` and ``versions``.
Returns:
The entries of the directory.
Raises:
FileNotFoundError: If nothing exists at the path, or nothing is
listed under the directory.
"""
if directory is None:
return cast(list[S3Object], self.ls(path, detail=True, **kwargs))
return self._ls_directory(
directory, refresh=kwargs.get("refresh", False), versions=kwargs.get("versions", False)
)
@staticmethod
def _walk_entries(
path: str, listing: list[S3Object]
) -> tuple[dict[str, S3Object], dict[str, S3Object]]:
"""Name the directories and files of a walked directory.
An entry named below the path is named by the rest of its name, which
an empty segment, or the folder marker of the path (its key followed
by ``/``), leaves empty. Another entry, such as the object that
``ls()`` returns for the path itself, is named as fsspec's ``walk()``
names it.
Args:
path: The path of the walked directory.
listing: The entries of the directory.
Returns:
The directories and the files, by name.
"""
dirs: dict[str, S3Object] = {}
files: dict[str, S3Object] = {}
for info in listing:
name = info.name
is_directory = info.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY
if name.startswith(path + "/"):
(dirs if is_directory else files)[name[len(path) + 1 :]] = info
continue
pathname = name.rstrip("/")
if is_directory and pathname != path:
dirs[pathname.rsplit("/", 1)[-1]] = info
elif pathname == path:
files[""] = info
else:
files[pathname.rsplit("/", 1)[-1]] = info
return dirs, files
[docs]
def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs):
"""Return the sizes of the objects below a path, as fsspec's ``du()`` does.
Unlike fsspec, each entry that ``find()`` returns is looked up by its
bucket and key (see :meth:`_entry_path`), never by parsing its name.
Args:
path: The path to search under.
total: Whether to return the sum of the sizes instead of the
sizes by name.
maxdepth: Maximum number of levels to descend, at least 1 (None
for unlimited).
withdirs: Whether to include the directories, and the path itself
if it is a directory.
**kwargs: Additional arguments passed to ``find()``.
Returns:
The total size, or the sizes by the names that ``info()`` gives.
"""
sizes: dict[str, int] = {}
if withdirs and self.isdir(path):
info = self.info(path)
sizes[info["name"]] = info["size"]
for _, target in self._find_paths(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs):
info = self.info(target)
sizes[info["name"]] = info["size"]
return sum(sizes.values()) if total else sizes
[docs]
def glob(self, path, maxdepth=None, **kwargs):
"""Find the paths that match a glob pattern.
As fsspec's ``glob()``, except that only the keys that start with the
part of the last path component before the first wildcard are listed,
as with fsspec's asynchronous ``_glob()``.
Args:
path: S3 glob pattern, as a string or an ``os.PathLike``.
maxdepth: Maximum depth of the paths below the literal part of the
pattern, at least 1.
**kwargs: Additional arguments passed to ``find``, such as
``detail``.
Returns:
The matching paths, or a dictionary of them and their objects with
``detail=True``.
"""
stripped = self._strip_protocol(path)
wildcards = [i for i in (stripped.find(c) for c in "*?[") if i >= 0]
stem = stripped[: min(wildcards)].rpartition("/")[2] if wildcards else ""
if stem and "prefix" not in kwargs:
kwargs["prefix"] = stem
return super().glob(stringify_path(path), maxdepth=maxdepth, **kwargs)
[docs]
def tree(
self,
path: str | os.PathLike[str] = "/",
recursion_limit: int = 2,
max_display: int = 25,
display_size: bool = False,
prefix: str = "",
is_last: bool = True,
first: bool = True,
indent_size: int = 4,
) -> str:
"""Return a tree of the paths below a path, as fsspec's ``tree()`` does.
Unlike fsspec, a listed directory is listed by its key prefix rather
than by its name, as in :meth:`walk`.
Args:
path: The root path, as a string or an ``os.PathLike``.
recursion_limit: Maximum depth of the tree.
max_display: Maximum number of entries shown per directory.
display_size: Whether to show the sizes of the files.
prefix: The prefix of each line.
is_last: Whether the path is the last entry of its directory.
first: Whether the path is the root of the tree.
indent_size: Number of spaces per level.
Returns:
The tree as a string.
"""
return _S3TreeListing(self).tree(
path=stringify_path(path),
recursion_limit=recursion_limit,
max_display=max_display,
display_size=display_size,
prefix=prefix,
is_last=is_last,
first=first,
indent_size=indent_size,
)
[docs]
def get_mapper(
self,
root: str | os.PathLike[str] = "",
check: bool = False,
create: bool = False,
missing_exceptions=None,
):
"""Return a key/value store at a path, as fsspec's ``get_mapper()`` does.
Args:
root: The root path of the store, as a string or an ``os.PathLike``.
check: Whether to check that the root is writable.
create: Whether to create the root if it does not exist.
missing_exceptions: The exceptions that mean a missing key.
Returns:
The ``fsspec.mapping.FSMap`` of the root.
"""
return super().get_mapper(
stringify_path(root),
check=check,
create=create,
missing_exceptions=missing_exceptions,
)
[docs]
def unstrip_protocol(self, name: str | os.PathLike[str]) -> str:
"""Add the protocol to a path, as fsspec's ``unstrip_protocol()`` does.
Args:
name: The path, as a string or an ``os.PathLike``.
Returns:
The path with the ``s3://`` protocol.
"""
return cast(str, super().unstrip_protocol(stringify_path(name)))
@staticmethod
def _stringify_paths(path: Any) -> Any:
"""Convert an ``os.PathLike`` path, or those of a list, to strings.
The paths are converted with fsspec's ``stringify_path``, as
``_strip_protocol()`` converts them, but their slashes are kept.
Args:
path: A path or a list of paths.
Returns:
The converted path, or a new list of the converted paths.
Strings are returned unchanged.
"""
if isinstance(path, list):
return [stringify_path(p) for p in path]
return stringify_path(path)
[docs]
def expand_path(self, path, recursive=False, maxdepth=None, **kwargs) -> list[str]:
"""Expand glob patterns and directories into the paths they match.
As in fsspec, except that a path with a version ID names that version
of an object: it is not a glob pattern, although its ``?`` is one in
fsspec, nor is anything expanded below it. With ``recursive``, it is
included only if the version exists, as fsspec includes a path that
exists.
Args:
path: S3 path, glob pattern, or list of them.
recursive: Whether to include the paths below the directories.
maxdepth: Maximum depth of the expansion, at least 1.
**kwargs: Additional arguments passed to ``glob`` and ``find``.
Returns:
The sorted matching paths.
Raises:
ValueError: If ``maxdepth`` is less than 1.
FileNotFoundError: If nothing matches.
"""
# The expansion is sorted by name; a name of two paths is returned once.
return list(
dict.fromkeys(
name for name, _ in self._expand_paths(path, recursive, maxdepth, **kwargs)
)
)
def _expand_paths(
self, path, recursive: bool = False, maxdepth: int | None = None, **kwargs
) -> list[tuple[str, str | S3Path]]:
"""Expand glob patterns and directories as :meth:`expand_path`, with the paths they name.
A path that the caller passes names itself, as the string that
:meth:`expand_path` returns. A path that a listing returns names the
bucket and key of its entry (see :meth:`_entry_path`), also when its
name ends in what looks like a version ID query, and the expansion
below a matched directory lists it by its key prefix. As in fsspec,
the directories are expanded below the matches of a recursive glob
pattern.
Args:
path: S3 path, glob pattern, or list of them.
recursive: Whether to include the paths below the directories.
maxdepth: Maximum depth of the expansion, at least 1.
**kwargs: Additional arguments passed to ``glob`` and ``find``,
and ``assume_literal``, as in fsspec.
Returns:
The names, as :meth:`expand_path` returns them, and the paths
that they name, sorted by name: the name itself for a path that
the caller passed, which the caller parses as a path string, and
an ``S3Path`` for a listed entry. A path that the caller passed
and a listed entry that name the same object are returned once,
as the first of them.
Raises:
ValueError: If ``maxdepth`` is less than 1.
FileNotFoundError: If nothing matches.
"""
if maxdepth is not None and maxdepth < 1:
raise ValueError("maxdepth must be at least 1")
versions, others = self._split_version_paths(path)
# Insertion-ordered set of the names and the paths that they name.
out: dict[tuple[str, str | S3Path], None] = {
(p, p): None for p in versions if not recursive or self.exists(p)
}
if others:
try:
out.update(self._expand_unversioned(others, recursive, maxdepth, **kwargs))
except FileNotFoundError:
if not out:
raise
if not out:
raise FileNotFoundError(path)
# The first name and path of each object, by what the path names.
expanded: dict[S3Path | str, tuple[str, str | S3Path]] = {}
for name, target in out:
try:
identity: S3Path | str = S3Path.of(target)
except ValueError:
# Such as the root path, which names no bucket.
identity = target
expanded.setdefault(identity, (name, target))
return sorted(expanded.values(), key=lambda item: item[0])
def _expand_unversioned(
self, paths: list[str], recursive: bool, maxdepth: int | None, **kwargs
) -> dict[tuple[str, str | S3Path], None]:
"""Expand paths without a version ID as fsspec's ``expand_path()`` does.
As :meth:`_expand_paths` describes: a path that the caller passes
names itself, and a listed entry names its bucket and key. As in
fsspec, a recursive glob pattern that matches nothing raises
``FileNotFoundError`` when the matches are expanded below.
Args:
paths: The paths and glob patterns, without the protocol.
recursive: Whether to include the paths below the directories.
maxdepth: Maximum depth of the expansion.
**kwargs: Additional arguments passed to ``glob`` and ``find``,
and ``assume_literal``, as in fsspec.
Returns:
The insertion-ordered names and the paths that they name.
Raises:
FileNotFoundError: If nothing matches, or a recursive glob pattern
matches nothing.
"""
assume_literal = kwargs.pop("assume_literal", False)
# The expansion asks glob() for the entries.
kwargs.pop("detail", None)
out: dict[tuple[str, str | S3Path], None] = {}
names: set[str] = set()
def add(name: str, target: str | S3Path) -> None:
"""Add a name and the path that it names.
Args:
name: The name.
target: The path.
"""
out[(name, target)] = None
names.add(name)
for p in paths:
if not assume_literal and has_magic(p):
matches = cast(
dict[str, S3Object], self.glob(p, maxdepth=maxdepth, detail=True, **kwargs)
)
for name, match in matches.items():
add(name, self._entry_path(match))
# As fsspec expands the matches, one level fewer deep.
if recursive and not (maxdepth is not None and maxdepth <= 1):
if not matches:
# As fsspec's expansion of no paths raises.
raise FileNotFoundError([])
depth = maxdepth - 1 if maxdepth is not None else None
for match in matches.values():
for f, target in self._find_under(
match,
depth,
withdirs=True,
prefix=kwargs.get("prefix", ""),
refresh=kwargs.get("refresh", False),
):
add(f.name, target)
continue
if recursive:
for f, target in self._find_paths(p, maxdepth=maxdepth, withdirs=True, **kwargs):
add(f.name, target)
if p not in names and (not recursive or self.exists(p)):
add(p, p)
if not out:
raise FileNotFoundError(paths)
return out
@staticmethod
def _entry_path(entry: S3Object) -> S3Path:
"""Return the path that an entry of a listing or a lookup names.
The bucket and the key of the entry are taken as they are, never
parsed from its name. A listing of the current objects, and a
lookup of a path without a version, name no version, also when a
``version_aware`` lookup records the version of the object.
Args:
entry: An entry that ``find()``, ``glob()`` or ``info()`` of a path
without a version returns.
Returns:
The path of the bucket, the directory or the object.
"""
return S3Path(entry.bucket, entry.key)
def _split_version_paths(self, path) -> tuple[list[str], list[str]]:
"""Split paths into those with a version ID and the others.
Args:
path: S3 path, glob pattern, or list of them.
Returns:
Tuple of the paths with a version ID and the other paths, without
the protocol.
"""
paths = [
self._strip_protocol(p)
for p in ([path] if isinstance(path, (str, os.PathLike)) else path)
]
versions: list[str] = []
others: list[str] = []
for p in paths:
(versions if S3Path.has_version_id(p) else others).append(p)
return versions, others
[docs]
def exists(self, path: str | os.PathLike[str], **kwargs) -> bool:
"""Check if an S3 path exists.
Determines whether a bucket, object, or prefix exists in S3.
Uses caching and efficient head operations to minimize API calls.
Args:
path: S3 path to check (e.g., "s3://bucket" or "s3://bucket/key").
**kwargs: Additional arguments including:
refresh: If True, bypass the cache and query S3.
ExpectedBucketOwner, RequestPayer, SSECustomerAlgorithm,
SSECustomerKey, SSECustomerKeyMD5: The request parameters
on which the authorization of the requests depends, as
described in :meth:`info`. With them, cached listings are
not used. Other request parameters are ignored.
Returns:
True if the path exists, False otherwise. A bucket that HeadBucket
denies access to (403) exists.
Raises:
PermissionError: If the lookup of an object is denied (403).
Without ``s3:ListBucket`` on the bucket, S3 also answers 403
for a key that does not exist.
Example:
>>> fs = S3FileSystem()
>>> fs.exists("s3://my-bucket/file.txt")
>>> fs.exists("s3://my-bucket/")
"""
refresh = kwargs.pop("refresh", False)
path = self._strip_protocol(path)
if path in ["", "/"]:
# The root always exists.
return True
s3_path = S3Path.parse(path)
lookup_kwargs = self._get_lookup_kwargs(kwargs)
# The cached listings are made without lookup parameters.
use_listings = not refresh and not lookup_kwargs
if s3_path.key:
try:
if use_listings and self._ls_from_cache(s3_path):
return True
info = self.info(path, refresh=refresh, **lookup_kwargs)
return bool(info)
except FileNotFoundError:
return False
if use_listings and self._ls_from_cache(S3Path(s3_path.bucket)):
return True
try:
file = self._head_bucket(s3_path.bucket, refresh=refresh, lookup_kwargs=lookup_kwargs)
except PermissionError:
# HeadBucket answers 403 for a bucket that exists but that the
# caller may not access.
return True
return bool(file)
[docs]
def rm_file(self, path: str | os.PathLike[str], **kwargs) -> None:
"""Delete an S3 object with DeleteObject.
Does nothing for a bucket path. If the path has a version ID, that
version is deleted.
Args:
path: S3 path (s3://bucket/key) of the object to delete.
**kwargs: Additional parameters passed to the DeleteObject API,
such as ``RequestPayer``. Those that it does not accept are
ignored, for fsspec compatibility.
Raises:
TypeError: If kwargs has ``bucket``, ``key`` or ``version_id``,
or ``Bucket``, ``Key`` or ``VersionId``, which the path gives.
"""
path = stringify_path(path)
s3_path = S3Path.parse(path)
if not s3_path.key:
return
for name in ("bucket", "key", "version_id", "Bucket", "Key", "VersionId"):
if name in kwargs:
raise TypeError(f"rm_file() got multiple values for keyword argument '{name}'")
_logger.debug(f"Delete object: {s3_path.uri}")
self.core.delete_object(s3_path, **self.core.operation_params("delete_object", kwargs))
self.invalidate_cache(path)
[docs]
def rm(self, path, recursive=False, maxdepth=None, batch_size=None, **kwargs) -> None:
"""Delete objects with DeleteObjects requests.
Expands the paths with ``expand_path`` and deletes the matched objects
in parallel requests of up to ``S3DeleteBatch.MAX_KEYS`` keys each,
one set of requests per bucket. A path with a version ID deletes that
version without expansion.
Args:
path: S3 path (s3://bucket/key) or list of paths to delete.
recursive: Whether to delete all objects below the paths.
maxdepth: Maximum depth to expand when ``recursive`` is True.
batch_size: Number of DeleteObjects requests to send at the same
time. By default the ``batch_size`` of the filesystem; None or
-1 there is ``max_workers``. ``max_workers`` is a deprecated
alias.
**kwargs: Additional parameters passed to the DeleteObjects API.
``Quiet`` (default True) sets the quiet mode of the requests.
Raises:
ValueError: If a path is a bucket, or ``batch_size`` is neither
positive, -1, nor None.
TypeError: If both ``batch_size`` and ``max_workers`` are given.
OSError: If S3 could not delete some of the objects.
"""
path = self._stringify_paths(path)
max_workers = self._delete_batch_limit(batch_size, kwargs)
paths = self._delete_paths(path, recursive=recursive, maxdepth=maxdepth)
self._delete_objects(paths, max_workers=max_workers, **kwargs)
def _delete_batch_limit(self, batch_size: int | None, kwargs: dict[str, Any]) -> int:
"""Return the number of DeleteObjects requests of ``rm()`` to send at once.
Args:
batch_size: The ``batch_size`` argument of ``rm()``.
kwargs: The other arguments of ``rm()``, from which its deprecated
alias ``max_workers`` is removed.
Returns:
The number of requests.
Raises:
ValueError: If the number is neither positive, -1, nor None.
TypeError: If both ``batch_size`` and ``max_workers`` are given.
"""
if "max_workers" in kwargs:
if batch_size is not None:
raise TypeError("rm() got both batch_size and its alias max_workers.")
warnings.warn(
"The max_workers argument of rm() is deprecated; use batch_size.",
DeprecationWarning,
stacklevel=3,
)
batch_size = kwargs.pop("max_workers")
return self._batch_limit(batch_size)
def _delete_paths(
self, path: str | list[str], recursive: bool = False, maxdepth: int | None = None
) -> list[str | S3Path]:
"""Expand the paths that ``rm()`` deletes (see :meth:`S3PathPairing.delete_paths`).
Args:
path: S3 path or list of paths.
recursive: Whether to include all objects below the paths.
maxdepth: Maximum depth to expand when ``recursive`` is True.
Returns:
The paths with a version ID as given, followed by the paths that
the expansion of the other paths names (see
:meth:`_expand_paths`): the path strings that the caller passed,
and the listed paths.
Raises:
ValueError: If a path is a bucket.
"""
versioned_paths, unversioned_paths = S3PathPairing.delete_paths(path)
paths: list[str | S3Path] = list(versioned_paths)
if not unversioned_paths:
# The expansion raises FileNotFoundError for no paths.
return paths
return paths + [
target
for _, target in self._expand_paths(
unversioned_paths, recursive=recursive, maxdepth=maxdepth
)
]
def _create_executor(self, max_workers: int) -> _S3PoolScope:
"""Create the executor of the parallel requests of a file or an operation.
The files of ``S3FileSystem`` and ``AioS3FileSystem`` and the
parallel operations of both take their executors from this method.
The requests run in the request pool of the filesystem, which they
share, at most ``max_workers`` of those of the executor at once;
shutting the executor down waits for its own requests only, and
leaves the pool running.
Args:
max_workers: The maximum number of requests of the executor in
the request pool at once.
Returns:
An executor on the request pool of the filesystem.
Raises:
ValueError: If ``max_workers`` is not positive.
"""
return _S3PoolScope(self._request_pool, max_workers)
def _map(
self,
fn: Callable[[ItemT], T],
items: Iterable[ItemT],
max_workers: int | None = None,
**kwargs: Any,
) -> S3MapResult[T]:
"""Run the parallel requests of an operation in a scope of their own.
See :meth:`S3Executor.map() <pyathena.filesystem.s3_executor.S3Executor.map>`.
The scope is shut down without another wait, since the map has waited
for the calls that it started, unless a second interrupt stopped it.
Args:
fn: The request to run for each item.
items: The items.
max_workers: Maximum number of parallel requests. Defaults to
``self.max_workers``.
**kwargs: The options of ``S3Executor.map()``.
Returns:
The outcomes of the requests.
"""
executor = self._create_executor(max_workers=max_workers or self.max_workers)
try:
return executor.map(fn, items, **kwargs)
finally:
executor.shutdown(wait=False)
@staticmethod
def _check_batch_size(batch_size: int | None) -> int | None:
"""Check a ``batch_size`` argument.
Args:
batch_size: The argument.
Returns:
The argument.
Raises:
ValueError: If it is neither positive, -1, nor None.
"""
if batch_size is not None and batch_size != -1 and batch_size <= 0:
raise ValueError("batch_size must be a positive integer, -1, or None.")
return batch_size
def _batch_limit(self, batch_size: int | None) -> int:
"""Return the number of calls of a batch operation that may run at once.
Args:
batch_size: The ``batch_size`` of the call. None takes that of the
filesystem; None or -1 there is ``max_workers``, the size of
the request pool.
Returns:
The number of calls.
Raises:
ValueError: If ``batch_size`` is neither positive, -1, nor None.
"""
batch_size = self._check_batch_size(batch_size)
if batch_size is None:
batch_size = self.batch_size
if batch_size is None or batch_size == -1:
return self.max_workers
return batch_size
def _batch_kwargs(self, concurrency: int, kwargs: dict[str, Any]) -> dict[str, Any]:
"""Return the arguments of the per-object calls of a batch operation.
With several calls at once, each object gets ``max_workers //
concurrency`` parallel requests, at least one, so that a batch keeps
about ``max_workers`` requests in the request pool. A ``max_workers``
of the caller is capped at that; None or 0 takes it.
Args:
concurrency: The number of calls that run at once.
kwargs: The arguments that the caller gave the operation.
Returns:
The arguments of each call.
"""
if concurrency <= 1:
return kwargs
window = max(1, self.max_workers // concurrency)
given = kwargs.get("max_workers")
return {**kwargs, "max_workers": min(given, window) if given else window}
def _run_batch(
self,
fn: Callable[[ItemT], T],
items: list[ItemT],
concurrency: int,
counts: Callable[[BaseException], bool] | None = None,
) -> S3MapResult[T]:
"""Run the per-object calls of a batch operation.
The calls are composite: each one waits for requests of its object in
the request pool. They run in threads of a pool of their own, at most
``concurrency`` at once, and never in the request pool. A single
item runs in the calling thread. See :meth:`S3Executor.map()
<pyathena.filesystem.s3_executor.S3Executor.map>` for the failures,
interrupts, and ``counts``.
Args:
fn: The call to run for each item.
items: The items, in the order of the outcomes.
concurrency: The number of calls that run at once.
counts: Whether a failure stops the batch, by default every one.
Returns:
The outcomes of the calls.
"""
if len(items) == 1:
try:
return S3MapResult([fn(items[0])], None)
except Exception as e:
if counts is None or counts(e):
raise
return S3MapResult([e], None)
executor = S3ThreadPool(max_workers=concurrency)
try:
return executor.map(fn, items, counts=counts)
finally:
# The map has waited for the calls that it started, unless a
# second interrupt stopped it.
executor.shutdown(wait=False)
def _delete_objects(
self, paths: list[str | S3Path], max_workers: int | None = None, **kwargs
) -> None:
"""Delete objects with DeleteObjects requests grouped by bucket.
Args:
paths: Paths of the objects to delete. Bucket paths are skipped.
max_workers: Maximum number of parallel requests. Defaults to
``self.max_workers``.
**kwargs: Additional parameters passed to the DeleteObjects API.
``Quiet`` (default True) sets the quiet mode of the requests.
Raises:
TypeError: If kwargs has ``Bucket`` or ``Delete``.
OSError: If S3 could not delete some of the objects.
"""
batches, params = self._delete_batches(paths, **kwargs)
if not batches:
return
# Every request runs also after another failed, and an interrupt
# cancels those that have not started.
results = self._map(
functools.partial(self._delete_batch, **params),
batches,
max_workers=max_workers,
stop_on_failure=False,
)
self._raise_delete_errors(results.outcomes)
@staticmethod
def _delete_batches(
paths: list[str | S3Path], **kwargs
) -> tuple[list[S3DeleteBatch], dict[str, Any]]:
"""Group the objects that ``rm`` deletes into DeleteObjects batches.
Args:
paths: Paths of the objects to delete. Bucket paths are skipped.
**kwargs: Additional parameters of the DeleteObjects API.
``Quiet`` (default True) sets the quiet mode of the batches.
Returns:
The batches, and the parameters of their requests without
``Quiet``.
Raises:
TypeError: If kwargs has ``Bucket`` or ``Delete``.
"""
for name in ("Bucket", "Delete"):
if name in kwargs:
raise TypeError(f"rm() got an unexpected keyword argument '{name}'")
quiet = kwargs.pop("Quiet", True)
s3_paths = [S3Path.parse(p) if isinstance(p, str) else p for p in paths]
batches = S3DeleteBatch.from_paths([p for p in s3_paths if p.key], quiet=quiet)
return batches, kwargs
def _delete_batch(self, batch: S3DeleteBatch, **kwargs) -> S3DeleteResult:
"""Delete a batch and invalidate the cache of its objects.
The cache is invalidated when the request has finished, also when it
fails, because S3 may have deleted some of the objects.
Args:
batch: The objects to delete.
**kwargs: Additional parameters passed to the DeleteObjects API.
Returns:
The result of the request.
"""
try:
return self.core.delete_objects(batch, **kwargs)
finally:
for path in batch.objects:
self._invalidate(path)
@staticmethod
def _raise_delete_errors(results: list[S3DeleteResult | BaseException]) -> None:
"""Raise an error for the batches that S3 could not delete.
Args:
results: The result or the exception of each batch.
Raises:
BaseException: The first exception of the batches, with a note
that lists the errors of the results, if any.
OSError: If no batch raised and a result has errors.
"""
exceptions: list[BaseException] = []
errors: list[str] = []
for result in results:
if isinstance(result, BaseException):
exceptions.append(result)
else:
errors.extend(str(e) for e in result.errors)
message = f"Failed to delete objects: {', '.join(sorted(errors))}" if errors else None
if exceptions:
if message:
exceptions[0].add_note(message)
raise exceptions[0]
if message:
raise OSError(message)
[docs]
def mkdir(self, path: str | os.PathLike[str], create_parents: bool = True, **kwargs) -> None:
"""Create an S3 bucket.
S3 has no real directories below the bucket level; creating a key
prefix requires no operation. This method creates the bucket when the
path points at a bucket (or when ``create_parents`` is True and the
bucket does not exist yet), and does nothing for key prefixes under
an existing bucket.
Bucket lifecycle operations are disabled by default because they are
infrastructure-level changes; pass ``allow_bucket_creation=True`` to
the filesystem constructor to enable bucket creation.
Args:
path: S3 path (e.g., "s3://bucket" or "s3://bucket/prefix").
create_parents: If True, create the bucket when it does not exist,
even if the path contains a key prefix.
**kwargs: Additional arguments including:
acl: Canned ACL to apply to the bucket.
region_name: Region to create the bucket in. Defaults to the
client's region.
Other parameters that the CreateBucket API accepts, such as
``ObjectLockEnabledForBucket``, are passed to it; the others
are ignored, for fsspec compatibility.
Raises:
FileExistsError: If the path is a bucket that already exists.
FileNotFoundError: If the bucket does not exist and
``create_parents`` is False.
PermissionError: If the bucket would be created but bucket
creation is not enabled on this filesystem instance.
ValueError: If the ACL is invalid, the path is empty, or
botocore rejects the parameters of the CreateBucket request.
"""
path = self._strip_protocol(path).rstrip("/")
if not path:
raise ValueError("Cannot create the root directory.")
s3_path = S3Path.parse(path)
if self.exists(s3_path.bucket):
if not s3_path.key:
# Requested to create a bucket, but the bucket already exists.
raise FileExistsError(s3_path.bucket)
# Do nothing as the bucket already exists.
elif not s3_path.key or create_parents:
if not self.allow_bucket_creation:
raise PermissionError(
"Bucket creation is disabled. "
"Set allow_bucket_creation=True on the filesystem to enable it."
)
self.core.create_bucket(
s3_path.bucket,
acl=kwargs.pop("acl", None),
region_name=kwargs.pop("region_name", None),
**self.core.operation_params("create_bucket", kwargs),
)
# invalidate_cache of the bucket keeps the cached bucket
# listing, so evict it directly.
self._evict_cache("")
self.invalidate_cache(s3_path.bucket)
else:
# exists() has already confirmed the bucket does not exist,
# and it is not requested to be created.
raise FileNotFoundError(s3_path.bucket)
[docs]
def makedirs(self, path: str | os.PathLike[str], exist_ok: bool = False) -> None:
"""Recursively create a directory, creating the bucket if necessary.
Creating the bucket requires ``allow_bucket_creation=True`` on the
filesystem constructor; see :meth:`mkdir`.
Args:
path: S3 path (e.g., "s3://bucket" or "s3://bucket/prefix").
exist_ok: If False, raise FileExistsError when the path is a
bucket that already exists.
Raises:
FileExistsError: If the path is a bucket that already exists and
``exist_ok`` is False.
PermissionError: If the bucket would be created but bucket
creation is not enabled on this filesystem instance.
"""
try:
self.mkdir(path, create_parents=True)
except FileExistsError:
if not exist_ok:
raise
[docs]
def rmdir(self, path: str | os.PathLike[str]) -> None:
"""Remove an S3 bucket, which must be empty.
S3 has no real directories below the bucket level, so only bucket
paths can be removed.
Bucket lifecycle operations are disabled by default because they are
infrastructure-level changes; pass ``allow_bucket_deletion=True`` to
the filesystem constructor to enable bucket deletion.
Args:
path: S3 bucket path (e.g., "s3://bucket").
Raises:
FileExistsError: If the path contains a key that exists. The user
may have meant ``rm(path, recursive=True)``.
FileNotFoundError: If the path contains a key that does not exist,
or the bucket does not exist.
PermissionError: If bucket deletion is not enabled on this
filesystem instance.
OSError: If the bucket is not empty.
"""
path = self._strip_protocol(path).rstrip("/")
s3_path = S3Path.parse(path)
if s3_path.key:
if self.exists(path):
# The user may have meant rm(path, recursive=True).
raise FileExistsError(path)
raise FileNotFoundError(path)
if not self.allow_bucket_deletion:
raise PermissionError(
"Bucket deletion is disabled. "
"Set allow_bucket_deletion=True on the filesystem to enable it."
)
self.core.delete_bucket(s3_path.bucket)
self.invalidate_cache(s3_path.bucket)
# invalidate_cache of the bucket keeps the cached bucket listing,
# so evict it directly.
self._evict_cache("")
[docs]
def touch(
self, path: str | os.PathLike[str], truncate: bool = True, **kwargs
) -> dict[str, Any]:
"""Create an empty object with PutObject.
Args:
path: S3 path (s3://bucket/key) of the object.
truncate: If True, replace an existing object with an empty one;
if False, raise if the object exists. A key that is only a
prefix of other keys has no object.
**kwargs: Additional parameters passed to the PutObject API,
except ``Body``: the object is empty.
Returns:
The PutObject response as a dictionary (see
:meth:`S3PutObject.to_dict`).
Raises:
TypeError: If kwargs has ``Body``.
ValueError: If the path has a version ID, is a bucket, or exists
while ``truncate`` is False.
"""
path = stringify_path(path)
s3_path = S3Path.parse(path)
if s3_path.version_id:
raise ValueError("Cannot touch the file with the version specified.")
if "Body" in kwargs:
raise TypeError("touch() creates an empty object; use pipe_file() to write data.")
if not truncate and self._key_exists(s3_path, self._get_lookup_kwargs(kwargs)):
raise ValueError("Cannot touch the existing file without specifying truncate.")
if not s3_path.key:
raise ValueError("Cannot touch the bucket.")
# The empty body is passed, so that a body keyword argument raises
# TypeError.
object_ = self.core.put_object(s3_path, None, **kwargs)
self.invalidate_cache(path)
return object_.to_dict()
[docs]
def mv(self, path1, path2, recursive=False, maxdepth=None, batch_size=None, **kwargs) -> None:
"""Move files from one S3 location to another.
Copies the files to the destinations that ``copy()`` gives them, and
then deletes only the source objects that were copied. fsspec's
``AbstractFileSystem.mv()`` instead removes ``path1`` by expanding it
again, which also deletes copies placed where ``path1`` matches them
and files that ``maxdepth`` kept from being copied. A file whose
destination is the file itself is left in place, and directories,
which S3 does not store as objects, are not copied. A ``null`` version
moved onto its key is copied and deleted only when bucket versioning
is enabled. Without versioning, or with versioning suspended, that
move is left in place because the copy would replace the version
that the deletion removes.
Comparing a ``null`` version with its unversioned key calls
GetBucketVersioning once per bucket during the move's planning and
requires ``s3:GetBucketVersioning``. The result is not cached across
moves. Directory buckets do not support versioning and need no lookup.
A source without a version whose destination conflicts with another
pair is looked up with HeadObject during the planning too.
The files are copied in parallel as ``copy()`` copies them, and are
deleted once every copy has succeeded; nothing is deleted after a
failed copy.
Args:
path1: Source S3 path, glob pattern, or list of paths.
path2: Destination S3 path, or list of paths when ``path1`` is a
list.
recursive: Whether to move the directories with their contents.
maxdepth: Maximum depth of a recursive move.
batch_size: Number of copies to run at the same time, as in
``copy()``.
**kwargs: Additional S3 copy parameters passed to ``cp_file()``.
The lookups and the deletion of the sources receive
``RequestPayer`` and the source's expected bucket owner (see
:meth:`S3Core.copy_source_params`); the lookups also receive
the source's SSE-C parameters.
Raises:
ValueError: If two sources have the same destination, or a
destination is another source, including one left in place,
which is checked before anything is copied. A source without
an object at its key, such as a directory, writes no
destination and does not conflict. Also if ``batch_size`` is
neither positive, -1, nor None.
FileNotFoundError: If a source does not exist, once the running
copies have ended; nothing is deleted.
OSError: If bucket versioning cannot be read.
"""
path1, path2 = self._stringify_paths(path1), self._stringify_paths(path2)
if path1 == path2:
return
limit = self._batch_limit(batch_size)
source_params = self.core.copy_source_params(kwargs)
pairs = self._move_pairs(
path1,
path2,
recursive=recursive,
maxdepth=maxdepth,
source_params=source_params,
)
if not pairs:
return
concurrency = min(limit, len(pairs))
call_kwargs = self._batch_kwargs(concurrency, kwargs)
def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> bool:
"""Copy a pair of the move.
Args:
pair: The source and its destination.
Returns:
Whether the source was copied, as ``_copy_file()`` returns it.
"""
return self._copy_file(*pair, **call_kwargs)
# Every copy has ended, and succeeded, before anything is deleted.
copied = self._run_batch(copy_pair, pairs, concurrency).outcomes
self._delete_objects(
[p1 for (p1, _), c in zip(pairs, copied, strict=True) if c],
**self.core.operation_params("delete_objects", source_params),
)
def _move_pairs(
self,
path1: str | list[str],
path2: str | list[str],
recursive: bool = False,
maxdepth: int | None = None,
source_params: Mapping[str, Any] | None = None,
) -> list[tuple[str | S3Path, str | S3Path]]:
"""Pair and check the paths of a move (see :meth:`S3PathPairing.move_pairs`).
Args:
path1: Source S3 path, glob pattern, or list of paths.
path2: Destination S3 path, or list of paths when ``path1`` is a
list.
recursive: Whether to include the contents of the directories.
maxdepth: Maximum depth of the expansion.
source_params: The parameters of the requests that read the
sources (see :meth:`S3Core.copy_source_params`), sent with
the lookups of the sources.
Returns:
The sources and destinations that are moved.
Raises:
ValueError: If the move has conflicting paths.
OSError: If bucket versioning cannot be read.
"""
pairing = S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth)
pairs = self._copy_pairs(pairing)
missing, versioned_buckets = self._conflict_lookups(pairing, pairs, source_params)
return pairing.move_pairs(
pairs, missing=missing, versioning_enabled_buckets=versioned_buckets
)
def _conflict_lookups(
self,
pairing: S3PathPairing,
pairs: list[tuple[str | S3Path, str | S3Path]],
source_params: Mapping[str, Any] | None = None,
) -> tuple[set[str | S3Path], set[str]]:
"""Look up what the conflict checks of a move or a copy need.
See :meth:`S3PathPairing.conflict_candidates`.
Args:
pairing: The paths of the move or the copy.
pairs: The pairs of the paths.
source_params: The parameters of the requests that read the
sources (see :meth:`S3Core.copy_source_params`), sent with
the lookups of the sources.
Returns:
The conflict candidates with no object at their key, and the
buckets whose versioning is enabled among those with a ``null``
version and its unversioned key in the pairs.
Raises:
OSError: If bucket versioning cannot be read.
"""
paths = {S3Path.of(p) for pair in pairs for p in pair}
unversioned = {p.name for p in paths if not p.version_id}
buckets = {
p.bucket
for p in paths
if p.version_id == "null" and p.name in unversioned and not p.is_directory_bucket
}
versioned_buckets = {
bucket for bucket in buckets if self.core.get_bucket_versioning(bucket) == "Enabled"
}
lookup_kwargs = self._get_lookup_kwargs(source_params or {})
missing = {
source
for source in pairing.conflict_candidates(
pairs, versioning_enabled_buckets=versioned_buckets
)
if self._head_object(S3Path.of(source), lookup_kwargs=lookup_kwargs) is None
}
return missing, versioned_buckets
def _copy_pairs(
self,
pairing: S3PathPairing,
local: bool = False,
**kwargs: Any,
) -> list[tuple[str | S3Path, str | S3Path]]:
"""Expand and pair the paths of a copy (see :meth:`S3PathPairing.s3_copy_pairs`).
The sources are expanded as :meth:`_expand_paths` names them: a path
that the caller passes as the string it was given, and a listed entry
as its bucket and key. The destination is looked up only when it
decides the pairing.
Args:
pairing: The paths of the copy.
local: Whether the destinations are the local paths of ``get()``,
which the local filesystem looks up, and which are named as
:meth:`S3PathPairing.copy_pairs` names them.
**kwargs: Passed to the expansion, as fsspec's ``get()`` passes its
keyword arguments to ``expand_path()``.
Returns:
The sources and their destinations.
"""
if not pairing.expands:
return list(pairing.copy_pairs())
sources = [
target
for name, target in self._expand_paths(
pairing.path1, recursive=pairing.recursive, maxdepth=pairing.maxdepth, **kwargs
)
if not (pairing.skips_directories and (trailing_sep(name) or self.isdir(target)))
]
isdir = LocalFileSystem().isdir if local else self.isdir
destination_is_dir = (
# A string, as looks_up_destination checks.
isdir(cast(str, pairing.path2)) if sources and pairing.looks_up_destination else None
)
if local:
return list(pairing.copy_pairs(sources, destination_is_dir))
return pairing.s3_copy_pairs(sources, destination_is_dir)
def _checked_copy_pairs(
self,
path1: str | list[str],
path2: str | list[str],
recursive: bool = False,
maxdepth: int | None = None,
source_params: Mapping[str, Any] | None = None,
) -> list[tuple[str | S3Path, str | S3Path]]:
"""Pair and check the paths of a copy (see :meth:`S3PathPairing.check_copy_pairs`).
Args:
path1: Source S3 path, glob pattern, or list of them.
path2: Destination S3 path, or list of paths when ``path1`` is a
list.
recursive: Whether to include the contents of the directories.
maxdepth: Maximum depth of the expansion.
source_params: The parameters of the requests that read the
sources (see :meth:`S3Core.copy_source_params`), sent with
the lookups of the sources.
Returns:
The sources and destinations that are copied.
Raises:
ValueError: If the copy has conflicting paths.
OSError: If bucket versioning cannot be read.
"""
pairing = S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth)
pairs = self._copy_pairs(pairing)
if pairs:
missing, versioned_buckets = self._conflict_lookups(pairing, pairs, source_params)
pairing.check_copy_pairs(
pairs, missing=missing, versioning_enabled_buckets=versioned_buckets
)
return pairs
[docs]
def copy(
self,
path1,
path2,
recursive=False,
maxdepth=None,
on_error=None,
batch_size=None,
**kwargs,
) -> None:
"""Copy files within S3.
As fsspec's ``copy()``, except that a source with a version ID copies
that version of the object to a destination named after its key, as
:meth:`~pyathena.filesystem.s3_path_pairing.S3PathPairing.copy_pairs`
pairs them, and that the files are copied in parallel, ``batch_size``
at a time. Pairs that would depend on the order of the copies raise
before anything is copied (see
:meth:`~pyathena.filesystem.s3_path_pairing.S3PathPairing.check_copy_pairs`).
An error that ``on_error`` does not ignore starts no further copy,
stops the running ones before their next request where it can (a
multipart copy sends no further part and is aborted), and is raised
once they have ended. Copies that ended before it stay in place.
Args:
path1: Source S3 path, glob pattern, or list of them.
path2: Destination S3 path, or list of paths when ``path1`` is a
list.
recursive: Whether to copy the directories with their contents.
maxdepth: Maximum depth of a recursive copy.
on_error: ``"raise"`` or ``"ignore"`` for a missing source; by
default ``"ignore"`` with ``recursive`` and ``"raise"``
otherwise.
batch_size: Number of copies to run at the same time. By default
the ``batch_size`` of the filesystem; None or -1 there is
``max_workers``. With ``n`` copies at once, the smaller of
``batch_size`` and the number of pairs, each multipart copy
sends at most ``max(1, max_workers // n)`` parts at once.
**kwargs: Additional S3 copy parameters passed to ``cp_file()``.
Raises:
ValueError: If two sources have the same destination, or a
destination is another source, except for a source without an
object at its key, such as a directory, which is not copied.
Also if ``batch_size`` is neither positive, -1, nor None.
OSError: If bucket versioning cannot be read for the checks.
"""
path1, path2 = self._stringify_paths(path1), self._stringify_paths(path2)
if on_error is None:
on_error = "ignore" if recursive else "raise"
limit = self._batch_limit(batch_size)
pairs = self._checked_copy_pairs(
path1,
path2,
recursive=recursive,
maxdepth=maxdepth,
source_params=self.core.copy_source_params(kwargs),
)
if not pairs:
return
concurrency = min(limit, len(pairs))
call_kwargs = self._batch_kwargs(concurrency, kwargs)
def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> None:
"""Copy a pair.
Args:
pair: The source and its destination.
"""
self.cp_file(*pair, **call_kwargs)
self._run_batch(
copy_pair,
pairs,
concurrency,
counts=None if on_error == "raise" else lambda e: not isinstance(e, FileNotFoundError),
)
[docs]
def get(
self,
rpath,
lpath,
recursive=False,
callback=_DEFAULT_CALLBACK,
maxdepth=None,
batch_size=None,
**kwargs,
) -> None:
"""Copy files from S3 to the local filesystem.
As fsspec's ``get()``, except that a source with a version ID
downloads that version of the object to a local path named after its
key, as :meth:`~pyathena.filesystem.s3_path_pairing.S3PathPairing.copy_pairs`
pairs them, and that the files are downloaded in parallel, as with
``copy()``. The destinations named after the sources are checked to
lie under ``lpath``. With ``outfile``, the files are written to it
one after another, in order.
A failed download starts no further one, the running ones read and
write no further block, and the error is raised once they have ended.
Args:
rpath: Source S3 path, glob pattern, or list of them.
lpath: Local destination path, or list of paths or file-like
objects when ``rpath`` is a list.
recursive: Whether to copy the directories with their contents.
callback: Progress callback. It and its branches for the files
are used from the threads that download them, one at a time.
maxdepth: Maximum depth of a recursive copy.
batch_size: Number of downloads to run at the same time, as in
``copy()``.
**kwargs: Additional parameters passed to ``get_file()``.
Raises:
ValueError: If a destination named after a source lies outside
``lpath``, two downloads write the same local file, or
``batch_size`` is neither positive, -1, nor None.
"""
rpath = self._stringify_paths(rpath)
limit = self._batch_limit(batch_size)
root = make_path_posix(lpath) if isinstance(lpath, (str, os.PathLike)) else lpath
pairs = self._checked_get_pairs(
S3PathPairing(rpath, root, recursive=recursive, maxdepth=maxdepth), kwargs, **kwargs
)
if not pairs:
return
# get_file() writes to an outfile instead of the local paths.
outfile = kwargs.get("outfile") is not None
concurrency = 1 if outfile else min(limit, len(pairs))
call_kwargs = self._batch_kwargs(concurrency, kwargs)
locked = _S3LockedCallback(callback, threading.Lock())
def get_pair(pair: tuple[str | S3Path, Any]) -> None:
"""Download a pair with a branch of the callback.
Args:
pair: The S3 source and its local destination. The branch
is named after the source as a string.
"""
rp, lp = pair
with locked.branched(str(rp), lp) as child:
self.get_file(rp, lp, callback=child, **call_kwargs)
locked.relative_update(1)
locked.set_size(len(pairs))
if outfile:
# One after another, in order, in the calling thread, which may
# own the stream.
for pair in pairs:
get_pair(pair)
return
self._run_batch(get_pair, pairs, concurrency)
def _checked_get_pairs(
self, pairing: S3PathPairing, file_kwargs: Mapping[str, Any], **kwargs: Any
) -> list[tuple[str | S3Path, Any]]:
"""Expand, pair and check the paths of a ``get()``.
The destinations named after the sources are checked to lie under the
local destination. Without ``outfile``, the files are downloaded in
parallel, so no two of them may write the same local file (see
:meth:`_check_local_destinations`).
Args:
pairing: The paths of the copy, with the local destination.
file_kwargs: The arguments of ``get_file()``, whose ``outfile``
and ``version_id`` the checks follow.
**kwargs: Passed to ``expand_path()``.
Returns:
The S3 sources and their local destinations.
Raises:
ValueError: If a destination named after a source lies outside
the local destination, or two downloads write the same local
file.
"""
pairs = self._copy_pairs(pairing, local=True, **kwargs)
if isinstance(pairing.path2, str):
check_contained(pairing.path2, [lp for _, lp in pairs])
if pairs and file_kwargs.get("outfile") is None:
self._check_local_destinations(pairs, file_kwargs.get("version_id"))
return pairs
def _check_local_destinations(
self, pairs: list[tuple[str | S3Path, Any]], version_id: str | None = None
) -> None:
"""Check that no two downloads of a batch write the same local file.
Destinations are compared by the object for file-like ones, by the
file for existing paths, such as hard links to one file, and by the
resolved path otherwise. A source that ``get_file()`` creates as a
local directory writes no file, so several of them may share a local
path; the sources are looked up only when their local path is shared.
Args:
pairs: The S3 sources and their local paths or file-like objects.
version_id: The version ID that ``get_file()`` receives.
Raises:
ValueError: If two downloads write the same file, or one creates
a directory where another writes a file.
"""
sources: dict[tuple[Any, ...], list[str | S3Path]] = {}
for rpath, destination in pairs:
key: tuple[Any, ...]
if isfilelike(destination):
key = ("object", id(destination))
else:
path = os.fspath(destination)
try:
stat = os.stat(path)
except OSError:
key = ("path", os.path.normcase(os.path.realpath(path)))
else:
if os.path.isdir(path):
# Directories are created, not written.
continue
key = ("file", stat.st_dev, stat.st_ino)
sources.setdefault(key, []).append(rpath)
for key, rpaths in sources.items():
# get_file() writes into a file-like object whatever the source.
if len(rpaths) > 1 and (
key[0] == "object"
or not all(self._is_directory_source(rp, version_id) for rp in rpaths)
):
raise ValueError("Cannot get several paths to the same destination.")
def _is_directory_source(
self, rpath: str | os.PathLike[str] | S3Path, version_id: str | None = None
) -> bool:
"""Return whether ``get_file()`` creates a local directory for a source.
A requested version always names an object, while ``isdir()`` would
look up the latest version, or the prefix of the same name when the
version does not exist.
Args:
rpath: The S3 source, or an ``S3Path``, as for ``get_file()``.
version_id: The version ID that ``get_file()`` receives.
Returns:
True if the source has no version and is a directory.
"""
path = rpath if isinstance(rpath, S3Path) else S3Path.parse(self._strip_protocol(rpath))
if version_id or path.version_id:
return False
return bool(self.isdir(rpath))
[docs]
def cp_file(
self,
path1: str | os.PathLike[str] | S3Path,
path2: str | os.PathLike[str] | S3Path,
recursive=False,
maxdepth=None,
on_error=None,
**kwargs,
):
"""Copy an S3 object to another S3 location.
Performs server-side copy of S3 objects, which is more efficient than
downloading and re-uploading. Automatically chooses between simple copy
and multipart copy based on object size.
Args:
path1: Source S3 path (s3://bucket/key), or an ``S3Path``, whose
bucket, key and version are used as they are, such as a key
that a listing returns.
path2: Destination S3 path (s3://bucket/key), or an ``S3Path``.
recursive: Unused parameter for fsspec compatibility.
maxdepth: Unused parameter for fsspec compatibility.
on_error: Unused parameter for fsspec compatibility.
**kwargs: Additional S3 copy parameters (e.g., metadata, storage
class). The ``block_size`` and ``max_workers`` parameters
control a multipart copy and are not sent to S3. The lookup
of the source receives ``RequestPayer``, and the source's
expected bucket owner and SSE-C parameters under the names of
the requests that read it (see
:meth:`S3Core.copy_source_params`).
Raises:
ValueError: If trying to copy to a versioned file or copy buckets,
or if a directive of a multipart copy has an invalid value.
Note:
Uses multipart copy for objects larger than the maximum part size
to optimize performance for large files. The copy operation is
performed entirely on the S3 service without data transfer.
The multipart copy applies the metadata, tagging and annotation
directives as CopyObject does (default ``COPY``).
A directory ``path1``, which recursive ``copy()`` passes along
with the files under it, is skipped.
"""
self._copy_file(
path1 if isinstance(path1, S3Path) else stringify_path(path1),
path2 if isinstance(path2, S3Path) else stringify_path(path2),
**kwargs,
)
def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool:
"""Copy an S3 object as :meth:`cp_file` does.
Args:
path1: Source S3 path (s3://bucket/key), or an ``S3Path``.
path2: Destination S3 path (s3://bucket/key), or an ``S3Path``.
**kwargs: Additional S3 copy parameters, as for :meth:`cp_file`.
Returns:
False if ``path1`` is a directory, which is skipped; True if the
object was copied.
Raises:
ValueError: If trying to copy to a versioned file or copy buckets.
"""
# Parameters of the multipart copy, not of the S3 requests.
block_size = kwargs.pop("block_size", None)
max_workers = kwargs.pop("max_workers", None)
source = S3Path.of(path1)
destination = S3Path.of(path2)
if destination.version_id:
raise ValueError("Cannot copy to a versioned file.")
if not source.key or not destination.key:
raise ValueError("Cannot copy buckets.")
# Looked up with the parameters that authorize the access to the
# source, such as its customer-provided key.
info1 = self.info(path1, **self.core.copy_source_params(kwargs))
if info1.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY:
# Recursive copy() passes the directories too; S3 has no object
# to copy for them.
return False
size1 = info1.get("size", 0)
try:
if size1 <= self.core.MULTIPART_UPLOAD_MAX_PART_SIZE:
raise_if_stopped(f"copy {source.uri} to {destination.uri}")
self.core.copy_object(source, destination, **kwargs)
else:
self._copy_object_with_multipart_upload(
source,
destination,
max_workers=max_workers,
block_size=block_size,
**kwargs,
)
finally:
# A multipart copy that fails to copy an annotation has already
# written the destination.
self._invalidate(destination)
return True
def _copy_object_with_multipart_upload(
self,
source: S3Path,
destination: S3Path,
max_workers: int | None = None,
block_size: int | None = None,
**kwargs,
) -> None:
"""Copy an object with a multipart upload of its byte ranges.
Runs the plan of :meth:`S3Core.plan_multipart_copy`, which reads the
source and lists its annotations before anything is written. The
parts are copied in parallel with UploadPartCopy, ``max_workers`` at
a time, each one sent once an earlier one has finished, and the
annotations are copied onto the destination after the upload
completes. A failed part or completion aborts the upload, and so does
an interrupt, after the creation of the upload and the running part
copies, which then make no further attempt, have finished. Further
interrupts while the creation is waited for are suppressed; one while
the part copies are waited for or the upload is aborted stops the
cleanup, and the upload ID is logged. A failed annotation copy is
raised and leaves the destination in place. The annotations are
copied, retries included, also once an enclosing stop scope, such as
that of a batch operation, has been stopped, since the destination is
committed; until then, a stop sends no further part or completion
and the upload is aborted. If HeadObject reports a
size that fits in a single CopyObject request, the reported version is
copied with CopyObject instead.
Args:
source: Source S3 path, with the version ID to copy, if any.
destination: Destination S3 path.
max_workers: Maximum number of parallel requests.
block_size: Size in bytes of the copied ranges.
**kwargs: The CopyObject parameters of the copy; each request
receives those that it accepts.
Raises:
ValueError: If ``block_size`` is out of the part size limits, a
directive has an invalid value, or HeadObject reports no
size.
"""
max_workers = max_workers if max_workers else self.max_workers
plan = self.core.plan_multipart_copy(source, destination, block_size, **kwargs)
if plan.fits_single_request:
# The size that the caller found, which may come from a cached
# listing, was larger than the copied version, which fits in a
# single CopyObject request.
raise_if_stopped(f"copy {plan.source.uri} to {plan.destination.uri}")
self.core.copy_object(plan.source, plan.destination, **kwargs)
return
with self._create_executor(max_workers=max_workers) as executor:
# The requests run on the executor, so that an interrupt lets the
# running ones finish and the upload be aborted.
session = _S3WriteSession(
self.core,
plan.destination,
executor,
block_size=block_size if block_size else self.core.MULTIPART_UPLOAD_MAX_PART_SIZE,
max_workers=max_workers,
prepared_params={
"create_multipart_upload": plan.create_params,
"upload_part_copy": plan.part_params,
"complete_multipart_upload": plan.complete_params,
"abort_multipart_upload": plan.abort_params,
},
)
try:
session.start()
for part_number, range_ in enumerate(plan.ranges, 1):
session.wait_ready()
session.submit_copy(part_number, plan.source, range_)
completed = session.complete()
except BaseException:
try:
session.abort()
except KeyboardInterrupt:
if (upload := session.upload) is not None:
_logger.warning(
f"Interrupted the abort of multipart upload {upload.upload_id} "
f"to s3://{upload.bucket}/{upload.key}."
)
raise
except Exception:
# Logged instead of raised, so that the error that made
# the copy fail is raised.
upload = cast(S3MultipartUpload, session.upload)
_logger.exception(
f"Failed to abort multipart upload {upload.upload_id} "
f"to s3://{upload.bucket}/{upload.key}."
)
raise
# The destination is committed, so its annotations are copied to the
# end also when an enclosing batch has stopped.
with unstoppable_retries():
for name in plan.annotations:
self.core.copy_object_annotation(
name,
plan.source,
plan.destination,
completed.version_id,
completed.etag,
**kwargs,
)
[docs]
def pipe(self, path, value=None, batch_size=None, **kwargs) -> None:
"""Write bytes into one or more paths.
As fsspec's ``pipe()``, except that the values of a dict are written
in parallel with ``pipe_file()``, as ``copy()`` copies its files.
Args:
path: S3 path to write ``value`` to, or a dict of the values to
write by path.
value: The bytes to write to a single path; ignored with a dict.
batch_size: Number of writes to run at the same time, as in
``copy()``.
**kwargs: Additional parameters passed to ``pipe_file()``.
Raises:
ValueError: If ``path`` is neither a string nor a dict, two paths
of a dict name the same object, or ``batch_size`` is neither
positive, -1, nor None.
"""
if not isinstance(path, dict):
path = stringify_path(path)
limit = self._batch_limit(batch_size)
if isinstance(path, str):
self.pipe_file(self._strip_protocol(path), value, **kwargs)
return
if not isinstance(path, dict):
raise ValueError("path must be str or dict")
items = self._pipe_items(path)
if not items:
return
concurrency = min(limit, len(items))
call_kwargs = self._batch_kwargs(concurrency, kwargs)
def pipe_item(item: tuple[str, Any]) -> None:
"""Write a value.
Args:
item: The path and its value.
"""
self.pipe_file(*item, **call_kwargs)
self._run_batch(pipe_item, items, concurrency)
def _pipe_items(self, values: Mapping[Any, Any]) -> list[tuple[str, Any]]:
"""Return the paths and values of a ``pipe()`` of a dict, which are written in parallel.
Args:
values: The values to write by path.
Returns:
The paths without the protocol, and their values.
Raises:
ValueError: If two paths name the same object.
"""
items = [(self._strip_protocol(k), v) for k, v in values.items()]
if len({k for k, _ in items}) != len(items):
raise ValueError("Cannot pipe several values to the same destination.")
return items
[docs]
def pipe_file(
self,
path: str | os.PathLike[str],
value: bytes | bytearray | memoryview,
mode: str = "overwrite",
**kwargs,
) -> None:
"""Write bytes into the path.
Writes data up to the block size with a single PutObject request
instead of the inherited ``open()`` + ``write()`` path. Larger data
and writes inside an fsspec transaction go through the buffered
path, which uploads the data as a parallel multipart upload and
keeps the deferred-commit semantics of transactions. A write that
fails on that path leaves the existing object unchanged. Both paths
write to the path without a trailing slash, as ``open()`` does.
Args:
path: S3 path (s3://bucket/key) to write to.
value: The bytes to write.
mode: "overwrite" (default) or "create". With "create", raise
FileExistsError when the object already exists, including
one created during the write, which is not replaced.
**kwargs: Additional parameters passed to the PutObject API
(e.g., ContentType, StorageClass) on the single-request
path. The ``block_size``, ``max_workers``, and
``s3_additional_kwargs`` parameters of the ``open()`` path
are also accepted, and so is ``compression``: the codec of
``open()`` to compress the value with before it is
uploaded, or ``"infer"`` to take it from the extension of
the path.
Raises:
FileExistsError: If the mode is "create" and the path already
exists, or an object is created at it before the write is
committed.
ValueError: If the path does not contain a key or specifies a
version, if the compression is not supported, or if the data
takes more than ``S3Core.MULTIPART_UPLOAD_MAX_PARTS`` blocks.
"""
# Normalized as open() normalizes it, so that the key written, and
# the codec that "infer" takes from its extension, do not depend on
# the path that the size of the value selects.
path = self._strip_protocol(path)
compression = get_compression(path, kwargs.pop("compression", None))
if compression is not None:
# Compressed up front, so that every path uploads the compressed
# bytes, and open() returns the file instead of a wrapper.
value = CompressedBuffer.compress(value, compression)
block_size = kwargs.get("block_size") or self.default_block_size
# The size in bytes; the length of a memoryview counts its items.
size = memoryview(value).nbytes
self.core.check_multipart_upload_size(size, block_size)
if self._intrans or size > min(block_size, self.core.MULTIPART_UPLOAD_MAX_PART_SIZE):
# Defer to the buffered open() path, which keeps the
# deferred-commit semantics of fsspec transactions and uploads
# large data as a parallel multipart upload.
self.open(path, "xb" if mode == "create" else "wb", **kwargs)._write_and_close(value)
return
s3_path = S3Path.parse(path)
if s3_path.version_id:
raise ValueError("Cannot write to the file with the version specified.")
if not s3_path.key:
raise ValueError("Cannot write to a bucket.")
kwargs.pop("block_size", None)
kwargs.pop("max_workers", None)
request_kwargs = {
**self.core.operation_params("put_object", self.s3_additional_kwargs),
**kwargs.pop("s3_additional_kwargs", {}),
**kwargs,
}
if mode == "create":
# Checked up front, as open() does in "xb" mode, and with
# IfNoneMatch for an object created since.
if self._object_exists(path, self._get_lookup_kwargs(request_kwargs)):
raise FileExistsError(path)
request_kwargs["IfNoneMatch"] = "*"
if not isinstance(value, bytes):
# Accept bytes-like values (bytearray, memoryview) as the
# buffered path does.
value = bytes(value)
raise_if_stopped(f"put {s3_path.uri}")
self.core.put_object(s3_path, value, **request_kwargs)
self.invalidate_cache(path)
[docs]
def cat(self, path, recursive=False, on_error="raise", batch_size=None, **kwargs):
"""Read the contents of one or more paths.
As fsspec's ``cat()``, except that the objects are read in parallel
with ``cat_file()``, as ``copy()`` copies its files. With
``on_error="raise"``, an error starts no further read and is raised
once the running reads have ended.
Args:
path: S3 path, glob pattern, or list of them.
recursive: Whether to read the files below the directories.
on_error: ``"raise"`` to raise an error of a read, ``"omit"`` to
leave out the paths whose read failed, or ``"return"`` to
return the error in place of their contents.
batch_size: Number of reads to run at the same time, as in
``copy()``.
**kwargs: Additional parameters passed to ``expand_path()`` and
``cat_file()``.
Returns:
The contents of the path, or a dict of the contents by path if
there are several paths or the path was expanded.
Raises:
ValueError: If ``batch_size`` is neither positive, -1, nor None.
"""
limit = self._batch_limit(batch_size)
targets = self._first_targets(self._expand_paths(path, recursive=recursive, **kwargs))
paths = list(targets)
if not (len(paths) > 1 or isinstance(path, list) or paths[0] != self._strip_protocol(path)):
return self.cat_file(targets[paths[0]], **kwargs)
outcomes: list[Any] | None = None
contents: dict[str, Any] | None = None
outcome = None
try:
outcomes = self._run_batch(
functools.partial(self.cat_file, **kwargs),
list(targets.values()),
min(limit, len(paths)),
counts=None if on_error == "raise" else lambda e: False,
).outcomes
contents = {}
for p, outcome in zip(paths, outcomes, strict=True):
if on_error != "omit" or not isinstance(outcome, Exception):
contents[p] = outcome
return contents
except BaseException:
# Not kept by this frame, which the traceback of an interrupt
# keeps (F5).
contents = outcomes = outcome = None
raise
@staticmethod
def _first_targets(expanded: list[tuple[str, str | S3Path]]) -> dict[str, str | S3Path]:
"""Return the first path that each name of an expansion names.
An explicit version and a listed key that ends in the same query
share a name. The contents are returned by name, so a name is read
once, as the first path that it names, which is the path that the
caller passed.
Args:
expanded: The names and the paths that they name, as
:meth:`_expand_paths` returns them.
Returns:
The paths by name, in the order of the names.
"""
targets: dict[str, str | S3Path] = {}
for name, target in expanded:
targets.setdefault(name, target)
return targets
[docs]
def cat_ranges(
self, paths, starts, ends, max_gap=None, on_error="return", batch_size=None, **kwargs
) -> list[bytes | Exception]:
"""Read byte ranges of one or more paths.
As fsspec's ``cat_ranges()``, except that the ranges are read in
parallel with ``cat_file()``, as ``copy()`` copies its files. Unless
``on_error`` is ``"return"``, an error starts no further read and is
raised once the running reads have ended.
Args:
paths: The S3 paths, one per range.
starts: The start offset of each range, or one for all of them.
ends: The end offset of each range, or one for all of them.
max_gap: Not supported; must be None.
on_error: ``"return"`` to return the error of a read in place of
its bytes; otherwise the error is raised.
batch_size: Number of reads to run at the same time, as in
``copy()``.
**kwargs: Additional parameters passed to ``cat_file()``.
Returns:
The bytes of each range, or the error of its read.
Raises:
NotImplementedError: If ``max_gap`` is given.
TypeError: If ``paths`` is not a list.
ValueError: If the numbers of paths, starts, and ends differ, or
if ``batch_size`` is neither positive, -1, nor None.
"""
if max_gap is not None:
raise NotImplementedError
if not isinstance(paths, list):
raise TypeError
if not isinstance(starts, list):
starts = [starts] * len(paths)
if not isinstance(ends, list):
ends = [ends] * len(paths)
if len(starts) != len(paths) or len(ends) != len(paths):
raise ValueError
limit = self._batch_limit(batch_size)
if not paths:
return []
def cat_range(item: tuple[str, int | None, int | None]) -> bytes:
"""Read a range.
Args:
item: The path, start, and end.
Returns:
The bytes of the range.
"""
return self.cat_file(*item, **kwargs)
return cast(
list[bytes | Exception],
self._run_batch(
cat_range,
list(zip(paths, starts, ends, strict=True)),
min(limit, len(paths)),
counts=None if on_error != "return" else lambda e: False,
).outcomes,
)
[docs]
def cat_file(
self,
path: str | os.PathLike[str] | S3Path,
start: int | None = None,
end: int | None = None,
**kwargs,
) -> bytes:
"""Read the contents of an S3 object with GetObject.
``start`` and ``end`` select bytes like a slice of the object: an
empty range, or one that starts at or past the end of the object,
returns ``b""``, and an end past the object reads up to its end.
Non-negative offsets are sent to S3 as they are, and so is a negative
``start`` without an ``end``, as a suffix range of the last bytes.
Other negative offsets are resolved against the size from
:meth:`info`, which also checks that the object exists for an empty
range and receives the lookup parameters among ``kwargs``.
Args:
path: S3 path (s3://bucket/key) of the object, or an ``S3Path``,
whose bucket, key and version are read as they are, such as a
key that a listing returns.
start: Byte offset to start reading at. A negative value counts
from the end of the object.
end: Byte offset to stop reading at (exclusive). A negative value
counts from the end of the object.
**kwargs: Additional parameters passed to the GetObject API,
except ``version_id``: the version ID to read when the path
has none.
Returns:
The bytes read from the object.
Raises:
FileNotFoundError: If the path has no key or the key does not
exist.
"""
if isinstance(path, S3Path):
return self._cat_object(path, start=start, end=end, **kwargs)
path = stringify_path(path)
return self._cat_object(S3Path.parse(path), start=start, end=end, name=path, **kwargs)
def _cat_object(
self,
path: S3Path,
start: int | None = None,
end: int | None = None,
name: str | None = None,
**kwargs,
) -> bytes:
"""Read the contents of the object of a path, as :meth:`cat_file` describes.
Args:
path: The path of the object, whose bucket, key and version are
used as they are.
start: Byte offset to start reading at, as in :meth:`cat_file`.
end: Byte offset to stop reading at, as in :meth:`cat_file`.
name: The path string that the caller passed, which ``info()``
looks up for the size of the object of a negative offset or
an empty range, and which a missing object is reported by;
None to look up ``path`` itself.
**kwargs: Additional parameters passed to the GetObject API, and
``version_id``, as in :meth:`cat_file`.
Returns:
The bytes read from the object.
Raises:
FileNotFoundError: If the path has no key or the key does not
exist.
"""
if not path.key:
raise FileNotFoundError(name or str(path))
version_id = kwargs.pop("version_id", None)
if path.version_id:
version_id = path.version_id
ranges: tuple[int, int | None] | None = None
if start is not None and start < 0 and end is None:
# S3 returns the last bytes, or the whole object when it is
# shorter, without the size of the object.
ranges = (start, None)
else:
if (start is not None and start < 0) or (
end is not None and (end < 0 or (start or 0) >= end)
):
# A negative offset needs the size of the object, and an
# empty range sends no GetObject request that would report a
# missing object.
lookup_kwargs = self._get_lookup_kwargs(kwargs)
info = self.info(
name if name is not None else path, version_id=version_id, **lookup_kwargs
)
if (
info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY
or info.key != path.key
):
# There is no object to read, as GetObject reports for
# the other ranges, or info() describes the key without
# the trailing slash of this one.
raise FileNotFoundError(name or str(path))
start, end, _ = slice(start, end).indices(info.get("size", 0))
if start is not None or end is not None:
start = start or 0
if end is not None and start >= end:
# S3 would return the whole object for an empty range.
return b""
ranges = (start, end)
try:
return self.core.get_object(path.with_version_id(version_id), ranges, **kwargs)
except OSError as e:
if ranges and S3ClientError.code_of(e) == "InvalidRange":
# The range starts at or past the end of the object.
return b""
raise
[docs]
def put(
self,
lpath,
rpath,
recursive=False,
callback=_DEFAULT_CALLBACK,
maxdepth=None,
batch_size=None,
**kwargs,
) -> None:
"""Copy files from the local filesystem to S3.
As fsspec's ``put()``, except that the files are uploaded in parallel
with ``put_file()``, as ``copy()`` copies its files. A failed upload
starts no further one, the running ones send no further part and do
not commit, and the error is raised once they have ended.
Args:
lpath: Local source path, glob pattern, or list of them.
rpath: S3 destination path, or list of paths when ``lpath`` is a
list.
recursive: Whether to copy the directories with their contents.
callback: Progress callback. It and its branches for the files
are used from the threads that upload them, one at a time.
maxdepth: Maximum depth of a recursive copy.
batch_size: Number of uploads to run at the same time, as in
``copy()``.
**kwargs: Additional parameters passed to ``put_file()``.
Raises:
ValueError: If two files have the same destination, or
``batch_size`` is neither positive, -1, nor None.
"""
lpath, rpath = self._stringify_paths(lpath), self._stringify_paths(rpath)
limit = self._batch_limit(batch_size)
pairs = self._put_pairs(lpath, rpath, recursive=recursive, maxdepth=maxdepth, **kwargs)
if not pairs:
return
concurrency = min(limit, len(pairs))
call_kwargs = self._batch_kwargs(concurrency, kwargs)
locked = _S3LockedCallback(callback, threading.Lock())
def put_pair(pair: tuple[str, str]) -> None:
"""Upload a pair with a branch of the callback.
Args:
pair: The local source and its S3 destination.
"""
lp, rp = pair
with locked.branched(lp, rp) as child:
self.put_file(lp, rp, callback=child, **call_kwargs)
locked.relative_update(1)
locked.set_size(len(pairs))
self._run_batch(put_pair, pairs, concurrency)
def _put_pairs(
self, lpath, rpath, recursive=False, maxdepth=None, **kwargs
) -> list[tuple[str, str]]:
"""Pair the local sources of ``put()`` with their S3 destinations as fsspec does.
The files are uploaded in parallel, so no two files may have the
same destination; a local directory uploads nothing.
Args:
lpath: Local source path, glob pattern, or list of them.
rpath: S3 destination path, or list of paths when ``lpath`` is a
list.
recursive: Whether to include the directories with their contents.
maxdepth: Maximum depth of a recursive copy.
**kwargs: Additional arguments passed to the ``expand_path()`` of
the local filesystem.
Returns:
The local paths, including the directories of a recursive copy,
with their destinations without the protocol.
Raises:
ValueError: If two files have the same destination.
"""
if isinstance(lpath, list) and isinstance(rpath, list):
pairs = [(lp, self._strip_protocol(rp)) for lp, rp in zip(lpath, rpath, strict=False)]
else:
source_is_str = isinstance(lpath, str)
if source_is_str:
lpath = make_path_posix(lpath)
local = LocalFileSystem()
lpaths = local.expand_path(lpath, recursive=recursive, maxdepth=maxdepth, **kwargs)
if source_is_str and (not recursive or maxdepth is not None):
# A string source without recursion does not copy directories.
lpaths = [p for p in lpaths if not (trailing_sep(p) or local.isdir(p))]
if not lpaths:
return []
dest_is_dir = isinstance(rpath, str) and (trailing_sep(rpath) or self.isdir(rpath))
# A list of destinations or one path, also path-like, as in fsspec.
rpath = self._strip_protocol(rpath)
exists = source_is_str and (
(has_magic(lpath) and len(lpaths) == 1)
or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath))
)
rpaths = other_paths(lpaths, rpath, exists=exists, flatten=not source_is_str)
pairs = list(zip(lpaths, rpaths, strict=True))
# put_file() uploads nothing for a local directory.
files = [rp for lp, rp in pairs if not os.path.isdir(lp)]
if len(set(files)) != len(files):
raise ValueError("Cannot put several paths to the same destination.")
return pairs
[docs]
def put_file(
self,
lpath: str | os.PathLike[str],
rpath: str | os.PathLike[str],
callback=_DEFAULT_CALLBACK,
mode: str = "overwrite",
**kwargs,
):
"""Upload a local file to S3.
Uploads a file from the local filesystem to an S3 location. Supports
automatic content type detection based on file extension and provides
progress callback functionality. An upload that fails before it is
completed, including one of a local file that cannot be read, leaves
the existing object unchanged.
Args:
lpath: Local file path to upload.
rpath: S3 destination path (s3://bucket/key).
callback: Progress callback for tracking upload progress.
mode: "overwrite" (default) or "create". With "create", the file
is written as with ``open()`` in ``xb`` mode: raise
FileExistsError when the object already exists, including
one created during the upload, which is not replaced.
**kwargs: Additional S3 parameters (e.g., ContentType, StorageClass).
The ``block_size``, ``max_workers``, and ``s3_additional_kwargs``
parameters of ``open()`` are also accepted.
Raises:
FileExistsError: If the mode is "create" and the path already
exists, or an object is created at it before the upload is
committed.
ValueError: If the file takes more than
``S3Core.MULTIPART_UPLOAD_MAX_PARTS`` blocks.
Note:
Directories are not supported for upload. If lpath is a directory,
the method returns without performing any operation. Bucket-only
destinations (without key) are also not supported.
"""
rpath = stringify_path(rpath)
if os.path.isdir(lpath):
# No support for directory uploads.
return
if not S3Path.parse(rpath).key:
# No support for bucket copy.
return
size = os.path.getsize(lpath)
block_size = kwargs.pop("block_size", None) or self.default_block_size
max_workers = kwargs.pop("max_workers", self.max_workers)
# The other parameters are S3 request parameters, as in pipe_file().
s3_additional_kwargs = {**kwargs.pop("s3_additional_kwargs", {}), **kwargs}
self.core.check_multipart_upload_size(size, block_size)
callback.set_size(size)
if "ContentType" not in {**self.s3_additional_kwargs, **s3_additional_kwargs}:
content_type, _ = mimetypes.guess_type(lpath)
if content_type is not None:
s3_additional_kwargs["ContentType"] = content_type
# The local file is opened first, so that an unreadable one fails
# before the remote file is opened.
with open(lpath, "rb") as local:
self.open(
rpath,
"xb" if mode == "create" else "wb",
block_size=block_size,
max_workers=max_workers,
s3_additional_kwargs=s3_additional_kwargs,
)._write_file_and_close(local, callback)
self.invalidate_cache(rpath)
[docs]
def get_file(
self,
rpath: str | os.PathLike[str] | S3Path,
lpath=None,
callback=_DEFAULT_CALLBACK,
outfile=None,
**kwargs,
):
"""Download an S3 file to local filesystem.
The object is read in waves of parallel range requests of the block
size, at most ``max_workers`` of them at once, but no more than the
request pool of the filesystem runs, and the bodies of each wave are
written in order before the next wave is read, so that the download
holds at most one wave of bodies; a thread of the pool that has just
read a range drops its body when its call returns (see
:meth:`S3File._download`). The reads are
conditional on the version and the ETag of the object looked up when
the download starts, as with ``open()``, so that an object replaced
during the download fails it instead of mixing two objects, unless
the download reads a version other than ``null``, which a
replacement does not change. A failed
download leaves the part written so far in a local file ``lpath``,
as fsspec's ``AbstractFileSystem.get_file()`` does.
As with fsspec's ``AbstractFileSystem.get_file()``, a directory
``rpath`` creates the local directory ``lpath``, and the parent
directories of a local file ``lpath`` are created as needed.
Args:
rpath: S3 source path (s3://bucket/key), or an ``S3Path``, whose
bucket, key and version are used as they are, such as a key
that a listing returns.
lpath: Local destination path, or a file-like object to write to.
Not needed when ``outfile`` is given.
callback: Progress callback for tracking download progress, with
the size of the object.
outfile: A file-like object to write to instead of ``lpath``.
**kwargs: The arguments of ``open()``, such as ``block_size``,
``max_workers``, ``version_id``, and ``s3_additional_kwargs``,
and additional S3 parameters. ``cache_type`` and
``cache_options`` are not used. With ``compression``, the
object is decompressed into the destination, read in order
through ``open()`` with a block cache, whose block size, if
not positive, is that of the filesystem, or
``DEFAULT_BLOCK_SIZE`` if that is not positive either, and the
size of the callback is unknown.
"""
s3_path = S3Path.of(rpath if isinstance(rpath, S3Path) else self._strip_protocol(rpath))
if outfile is None and isfilelike(lpath):
outfile = lpath
elif outfile is None and self._is_directory_source(rpath, kwargs.get("version_id")):
os.makedirs(lpath, exist_ok=True)
return
# The codec, if any, as fsspec's open() resolves it, with "infer"
# from the key without the version.
codec = get_compression(s3_path.key or "", kwargs.pop("compression", None))
# Not the caller's cache: one such as "all" would read the whole
# object when the file is opened, and "background" reads outside
# the stop scopes of the caller. Without a codec, the object is read
# in ranges without a cache; with one, the decompressor's small reads
# go through a block cache, which needs a positive block size.
kwargs.pop("cache_options", None)
kwargs["cache_type"] = "none" if codec is None else "bytes"
if codec is not None and not (
isinstance(kwargs.get("block_size"), int) and kwargs["block_size"] > 0
):
kwargs["block_size"] = (
self.default_block_size if self.default_block_size > 0 else self.DEFAULT_BLOCK_SIZE
)
# The remote file is opened first so that no local file is created
# when open() finds no object at the path.
with contextlib.ExitStack() as stack:
remote = stack.enter_context(
self._open(rpath, "rb", **kwargs)
if isinstance(rpath, S3Path)
else self.open(rpath, "rb", **kwargs)
)
reader: Any = None
if codec is not None:
# As fsspec's open() wraps the file, but closed with it.
reader = stack.enter_context(compr[codec](remote, mode="r"))
if outfile is None:
# Not abspath(), which would resolve ".." before symlinks.
if parent := os.path.dirname(lpath):
os.makedirs(parent, exist_ok=True)
outfile = stack.enter_context(open(lpath, "wb"))
if reader is None:
callback.set_size(remote.size)
remote._download(outfile, callback)
return
# The size of the decompressed data is unknown.
callback.set_size(None)
while data := reader.read(remote.blocksize):
# A block read when the stop came is not written.
raise_if_stopped(f"write the contents of {rpath}")
outfile.write(data)
callback.relative_update(len(data))
[docs]
def checksum(self, path: str | os.PathLike[str], **kwargs):
"""Get checksum for S3 object or directory.
Computes a checksum for the specified S3 path. For individual objects,
returns the ETag converted to an integer. For directories, returns a
checksum based on the directory's tokenized representation.
Args:
path: S3 path (s3://bucket/key) to get checksum for.
**kwargs: Additional arguments including:
refresh: If True, refresh cached info before computing checksum.
Returns:
Integer checksum value derived from S3 ETag or directory token.
Note:
For multipart uploads, ETag format is different and only the first
part before the dash is used for checksum calculation.
"""
refresh = kwargs.pop("refresh", False)
info = self.info(path, refresh=refresh)
if info.get("type") != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY:
return int(info.get("etag").strip('"').split("-")[0], 16)
return int(tokenize(info), 16)
[docs]
def sign(self, path: str | os.PathLike[str], expiration: int = 3600, **kwargs):
"""Generate a presigned URL for S3 object access.
Creates a presigned URL that allows temporary access to an S3 object
without requiring AWS credentials. Useful for sharing files or providing
time-limited access to resources.
Args:
path: S3 path (s3://bucket/key) to generate URL for.
expiration: URL expiration time in seconds. Defaults to 3600 (1 hour).
**kwargs: Additional parameters including:
client_method: S3 operation ('get_object', 'put_object', etc.).
Defaults to 'get_object'.
Additional parameters passed to the S3 operation.
Returns:
Presigned URL string that provides temporary access to the S3 object.
Example:
>>> fs = S3FileSystem()
>>> url = fs.sign("s3://my-bucket/file.txt", expiration=7200)
>>> # URL valid for 2 hours
>>>
>>> # Generate upload URL
>>> upload_url = fs.sign(
... "s3://my-bucket/upload.txt",
... client_method="put_object"
... )
"""
path = stringify_path(path)
client_method = kwargs.pop("client_method", "get_object")
return self.core.generate_presigned_url(
S3Path.parse(path), client_method, expiration, **kwargs
)
[docs]
def getxattr(self, path: str | os.PathLike[str], attr_name: str, **kwargs) -> str | None:
"""Get an attribute from the user-defined metadata of the path.
Args:
path: S3 path (s3://bucket/key) to get the attribute for.
attr_name: The name of the attribute.
**kwargs: Additional parameters passed to :meth:`metadata`.
Returns:
The value of the attribute, or None if the attribute is not set.
"""
return self.metadata(path, **kwargs).get(attr_name)
[docs]
def setxattr(
self, path: str | os.PathLike[str], copy_kwargs: dict[str, Any] | None = None, **kw_args
) -> None:
"""Set the user-defined metadata of the path.
S3 does not allow updating the metadata of an existing object in
place, so the object is copied onto itself with the REPLACE metadata
directive. Note that this rewrites the object and updates its
last-modified time. The system-defined metadata (e.g.,
``ContentType`` and ``CacheControl``), the storage class, and the
server-side encryption algorithm and KMS key of the object are kept.
HeadObject does not return the KMS encryption context, and an
``Expires`` value that botocore cannot parse as a date is not kept.
Args:
path: S3 path (s3://bucket/key) to set metadata for. A path with
a version ID is rejected, since copying a version onto the
key would replace the current object with it.
copy_kwargs: Additional parameters to use for the underlying
CopyObject API call. They take precedence over the kept
system-defined metadata and storage class. Any encryption
parameter replaces all kept encryption settings. The lookup
of the object receives ``RequestPayer``, and the source's
expected bucket owner and SSE-C parameters (e.g.,
``CopySourceSSECustomerKey``) under the names of HeadObject
(see :meth:`S3Core.copy_source_params`).
**kw_args: Key-value pairs to set, where the values must be
strings. The keys are used as-is; names that are not valid
Python identifiers (e.g., containing hyphens) can be passed
by unpacking a dictionary. Does not alter existing fields,
unless the field appears here - if the value is None, delete
the field.
Example:
>>> fs = S3FileSystem()
>>> fs.setxattr("s3://bucket/key", attribute1="value1")
>>> fs.setxattr("s3://bucket/key", **{"attribute-2": "value2"})
Raises:
ValueError: If the path is a bucket or has a version ID.
"""
path = stringify_path(path)
s3_path = S3Path.parse(path)
if not s3_path.key:
raise ValueError("Cannot set metadata of a bucket.")
if s3_path.version_id:
raise ValueError("Cannot set metadata of a version.")
copy_kwargs = copy_kwargs or {}
# Looked up with the parameters that authorize the access to the
# object as the source of the copy, such as its customer-provided key.
head = self.metadata(path, **self.core.copy_source_params(copy_kwargs))
metadata = dict(head)
for k, v in kw_args.items():
if v is None:
metadata.pop(k, None)
else:
metadata[k] = v
self.core.replace_object_metadata(s3_path, head, metadata, **copy_kwargs)
self.invalidate_cache(path)
[docs]
def chmod(
self, path: str | os.PathLike[str], acl: str, recursive: bool = False, **kwargs
) -> None:
"""Set the Access Control on a bucket/key.
See https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
Args:
path: S3 path (s3://bucket or s3://bucket/key) to set the ACL on.
acl: The value of the canned ACL to apply.
recursive: Whether to apply the ACL to all keys below the given
path too, and to the object at the key, if any. The keys are
updated in parallel; once one fails, the updates that have not
started are cancelled, and the running ones make no further
attempt. A path with a version ID updates that version only.
**kwargs: Additional parameters passed to the PutObjectAcl or
PutBucketAcl API.
"""
path = stringify_path(path)
s3_path = S3Path.parse(path)
# Validate before any ACL is applied so that a recursive call cannot
# partially apply object ACLs and then fail on the bucket ACL.
if not s3_path.key and acl not in self.core.BUCKET_ACLS:
raise ValueError(f"ACL not in {self.core.BUCKET_ACLS}.")
if s3_path.key and acl not in self.core.OBJECT_ACLS:
raise ValueError(f"ACL not in {self.core.OBJECT_ACLS}.")
if recursive and not (s3_path.key and s3_path.version_id):
found = self._find_paths(path, withdirs=False)
items: list[tuple[str | S3Path, bool]] = [(target, False) for _, target in found]
name = self._strip_protocol(path)
# The key of the path as find() names it ("bucket//" is the bucket).
if S3Path.parse(name).key and found and name not in {f.name for f, _ in found}:
# Objects lie below the key, so find() has not looked the key
# itself up; it may be an object too, which is skipped if missing.
items.append((name, True))
self._map(lambda item: self._chmod_object(*item, acl, **kwargs), items)
if s3_path.key:
return
if s3_path.key:
self.core.put_object_acl(s3_path, acl, **kwargs)
else:
self.core.put_bucket_acl(s3_path.bucket, acl, **kwargs)
def _chmod_object(self, path: str | S3Path, missing_ok: bool, acl: str, **kwargs) -> None:
"""Apply a canned ACL to an object in a recursive ``chmod``.
Args:
path: S3 path of the object.
missing_ok: Whether to skip the object if it does not exist.
acl: The canned ACL, one of ``OBJECT_ACLS``.
**kwargs: Additional parameters passed to the PutObjectAcl API.
Raises:
FileNotFoundError: If the object does not exist and ``missing_ok``
is False.
"""
try:
self.core.put_object_acl(
S3Path.parse(path) if isinstance(path, str) else path, acl, **kwargs
)
except FileNotFoundError:
if not missing_ok:
raise
[docs]
def list_multipart_uploads(self, path: str | os.PathLike[str]) -> list[S3MultipartUpload]:
"""List in-progress (incomplete) multipart uploads in a bucket.
Incomplete multipart uploads continue to accrue storage costs until
they are completed or aborted. Use :meth:`clear_multipart_uploads`
to abort all of them.
Args:
path: S3 bucket or key path (e.g., "bucket", "s3://bucket" or
"s3://bucket/prefix"). If the path contains a key, only the
uploads to that key and to the keys under ``key/`` are
listed, not those to sibling keys that merely start with the
same characters (e.g., ``prefix2/a``).
Returns:
List of S3MultipartUpload instances describing the in-progress
multipart uploads.
"""
path = stringify_path(path)
s3_path = S3Path.parse(path)
# S3 matches Prefix as a plain string, so the uploads are filtered to
# the key itself and the keys under it.
prefix = f"{s3_path.key.rstrip('/')}/" if s3_path.key else ""
_logger.debug(f"List multipart uploads: {s3_path.uri}")
return [
upload
for page in self.core.list_multipart_uploads(s3_path.bucket, prefix=s3_path.key or None)
for upload in page.uploads
if upload.key is not None
and (upload.key == s3_path.key or upload.key.startswith(prefix))
]
[docs]
def object_version_info(
self, path: str | os.PathLike[str], delete_markers: bool = False, **kwargs
) -> list[S3ObjectVersion]:
"""List the versions of the object or of the objects under the path.
A key path without a trailing slash selects that key if it has any
versions or delete markers, and otherwise the keys under ``key/``.
The choice does not depend on ``delete_markers``, so a key that has
only delete markers yields no versions without them. A key path with
a trailing slash selects the keys under it, and a bucket path selects
all the keys in the bucket. Sibling keys that merely start with the
same characters (e.g., ``key.bak``) are never included.
Args:
path: S3 path (s3://bucket/key or a key prefix) to list the
versions for.
delete_markers: Whether to include delete markers in the result.
**kwargs: Additional parameters passed to the ListObjectVersions
API.
Returns:
List of S3ObjectVersion instances describing the versions.
"""
path = stringify_path(path)
s3_path = S3Path.parse(path)
# S3 matches Prefix as a plain string, so the versions are filtered to
# the key itself or the keys under it.
prefix = f"{s3_path.key.rstrip('/')}/" if s3_path.key else ""
_logger.debug(f"List object versions: {s3_path.uri}")
versions: list[S3ObjectVersion] = []
# Explicit markers start the listing, which the pages then advance.
key_marker = kwargs.pop("KeyMarker", None)
version_id_marker = kwargs.pop("VersionIdMarker", None)
for page in self.core.list_object_versions(
s3_path.bucket,
prefix=s3_path.key or "",
key_marker=key_marker,
version_id_marker=version_id_marker,
**kwargs,
):
versions.extend(page.versions)
# Delete markers are kept until the key is chosen, so that the
# choice is the same with and without them.
versions.extend(page.delete_markers)
# botocore decodes the keys only when it sets EncodingType itself, so
# the keys of an explicit EncodingType="url" are decoded for matching.
url_encoded = kwargs.get("EncodingType") == "url"
keys = [unquote_plus(v.key) if url_encoded else v.key for v in versions]
if s3_path.key and not s3_path.key.endswith("/") and s3_path.key in keys:
selected = [v for v, k in zip(versions, keys, strict=True) if k == s3_path.key]
else:
selected = [v for v, k in zip(versions, keys, strict=True) if k.startswith(prefix)]
return [v for v in selected if delete_markers or not v.is_delete_marker]
[docs]
def clear_multipart_uploads(self, path: str | os.PathLike[str]) -> None:
"""Abort any incomplete multipart uploads in the bucket.
Uploads completed or aborted after listing are already cleared.
All abort results are checked before another abort error is raised.
Args:
path: S3 bucket or key path (e.g., "bucket", "s3://bucket" or
"s3://bucket/prefix"). If the path contains a key, only the
uploads to that key and to the keys under ``key/`` are
aborted, as listed by :meth:`list_multipart_uploads`.
"""
uploads = self.list_multipart_uploads(path)
if not uploads:
return
# Every abort runs also after another failed, and an interrupt cancels
# those that have not started.
results = self._map(
self.core.abort_multipart_upload,
uploads,
stop_on_failure=False,
counts=self._is_not_no_such_upload,
)
if results.failure is not None:
raise results.failure
@staticmethod
def _is_not_no_such_upload(error: BaseException) -> bool:
"""Return whether an error of an abort is other than an upload that no longer exists.
Args:
error: The error of ``abort_multipart_upload``.
Returns:
False for the ``FileNotFoundError`` of a ``NoSuchUpload`` error.
"""
return not (
isinstance(error, FileNotFoundError) and S3ClientError.code_of(error) == "NoSuchUpload"
)
[docs]
def created(self, path: str | os.PathLike[str]) -> datetime:
"""Return the creation time of the path.
Returns the same value as :meth:`modified`.
Args:
path: S3 path (s3://bucket/key).
Returns:
The last-modified time of the object.
"""
return self.modified(path)
[docs]
def modified(self, path: str | os.PathLike[str]) -> datetime:
"""Return the last-modified time of the path.
Args:
path: S3 path (s3://bucket/key).
Returns:
The ``last_modified`` field from :meth:`info`, which is None for
buckets and directories.
"""
info = self.info(path)
return cast(datetime, info.get("last_modified"))
[docs]
def invalidate_cache(self, path: str | os.PathLike[str] | None = None) -> None:
"""Remove the cached entries of the path and its parent paths.
The cached bucket listing is removed only by the root path (``""``,
``"/"`` or ``"s3://"``), not by the paths of buckets or keys.
A version-qualified path invalidates the lookup of the version, and
also the object path without the version, because deleting or
copying a version can change the current version of the object. It
also invalidates the keys that end in its query, in each spelling
that names the version, such as the keys ``a/x?versionId=v1`` and
``a/x?version_id=v1`` of ``bucket/a/x?versionId=v1``, which a listing
names as the paths. Only the path itself is parsed for a version: its
parent paths are directories, named as they are listed, such as
``bucket/a/dir?versionId=v1`` for the key ``a/dir?versionId=v1/x``.
A path that ends in slashes is invalidated both with and without
them, since lookups strip them while a key such as ``dir//`` is
listed in the directory ``bucket/dir/``.
Args:
path: The path to invalidate. If None, clear the whole cache.
"""
if path is None:
self.dircache.clear()
return
raw = stringify_path(path)
stripped = self._strip_protocol(raw)
if not stripped:
self._evict_cache("")
return
trailing = len(raw) - len(raw.rstrip("/"))
for spelling in (stripped, stripped + "/" * trailing) if trailing else (stripped,):
base, version_id = S3Path.split_version_id(spelling)
if not version_id:
# Also the directory of a trailing slash.
self._invalidate_name(spelling)
continue
# The lookup of the version, as _lookup_cache_key() keys it, the
# keys that end in the query in each spelling that names the
# version, and the object path without the version.
self._evict_lookup((base, _VERSION_CACHE_KEY, version_id))
for query in ("versionId", "versionID", "versionid", "version_id"):
self._invalidate_name(f"{base}?{query}={version_id}")
self._invalidate_name(base)
def _invalidate(self, path: S3Path) -> None:
"""Remove the cached entries of a path and of the directories above it.
A path with a version removes the lookup of that version (see
:meth:`_lookup_cache_key`), and then the entries of its key. The
names of the directories above it are cut at their last slash
without parsing them, so that a directory named with a version ID
query or an empty segment is invalidated as it is listed.
Args:
path: The path of the bucket or the object.
"""
if path.version_id:
self._evict_lookup(self._lookup_cache_key(path))
self._invalidate_name(path.name)
def _invalidate_name(self, name: str) -> None:
"""Remove the cached entries of a name and of the directories above it.
The names of the directories are cut at their last slash without
parsing them, so that a directory named with a version ID query or
an empty segment is invalidated as it is listed.
Args:
name: The name of the bucket, the directory or the object, without
the protocol.
"""
while name:
# _ls_prefix caches listings under (name, delimiter).
for cache_key in ((name, "/"), (name, "")):
self._evict_cache(cache_key)
self._evict_lookup(name)
name = name.rpartition("/")[0]
def _evict_lookup(self, key: str | tuple[str, str, str]) -> None:
"""Remove the cached lookups under a lookup cache key, with and without parameters.
Args:
key: The lookup cache key (see :meth:`_lookup_cache_key`).
"""
self._evict_cache(key)
self._evict_cache((key, _LOOKUPS_CACHE_KEY))
def _evict_cache(self, key: str | tuple[Any, ...]) -> None:
"""Remove a dircache entry if it exists.
``DirCache.pop()`` reads and then deletes the entry, so it raises
KeyError when another thread removes the same entry in between,
such as the request threads of ``rm()`` invalidating a shared parent
at once. A single ``del`` raises KeyError only when the entry is
already gone, which this ignores.
Args:
key: The dircache key, a path, a ``(path, delimiter)`` listing
key, or a ``(path, _LOOKUPS_CACHE_KEY)`` key.
"""
with contextlib.suppress(KeyError):
del self.dircache[key]
def _key_exists(self, path: S3Path, lookup_kwargs: Mapping[str, Any]) -> bool:
"""Check whether an object exists at the key of the path as written.
Unlike :meth:`_object_exists`, a key that ends in a slash, such as a
folder marker, is looked up as it is, for a write of that key.
Args:
path: The path of the object.
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``)
to send with the requests.
Returns:
True if an object exists at the key.
"""
if not (path.key or "").endswith("/"):
return self._object_exists(path, lookup_kwargs)
try:
self.core.head_object(path, **lookup_kwargs)
except FileNotFoundError:
return False
return True
def _object_exists(self, path: str | S3Path, lookup_kwargs: Mapping[str, Any]) -> bool:
"""Check whether an object exists at the path, for a write that must not replace one.
A key that is only a prefix of other keys has no object, as when
reading.
Args:
path: S3 path (s3://bucket/key).
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``)
to send with the requests.
Returns:
True if an object exists at the path.
"""
try:
info = self.info(path, **lookup_kwargs)
except FileNotFoundError:
return False
return bool(info.get("type") != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY)
@staticmethod
def _get_lookup_kwargs(kwargs: Mapping[str, Any]) -> dict[str, Any]:
"""Select the request parameters that lookups send.
Lookups (``info()`` and ``exists()``) send only the parameters on
which their authorization depends, which also select their cached
results. Other parameters that HeadObject accepts, such as
``IfMatch``, ``PartNumber`` or ``ResponseContentType``, are not sent,
because the result would depend on them.
Args:
kwargs: The parameters to select from.
Returns:
The lookup parameters.
"""
return {k: v for k, v in kwargs.items() if k in _LOOKUP_REQUEST_PARAMETERS}
def _get_cached_lookup(
self, path: str | tuple[str, str, str], lookup_kwargs: Mapping[str, Any]
) -> S3Object | None:
"""Get the cached HeadObject or HeadBucket result of a lookup.
A lookup without lookup parameters uses the entry under the path. A
lookup with them uses only the result of a lookup with the same
values, cached under ``(path, _LOOKUPS_CACHE_KEY)``, because the
authorization of the request depends on them. Each of these results
expires on its own after the ``listings_expiry_time`` of the
dircache.
Args:
path: The lookup cache key of the object (see
:meth:`_lookup_cache_key`), or the bucket name.
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``).
Returns:
The cached object, or None if there is none.
"""
if not lookup_kwargs:
return cast("S3Object | None", self.dircache.get(path))
lookups = self.dircache.get((path, _LOOKUPS_CACHE_KEY))
cached = lookups.get(self._get_lookup_cache_id(lookup_kwargs)) if lookups else None
if cached is None:
return None
cached_at, file = cached
expiry_time = self.dircache.listings_expiry_time
if expiry_time and time.time() - cached_at > expiry_time:
# Caching the result of any parameters renews the dircache entry
# of the path, so each result expires on its own.
return None
return cast(S3Object, file)
def _cache_lookup(
self, path: str | tuple[str, str, str], lookup_kwargs: Mapping[str, Any], file: S3Object
) -> None:
"""Cache the HeadObject or HeadBucket result of a lookup.
Args:
path: The lookup cache key of the object (see
:meth:`_lookup_cache_key`), or the bucket name.
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``)
that the request was made with.
file: The object to cache.
"""
if not lookup_kwargs:
self.dircache[path] = file
return
key = (path, _LOOKUPS_CACHE_KEY)
lookups = self.dircache.get(key)
if lookups is None:
lookups = {}
# Updated in place: a copy written back could replace a result that
# another thread has cached since for other parameters.
lookups[self._get_lookup_cache_id(lookup_kwargs)] = (time.time(), file)
# Set again so that the expiry time of the entry is renewed.
self.dircache[key] = lookups
@staticmethod
def _get_lookup_cache_id(lookup_kwargs: Mapping[str, Any]) -> tuple[tuple[str, Any], ...]:
"""Identify a set of lookup parameters in the cache.
Args:
lookup_kwargs: The lookup parameters (see ``_get_lookup_kwargs``).
Returns:
The sorted parameters, with ``SSECustomerKey`` replaced by its
SHA-256 digest so that the cache does not keep the key.
"""
return tuple(
sorted(
(
k,
hashlib.sha256(v if isinstance(v, bytes) else str(v).encode()).hexdigest()
if k == "SSECustomerKey"
else v,
)
for k, v in lookup_kwargs.items()
)
)
def _ls_from_cache(self, path: S3Path) -> list[S3Object] | S3Object | None:
"""Check the dircache for a cached entry of the path.
fsspec's implementation looks up listings under the path itself, but
S3FileSystem caches a single S3Object under the lookup cache key of
an object or a bucket (HeadObject/HeadBucket results, see
:meth:`_lookup_cache_key`), the bucket listing under ``""``, and the
other listings under ``(name, delimiter)`` (see ``_ls_prefix``).
Args:
path: The path of the bucket or the object.
Returns:
The cached entry of the path, the entries of a cached parent
listing named as the path, or None if no cached entry describes
the path. A listing of the path itself is not used, because it
cannot tell whether an object of the same name exists. A path
with a version uses only its own entry, because listings
describe the current versions.
Raises:
FileNotFoundError: If a cached listing of the parent directory of
a key path does not contain the path.
"""
cache = self.dircache.get(self._lookup_cache_key(path))
if cache is not None:
return cast("list[S3Object] | S3Object", cache)
if path.version_id:
return None
if path.key:
parent_cache = self.dircache.get((self._listing_name(path), "/"))
else:
parent_cache = self.dircache.get("")
if parent_cache is not None:
files = [f for f in parent_cache if f["name"] == path.name]
if files:
return files
if path.key:
raise FileNotFoundError(str(path))
# The bucket listing holds only the buckets that the caller owns,
# so a bucket missing from it is looked up with HeadBucket.
return None
def _open(
self,
path: str | S3Path,
mode: str = "rb",
block_size: int | None = None,
cache_type: str | None = None,
autocommit: bool = True,
cache_options: dict[Any, Any] | None = None,
**kwargs,
) -> S3File:
if block_size is None:
block_size = self.default_block_size
if cache_type is None:
cache_type = self.default_cache_type
max_workers = kwargs.pop("max_workers", None)
if max_workers is None:
# None takes the default of the filesystem.
max_workers = self.max_workers
# The parameters of the call take precedence over those of the
# filesystem; the caller's dictionary is not modified.
s3_additional_kwargs = {
**self.s3_additional_kwargs,
**kwargs.pop("s3_additional_kwargs", {}),
}
return S3File(
self,
path,
mode,
max_workers=max_workers,
executor=self._create_executor(max_workers=max_workers),
block_size=block_size,
cache_type=cache_type,
autocommit=autocommit,
cache_options=cache_options,
s3_additional_kwargs=s3_additional_kwargs,
**kwargs,
)
def _call(self, method: str | Callable[..., Any], **kwargs) -> dict[str, Any]:
"""Send a request with the core (see :meth:`S3Core.call`).
Args:
method: The name of the client method, or the method itself.
**kwargs: The request parameters.
Returns:
The response.
"""
return self._core.call(method, **kwargs)
# The parameters of the encryption of a written object, kept from an appended
# object as a whole.
_ENCRYPTION_PARAMS = frozenset(
[
"ServerSideEncryption",
"SSEKMSKeyId",
"SSEKMSEncryptionContext",
"BucketKeyEnabled",
"SSECustomerAlgorithm",
"SSECustomerKey",
"SSECustomerKeyMD5",
]
)
# The object lock of an appended object, which the rewritten object does not
# keep.
_OBJECT_LOCK_PARAMS = frozenset(
["ObjectLockMode", "ObjectLockRetainUntilDate", "ObjectLockLegalHoldStatus"]
)
[docs]
@dataclass
class S3FileTarget:
"""The object that a file reads or writes, resolved when it is opened.
In read mode, the object is looked up, and the reads are made conditional
on its ETag (``IfMatch``). In append mode, an existing object is looked
up; one smaller than ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` is also
read, so that the file rewrites it from its buffer, and a larger one is
copied by the file as the first parts of a multipart upload. In
exclusive-create mode, the object must not exist, and the upload is
committed with ``IfNoneMatch="*"``.
Attributes:
path: The path of the file. A version that is read is carried in the
path as a ``?versionId=`` suffix, so that a reopened file reads
the same version.
bucket: The bucket of the object.
key: The key of the object.
version_id: The version that is read, if any.
request_kwargs: The parameters of the S3 requests of the file, with
the conditions above and, in append mode, the attributes of the
existing object that the request parameters (those of the call
and of the filesystem) do not give, which the rewritten object
keeps: its system-defined and user-defined metadata, storage
class, and encryption, unless they give any encryption
parameter. Its object lock is not kept.
details: The object that is read or appended, or an empty dictionary.
size: The size of the object that is read, if known.
appended: The existing object in append mode, if any.
append_data: The data of an appended object that the file rewrites
from its buffer, if any.
"""
path: str
bucket: str
key: str
version_id: str | None
request_kwargs: dict[str, Any]
details: S3Object | dict[str, Any] = field(default_factory=dict)
size: int | None = None
appended: S3Object | None = None
append_data: bytes | None = None
[docs]
@classmethod
def resolve(
cls,
fs: S3FileSystem,
path: str | S3Path,
mode: str,
*,
block_size: int,
version_id: str | None = None,
size: int | None = None,
request_kwargs: Mapping[str, Any] | None = None,
) -> S3FileTarget:
"""Validate the arguments of a file and look up its object.
Sends the lookup requests of the mode, which block.
Args:
fs: The filesystem that the file belongs to.
path: S3 path (s3://bucket/key) of the file, or an ``S3Path``,
whose bucket, key and version are used as they are, such as
a key that a listing returns, and which is looked up by them.
The path of the file is then the string of the ``S3Path``.
mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``.
block_size: The block size of the file.
version_id: The version ID to read. Must match the version ID in
the path if both are given. A version cannot be given, in
either form, for writing or appending.
size: The size of the object to read, if known.
request_kwargs: The parameters of the S3 requests of the file. A
copy is kept.
Returns:
The resolved target.
Raises:
FileExistsError: If an object exists at the path in
exclusive-create mode.
FileNotFoundError: If no object exists at the path when reading,
including when the path is a prefix.
ValueError: If the path has no key, the version IDs do not match,
a version is given for writing, or the block size is not
between ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` and
``S3Core.MULTIPART_UPLOAD_MAX_PART_SIZE`` for writing.
"""
# A new dictionary, so that the caller's is not modified.
kwargs: dict[str, Any] = dict(request_kwargs or {})
# The path as it is looked up: a path string as a string, and an
# S3Path by its bucket, key and version.
target = path
s3_path = S3Path.of(path)
path = str(path)
path_version_id = s3_path.version_id
if not s3_path.key:
raise ValueError("The path does not contain a key.")
if version_id and path_version_id:
if version_id != path_version_id:
raise ValueError(
f"The version_id: {version_id} specified in the argument and "
f"the version_id: {path_version_id} specified in the path do not match."
)
elif path_version_id:
version_id = path_version_id
if version_id and "r" not in mode:
raise ValueError("Cannot write to the file with the version specified.")
if version_id and not path_version_id:
# Carry the version in the path, as with the ?versionId= suffix,
# so that a reopened (e.g., unpickled) file reads the same version.
path = f"{path}?versionId={version_id}"
target = s3_path.with_version_id(version_id) if isinstance(target, S3Path) else path
if "r" not in mode and not (
fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE
<= block_size
<= fs.core.MULTIPART_UPLOAD_MAX_PART_SIZE
):
# When writing, every full block is uploaded as a part of a
# multipart upload.
raise ValueError(
"Block size for writing must be between "
f"5 MiB ({fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE} bytes) and "
f"5 GiB ({fs.core.MULTIPART_UPLOAD_MAX_PART_SIZE} bytes), inclusive: {block_size}."
)
details: S3Object | dict[str, Any] = {}
appended: S3Object | None = None
append_data: bytes | None = None
# The lookups need the parameters of the file on which their
# authorization depends, such as the customer-provided key.
lookup_kwargs = fs._get_lookup_kwargs(kwargs)
if "r" in mode:
# Looked up here; the base class initializer of a file would
# otherwise take the size from the latest version of the object.
info = fs.info(target, version_id=version_id, **lookup_kwargs)
if info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY:
# A prefix has no object to read.
raise FileNotFoundError(path)
if fs.version_aware and not version_id:
# Pin the version observed at open time so that reads are
# consistent even if the object is overwritten. info() heads
# the object when the cached entry carries no version.
version_id = info.get("version_id")
if version_id:
# Carried in the path as an explicit version is, so that
# the methods of the file use it too.
path = f"{path}?versionId={version_id}"
if etag := info.get("etag"):
# A caller's condition is sent as given, so that a read of
# another object fails as S3 answers it (412).
kwargs.setdefault("IfMatch", etag)
details = info
if size is None:
size = info.get("size")
elif "a" in mode:
# The rewritten object keeps the metadata of the existing one,
# which a cached listing entry lacks, so look up the object.
with contextlib.suppress(FileNotFoundError):
appended = fs.info(target, refresh=True, **lookup_kwargs)
if (
appended is not None
and appended.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY
):
# A prefix has no object to append to, as when reading.
appended = None
if appended is not None:
etag = appended.get("etag")
if appended.get("size", 0) < fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE:
# Too small to be a part of a multipart upload: rewritten
# from the buffer. Only the lookup parameters, and the
# condition on the object looked up, are sent, so that
# the whole object is read, and an object replaced since
# is not.
condition = {"IfMatch": etag} if etag else {}
append_data = fs.cat_file(target, **condition, **lookup_kwargs)
# The attributes of the object looked up are kept where the
# caller gives none: the encryption as a whole, so that its
# fields are not combined with another encryption, and
# without the object lock, which the caller sets.
inherited = appended.to_api_repr()
if _ENCRYPTION_PARAMS.intersection(kwargs):
inherited = {k: v for k, v in inherited.items() if k not in _ENCRYPTION_PARAMS}
for k, v in inherited.items():
if k not in _OBJECT_LOCK_PARAMS:
kwargs.setdefault(k, v)
# The part copies read the object looked up, with the
# parameters that authorize the access to it under the names
# of the copy source; a caller's values are kept.
copy_source = fs.core.copy_from_source_params(lookup_kwargs)
if etag:
copy_source["CopySourceIfMatch"] = etag
for k, v in copy_source.items():
kwargs.setdefault(k, v)
details = appended
elif "x" in mode:
# Checked up front so that no data is uploaded for an existing
# object, and on commit with IfNoneMatch for one created since.
if fs._object_exists(target, lookup_kwargs):
raise FileExistsError(path)
kwargs.update({"IfNoneMatch": "*"})
return cls(
path=path,
bucket=s3_path.bucket,
key=s3_path.key,
version_id=version_id,
request_kwargs=kwargs,
details=details,
size=size,
appended=appended,
append_data=append_data,
)
[docs]
class S3File(AbstractBufferedFile):
"""A buffered file object for reading and writing an S3 object.
Instances are returned by ``S3FileSystem.open()``.
"""
fs: S3FileSystem
buffer: BytesIO | None
[docs]
def __init__(
self,
fs: S3FileSystem,
path: str | S3Path,
mode: str = "rb",
version_id: str | None = None,
max_workers: int | None = None,
executor: S3Executor | None = None,
block_size: int = S3FileSystem.DEFAULT_BLOCK_SIZE,
cache_type: str = "bytes",
autocommit: bool = True,
cache_options: dict[Any, Any] | None = None,
size: int | None = None,
s3_additional_kwargs: dict[str, Any] | None = None,
**kwargs,
) -> None:
"""Initialize the file for the path and mode.
In read mode, the object is looked up with ``info()`` and the reads
are made conditional on its ETag (``IfMatch``). In append mode, an
existing object smaller than ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` is
read into the write buffer; a larger one is copied with
``UploadPartCopy`` as the first parts of a multipart upload, whatever
the block size. In exclusive-create mode, the object must not exist
when the file is opened, and the upload is committed with
``IfNoneMatch="*"`` so that it does not replace an object created in
the meantime.
Args:
fs: The filesystem that the file belongs to.
path: S3 path (s3://bucket/key) of the file, or an ``S3Path``
(see :meth:`S3FileTarget.resolve`).
mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``.
version_id: The version ID to read. Must match the version ID in
the path if both are given. A version cannot be given, in
either form, for writing or appending.
max_workers: The number of parallel requests of the file: range
reads, part copies, and part uploads. A write keeps at most
``max_workers`` parts, including the part copies of an append,
in flight, waiting for one of them to finish before it submits
another. If None, the ``max_workers`` of the filesystem.
executor: The executor for parallel operations. If None, one on
the request pool of the filesystem (``fs._create_executor()``).
block_size: The block size for reads and writes. Must be between
``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` and
``S3Core.MULTIPART_UPLOAD_MAX_PART_SIZE``, inclusive, unless reading.
cache_type: The fsspec cache type for reads; see
``S3FileSystem.__init__()``.
autocommit: Whether to commit the written data when the file is
closed. If False, :meth:`commit` must be called.
cache_options: Options for the fsspec cache.
size: The size of the object, if known. Passed to
``fsspec.spec.AbstractBufferedFile``.
s3_additional_kwargs: Additional parameters for the S3 requests of
the file, such as ``ContentType`` or ``RequestPayer``. Each
request receives those that its operation accepts.
**kwargs: Additional parameters for the S3 requests of the file,
which take precedence over ``s3_additional_kwargs``.
Raises:
FileExistsError: If an object exists at the path in
exclusive-create mode.
FileNotFoundError: If no object exists at the path when reading,
including when the path is a prefix.
ValueError: If the path has no key, the version IDs do not match,
a version is given for writing, or the block size is not
between ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` and
``S3Core.MULTIPART_UPLOAD_MAX_PART_SIZE`` for writing, or
``max_workers`` is not positive.
"""
if max_workers is None:
max_workers = fs.max_workers
if max_workers <= 0:
raise ValueError("max_workers must be greater than 0")
# The process that opened the file; a forked child rejects it.
self._pid = os.getpid()
self.max_workers = max_workers
# The arguments are validated, and the objects looked up, before the
# base class initializer: a file that fails here is never opened, so
# its garbage collection does not close (flush and commit) it.
target = S3FileTarget.resolve(
fs,
path,
mode,
block_size=block_size,
version_id=version_id,
size=size,
request_kwargs={**(s3_additional_kwargs or {}), **kwargs},
)
self.bucket = target.bucket
self.key = target.key
self.version_id: str | None = target.version_id
self.s3_additional_kwargs: dict[str, Any] = target.request_kwargs
self._details: S3Object | dict[str, Any] = target.details
self._executor: S3Executor = executor or fs._create_executor(max_workers=max_workers)
# The requests and the state of the write; None when reading.
self._session: _S3WriteSession | None = None
if mode != "rb":
self._session = _S3WriteSession(
fs.core,
S3Path(target.bucket, target.key),
self._executor,
block_size=block_size,
max_workers=max_workers,
# The mapping itself, which the requests read when they are
# sent, as before; a mapping assigned later is not read.
request_kwargs=self.s3_additional_kwargs,
)
super().__init__(
fs=fs,
path=target.path,
mode=mode,
block_size=block_size,
autocommit=autocommit,
cache_type=cache_type,
cache_options=cache_options,
size=target.size,
# Kept as ``self.kwargs``, which fsspec passes to open() when it
# reopens the file (e.g., when it is unpickled): the parameters
# of the requests, including the condition on the object looked
# up, so that the reopened file reads the same object or fails.
**(target.request_kwargs if mode == "rb" else {}),
)
self.append_block = False
# The size of the appended object, as looked up when opening.
self._append_size = 0
if target.appended is not None:
if target.append_data is not None:
self.write(target.append_data)
else:
# Copied with UploadPartCopy as the leading part(s).
self.append_block = True
self._append_size = self.loc = target.appended.get("size", 0)
def _get_request_kwargs(self, method: str) -> dict[str, Any]:
"""Select the parameters of the file that an S3 operation accepts.
Args:
method: The name of the client method, such as ``get_object``.
Returns:
The parameters in ``s3_additional_kwargs`` that the operation
accepts.
"""
return self.fs.core.operation_params(method, self.s3_additional_kwargs)
[docs]
def close(self) -> None:
"""Close the file, flushing any written data, and shut down its executor.
In a forked child process, a file that the parent opened is only
marked closed: its data is neither flushed nor committed, and its
upload is not aborted, since they belong to the parent. Called by
another finalizer while the cyclic garbage collector runs in this
thread, such as that of a ``TextIOWrapper``, the file is closed in a
new thread once the collection has ended (see :meth:`__del__`), and
this returns before that. A close handed over to another thread is
waited for, except during a collection or in a thread of an
:class:`S3ThreadPool`, where this returns without waiting.
"""
if self._inherited():
self.closed = True
return
if not self.closed and S3File._in_collection():
self._close_later()
return
if self._wait_for_close_later():
self._close()
def _close(self) -> None:
"""Close the file in this thread, flushing any written data, and shut down its executor."""
try:
super().close()
finally:
# The executor is shut down even if the final flush fails.
self._executor.shutdown()
[docs]
def __del__(self) -> None:
"""Close the file when it is finalized without having been closed.
A file whose last reference is dropped in a user thread is closed
here. A file that the cyclic garbage collector finalizes, or that is
finalized in a thread of an :class:`S3ThreadPool`, is closed in a new
thread instead, since a wait here could last forever (T1): the
requests of the file may be queued behind this thread or include the
one that runs the collection, and this thread may hold a lock that
the close takes. During a collection, the thread starts once the
collection has ended, after the other finalizers of the collection,
such as that of a ``TextIOWrapper``, have written to the file.
"""
if getattr(self, "closed", True) or self._inherited():
# The upload of a file inherited across a fork is the parent's.
return
if S3ThreadPool.in_pool_thread():
self._close_later()
else:
# Hands the close over itself during a collection.
self.close()
@staticmethod
def _track_collection(phase: str, info: dict[str, int]) -> None:
"""Record whether the cyclic garbage collector runs in this thread.
Registered in ``gc.callbacks``, which the collector calls in the
thread that runs it. When the collection stops, the closes that it
handed over start.
Args:
phase: ``"start"`` or ``"stop"``.
info: The details of the collection, which are not used.
"""
_collecting.active = phase == "start"
pending = getattr(_collecting, "pending", None)
if not _collecting.active and pending:
_collecting.pending = []
for file, complete in pending:
file._start_close(complete)
@staticmethod
def _in_collection() -> bool:
"""Whether the cyclic garbage collector runs in this thread.
Returns:
True while a collection runs in the calling thread.
"""
return getattr(_collecting, "active", False)
def _close_later(self) -> None:
"""Close the file in a new thread, once.
The written data is copied first: the collector finalizes every
object that it found unreachable before it checks whether a
finalizer has made it reachable again, and finalizing the buffer
closes it. If the buffer is already closed, the data is incomplete,
so the write is discarded instead of committed. During a collection,
the thread starts once the collection has ended, so that the other
finalizers of the collection can still write to the file.
"""
if getattr(self, "_closing_later", False):
return
self._closing_later = True
self._closed_later = threading.Event()
complete = True
if self.mode != "rb" and self.buffer is not None:
if self.buffer.closed:
complete = False
else:
buffer = BytesIO(self.buffer.getvalue())
buffer.seek(self.buffer.tell())
self.buffer = buffer
_handed_over_closes.add(self._closed_later)
if S3File._in_collection():
if not hasattr(_collecting, "pending"):
_collecting.pending = []
_collecting.pending.append((self, complete))
else:
self._start_close(complete)
def _start_close(self, complete: bool) -> None:
"""Start the close that :meth:`_close_later` handed over.
The file is closed in a thread of the ``_thread`` module, since
``Thread.start()`` takes locks of the ``threading`` module, which this
thread, in which a finalizer or the collector runs, may hold, and
waits for the new thread, which may take another. The interpreter
does not join such a thread, so an exit waits for the close instead
(see :meth:`_wait_for_handed_over_closes`). The thread runs with an
empty context, so that the stop scopes of the finalizing thread do
not stop the requests of the file. During interpreter finalization,
and once the exit has waited for the handed-over closes, the file is
closed in this thread without starting one, since no thread started
during finalization runs, and none started afterwards is waited for.
No other thread runs requests then, and the pools take no more
calls, so the close does not wait for a part queued behind it. If no
thread can be started otherwise, the file is left unclosed rather
than closed in this thread, where it could wait forever, until a
later :meth:`close`, :meth:`commit`, or :meth:`discard` closes it
(see :meth:`_wait_for_close_later`).
Args:
complete: Whether the written data is complete.
"""
if self._inherited():
# Handed over in the parent, which forked before the collection
# ended. Its event, whose lock the fork may have copied held, is
# left alone.
self.closed = True
return
if sys.is_finalizing() or _handed_over_closes_waited:
self._close_collected(complete)
return
try:
_thread.start_new_thread(contextvars.Context().run, (self._close_collected, complete))
except Exception:
# Such as RuntimeError, when no more threads can be started.
_logger.warning(
f"s3://{self.bucket}/{self.key} was not closed, and no thread could be "
"started to close it when it was finalized; its data was not committed."
)
self._unstarted_close: bool | None = complete
_handed_over_closes.discard(self._closed_later)
# Wakes the callers that wait for it, which then close the file.
self._closed_later.set()
def _close_collected(self, complete: bool) -> None:
"""Close a finalized file, logging a failure instead of raising it.
Args:
complete: Whether the written data is complete. If not, the
write is discarded instead of committed.
"""
self._closer_ident = threading.get_ident()
try:
if complete:
self._close()
else:
_logger.warning(
f"s3://{self.bucket}/{self.key} was not closed, and its buffer was "
"finalized before it; its data was not committed."
)
self._close_without_commit()
if getattr(self, "_discard_later", False):
self.discard()
except Exception:
_logger.exception(f"Failed to close s3://{self.bucket}/{self.key}.")
finally:
self._end_close_later()
def _end_close_later(self) -> None:
"""Mark the close that :meth:`_close_later` handed over as ended."""
_handed_over_closes.discard(self._closed_later)
self._closed_later.set()
@staticmethod
def _wait_for_handed_over_closes() -> None:
"""Wait for the closes that run in threads of the ``_thread`` module.
Registered with :mod:`atexit`, since the interpreter does not join
these threads when it exits. Also waits for the closes that are
handed over while it waits; later ones, such as in an exit handler
that runs after it, are closed in the finalizing thread.
"""
global _handed_over_closes_waited
while _handed_over_closes:
for closed in list(_handed_over_closes):
closed.wait()
_handed_over_closes_waited = True
def _wait_for_close_later(self) -> bool:
"""Wait for a close that :meth:`_close_later` handed over to end.
It is not waited for in the thread that runs it, during a
collection, whose end starts it, or in a thread of an
:class:`S3ThreadPool`, since it may wait for a request queued behind
that thread. A close for which no thread could be started is run
here instead, where it can be waited for.
Returns:
False if a handed-over close has not ended and cannot be waited
for in this thread; otherwise True.
"""
closed_later = getattr(self, "_closed_later", None)
if closed_later is None or getattr(self, "_closer_ident", None) == threading.get_ident():
return True
unsafe = S3File._in_collection() or S3ThreadPool.in_pool_thread()
if not closed_later.is_set():
if unsafe:
return False
closed_later.wait()
if getattr(self, "_unstarted_close", None) is not None:
if unsafe:
return False
unstarted, self._unstarted_close = self._unstarted_close, None
self._close_collected(cast(bool, unstarted))
return True
def _raise_unless_closed_later(self) -> None:
"""Wait for a handed-over close to end, or raise if it cannot be waited for here.
Raises:
RuntimeError: If a close of the file was handed over to another
thread and runs, and this is called during a collection or
in a thread of an :class:`S3ThreadPool`.
"""
if not self._wait_for_close_later():
raise RuntimeError(
f"s3://{self.bucket}/{self.key} is being closed in another thread, which "
"cannot be waited for during a garbage collection or in a thread of an "
"S3ThreadPool."
)
[docs]
def read(self, length: int = -1) -> bytes:
"""Read bytes from the file.
Args:
length: The number of bytes to read; -1 reads to the end.
Returns:
The bytes read.
Raises:
RuntimeError: In a forked child process of the process that
opened the file.
ValueError: If the file is closed or not open for reading.
"""
self._check_process()
data: bytes = super().read(length)
return data
[docs]
def flush(self, force: bool = False) -> None:
"""Upload the buffered data as a part once a block is full, or all of it with ``force``.
Args:
force: Whether to upload the remaining data and commit the file,
after which it cannot be written.
Raises:
RuntimeError: In a forked child process of the process that
opened the file.
ValueError: If ``force`` is given again after a forced flush.
"""
self._check_process()
super().flush(force=force)
def _inherited(self) -> bool:
"""Whether this process is a forked child of the process that opened the file.
Returns:
True in a forked child process of the process that opened the
file.
"""
# Compared with the process ID, not a count that an at-fork hook
# updates, since hooks registered before it may close the file.
return getattr(self, "_pid", os.getpid()) != os.getpid()
def _check_process(self) -> None:
"""Reject the use of a file that the parent of a forked process opened.
Raises:
RuntimeError: In a forked child process of the process that
opened the file.
"""
if self._inherited():
raise RuntimeError(
f"s3://{self.bucket}/{self.key} was opened by the parent of this forked "
"process and cannot be used here; open it again in this process."
)
[docs]
def write(self, data: bytes | bytearray | memoryview) -> int:
"""Write data to the buffer, uploading each full block as a part.
Args:
data: The bytes to write.
Returns:
The number of bytes written.
Raises:
BaseException: The error of a request of the write that failed
since the last write, after which the file is closed without
committing it.
RuntimeError: In a forked child process of the process that
opened the file.
S3MultipartStateError: If the write was discarded.
ValueError: If the file is not open for writing.
"""
self._check_process()
if self._session is not None and not self.closed:
try:
self._session.check()
except BaseException:
self._close_without_commit()
raise
written: int = super().write(data)
return written
def _close_without_commit(self) -> None:
"""Close the file without uploading the written data.
Drops the buffered data, and discards the write (see
:meth:`discard`), so that neither close() nor a deferred commit()
uploads the data. An abort failure is logged instead of raised, so it
does not mask the error that the caller is handling; the upload is
then kept, so that :meth:`discard` can abort it. The executor is shut
down here, as fsspec does not close a closed file again when it is
garbage collected.
"""
if self.buffer is None and getattr(self, "closed", False):
return
self.buffer = None
self.closed = True
try:
if self._wait_for_close_later():
self.discard()
else:
# The handed-over close, which cannot be waited for here,
# discards the write once it ends.
self._discard_later = True
except Exception:
upload_id = cast(S3MultipartUpload, self.multipart_upload).upload_id
_logger.exception(
f"Failed to abort multipart upload {upload_id} to s3://{self.bucket}/{self.key}."
)
finally:
self._executor.shutdown()
def _write_and_close(self, value: bytes | bytearray | memoryview) -> None:
"""Write the whole value and close the file.
Unlike a ``with`` block, a failed write closes the file without
committing it, so the existing object is left unchanged.
Args:
value: The bytes to write.
"""
try:
if isinstance(value, memoryview) and not value.c_contiguous:
# The buffer of the file cannot write a non-contiguous memoryview.
value = value.tobytes()
self.write(value)
except BaseException:
self._close_without_commit()
raise
self.close()
def _write_file_and_close(self, local: BinaryIO, callback: Callback) -> None:
"""Write the rest of a local file and close the file.
Unlike a ``with`` block, a failed read, write, or progress update
closes the file without committing it, so the existing object is
left unchanged.
Args:
local: The local file to read from.
callback: Progress callback, updated with the size of each block.
"""
try:
while data := local.read(self.blocksize):
self.write(data)
callback.relative_update(len(data))
except BaseException:
self._close_without_commit()
raise
self.close()
@property
def multipart_upload(self) -> S3MultipartUpload | None:
"""The multipart upload of the write, from its creation until it is aborted."""
return self._session.upload if self._session is not None else None
@property
def completed_parts(self) -> tuple[S3MultipartUploadPart, ...]:
"""The uploaded or copied parts in part-number order; empty after an abort."""
return self._session.completed_parts if self._session is not None else ()
def _initiate_upload(self) -> None:
if not self.append_block and self.tell() < self.blocksize:
return
session = cast(_S3WriteSession, self._session)
try:
session.start()
session.wait_ready()
# Raises the failure of the creation.
session.check()
if self.append_block:
# The size of the object looked up when opening, which the
# part copies are conditional on.
size = self._append_size
whole_source = size <= self.fs.core.MULTIPART_UPLOAD_MAX_PART_SIZE
source = S3Path(self.bucket, self.key)
for part_number, range_ in session.iter_copy_parts(size):
# The part copies count toward max_workers, as the parts do.
session.wait_ready()
session.submit_copy(part_number, source, None if whole_source else range_)
except BaseException:
# A creation that has started may still create an upload, which
# the abort waits for.
self._close_without_commit()
raise
def _upload_chunk(self, final: bool = False) -> bool:
"""Upload the buffered data as parts of the multipart upload.
A file smaller than the block size, unless an append copied the
existing object as parts, is left in the buffer for :meth:`commit`,
which uploads it with PutObject. Otherwise, the buffered data is
uploaded as parts, and a part is submitted only while fewer than
``max_workers`` parts, including the part copies of an append, are in
flight; the write waits for one of them to finish first. A request
found failed before a part is submitted stops the write with its
error; one that fails afterwards is reported by the next write or by
:meth:`commit`. If the upload of the chunk fails or is interrupted,
the file is closed without committing it.
Args:
final: Whether this is the last flush of the file, after which
the upload is committed if ``autocommit`` is set.
Returns:
Whether fsspec's ``flush()`` resets the buffer afterwards, which
it does for any value other than False.
Raises:
BaseException: The error of a request of the write that failed
before a part is submitted.
ValueError: If the data needs more parts than a multipart upload
allows.
"""
# Returning ``not final`` keeps the buffer intact on the final flush so a
# deferred commit() (autocommit=False, i.e. inside an fsspec transaction)
# can still read the bytes; resetting it there would upload an empty
# object for small files. Mid-stream chunks (final=False) return True so
# fsspec clears the already-uploaded buffer between parts.
if not self.append_block and self.tell() < self.blocksize:
# Files smaller than block size in size cannot be multipart uploaded.
if self.autocommit and final:
self.commit()
return not final
session = cast(_S3WriteSession, self._session)
# fsspec's flush() never calls this on a closed file, whose buffer
# may have been dropped.
buffer = cast(BytesIO, self.buffer)
buffer.seek(0)
try:
for part_number, data in session.iter_parts(buffer):
# Otherwise the parts waiting for a worker pile up in memory.
session.wait_ready()
session.submit_part(part_number, data)
except BaseException:
self._close_without_commit()
raise
if self.autocommit and final:
self.commit()
return not final
[docs]
def commit(self) -> None:
"""Commit the written data.
Creates an empty object if nothing was written, uploads the buffered
data with PutObject if no multipart upload was started, and otherwise
completes the multipart upload once its parts have been uploaded. A
failed part is raised as soon as it fails, without waiting for the
parts before it. If a part or the completion fails or is interrupted,
the file is closed without committing it, and its upload is aborted;
if the abort also fails, the upload is kept so that :meth:`discard`
can abort it. Committing again after a successful commit sends no
request. Invalidates the cache of the path afterwards. Does nothing
for a file opened for reading. Waits first for a close of the file
that was handed over to another thread (see :meth:`close`).
Raises:
FileExistsError: If an object was created at the path after the
file was opened in exclusive-create mode.
RuntimeError: In a forked child process of the process that
opened the file, or if a close of the file that was handed
over to another thread runs and this is called during a
garbage collection or in a thread of an :class:`S3ThreadPool`.
S3MultipartStateError: If the write was discarded, or the write
or an earlier commit failed.
"""
self._check_process()
self._raise_unless_closed_later()
if self._session is None:
# A file opened for reading has nothing to commit.
return
session = self._session
if not session.multipart:
body = None
if self.buffer is not None and self.tell():
self.buffer.seek(0)
body = self.buffer.read()
# None writes an empty object.
session.put(body)
else:
try:
session.complete()
except BaseException:
# Closed without the buffer, which holds only the data of the
# last part.
self._close_without_commit()
raise
self.fs._invalidate(S3Path(self.bucket, self.key))
[docs]
def discard(self) -> None:
"""Discard the written data, and abort the multipart upload, if any.
The part requests that have not started are cancelled, and the
running ones make no further attempt and are waited for before the
abort, whose own retries are not stopped. If the abort fails or is
interrupted, the upload is kept, and calling this again retries the
abort. Afterwards, :meth:`commit` raises
:class:`~pyathena.filesystem.s3_writer.S3MultipartStateError`. Does
nothing for a committed file or a file opened for reading. Waits
first for a close of the file that was handed over to another thread
(see :meth:`close`).
Raises:
RuntimeError: In a forked child process of the process that
opened the file, or if a close of the file that was handed
over to another thread runs and this is called during a
garbage collection or in a thread of an :class:`S3ThreadPool`.
"""
self._check_process()
self._raise_unless_closed_later()
if self._session is not None:
self._session.abort()
def _lookup_kwargs(self) -> dict[str, Any]:
"""Select the parameters of the file on which the access to its object depends.
Returns:
The lookup parameters of the file (see
``S3FileSystem._get_lookup_kwargs``), such as ``RequestPayer`` or
the ``SSECustomer*`` parameters of an object encrypted with a
customer-provided key.
"""
return self.fs._get_lookup_kwargs(self.s3_additional_kwargs)
[docs]
def url(self, expiration: int = 3600, **kwargs) -> str:
"""Generate a presigned HTTP URL to read this file (if it already exists).
The URL is signed with the lookup parameters of the file that the
signed operation accepts, such as ``RequestPayer`` or the
``SSECustomer*`` parameters, whose headers the request with the URL
must then send.
Args:
expiration: URL expiration time in seconds. Defaults to 3600 (1 hour).
**kwargs: Additional parameters passed to :meth:`S3FileSystem.sign`,
which take precedence over those of the file.
Returns:
Presigned URL string that provides temporary access to the S3 object.
"""
# Only the parameters of the file that the signed operation accepts.
params = self.fs.core.operation_params(
kwargs.get("client_method", "get_object"), self._lookup_kwargs()
)
return cast(str, self.fs.sign(self.path, expiration=expiration, **{**params, **kwargs}))
[docs]
def getxattr(self, xattr_name: str, **kwargs) -> str | None:
"""Get an attribute from the user-defined metadata of the file.
See :meth:`S3FileSystem.getxattr`, and :meth:`metadata` for the
parameters of the request.
Args:
xattr_name: The name of the attribute.
**kwargs: Additional parameters passed to the HeadObject API,
which take precedence over those of the file.
Returns:
The value of the attribute, or None if the attribute is not set.
"""
return self.fs.getxattr(self.path, xattr_name, **{**self._lookup_kwargs(), **kwargs})
[docs]
def setxattr(self, copy_kwargs: dict[str, Any] | None = None, **kwargs) -> None:
"""Set the user-defined metadata of the file.
See :meth:`S3FileSystem.setxattr`. The copy of the object onto itself
receives the lookup parameters of the file both for its source and
for its destination: ``RequestPayer``, the expected bucket owner as
``ExpectedSourceBucketOwner`` and ``ExpectedBucketOwner``, and the
``SSECustomer*`` parameters also as ``CopySourceSSECustomer*``, so
that an object encrypted with a customer-provided key keeps it. A
``ServerSideEncryption`` in ``copy_kwargs`` replaces the
``SSECustomer*`` parameters of the file for the destination; those
in the ``s3_additional_kwargs`` of the filesystem are still sent.
Args:
copy_kwargs: Additional parameters to use for the underlying
CopyObject API call, which take precedence over those of the
file.
**kwargs: Key-value pairs of metadata to set.
"""
if self.writable():
raise NotImplementedError("Cannot update metadata while the file is open for writing.")
copy_kwargs = copy_kwargs or {}
lookup_kwargs = self._lookup_kwargs()
destination = lookup_kwargs
if "ServerSideEncryption" in copy_kwargs:
# Another encryption of the copy replaces the customer-provided key.
destination = {
k: v for k, v in lookup_kwargs.items() if not k.startswith("SSECustomer")
}
self.fs.setxattr(
self.path,
copy_kwargs={
**destination,
**self.fs.core.copy_from_source_params(lookup_kwargs),
**copy_kwargs,
},
**kwargs,
)
def _fetch_range(self, start: int, end: int) -> bytes:
"""Read a byte range of the object for the fsspec cache.
The range is clamped to the size of the object, since fsspec caches
may request a range that is empty or reaches past the end of the
object. S3 would answer the former with the whole object and a range
starting past the end with an ``InvalidRange`` error. A range larger
than the block size is read with parallel requests (see
:meth:`S3Executor.map() <pyathena.filesystem.s3_executor.S3Executor.map>`);
when one fails, those that have not started are cancelled, the running
ones make no further attempt, and its error is raised once they have
ended. Once an enclosing stop scope is stopped, such as that of a
batch operation after another object failed, no further request is
sent (see :func:`~pyathena.util.raise_if_stopped`).
Args:
start: The offset of the first byte to read.
end: The offset to stop reading at (exclusive).
Returns:
The bytes read, empty if the clamped range is empty.
Raises:
concurrent.futures.CancelledError: If an enclosing stop scope has
been stopped.
"""
end = min(end, self.size)
if start >= end:
return b""
ranges = self._get_ranges(
start, end, max_workers=self.max_workers, worker_block_size=self.blocksize
)
path = S3Path(self.bucket, self.key, self.version_id)
request_kwargs = self._get_request_kwargs("get_object")
if len(ranges) > 1:
# Not kept by a variable of this frame, which the traceback of an
# interrupt keeps (F5).
return b"".join(
cast(
list[bytes],
self._executor.map(
functools.partial(self._get_range, path, **request_kwargs),
ranges,
).outcomes,
)
)
return self._get_range(path, ranges[0], **request_kwargs)
def _get_range(self, path: S3Path, range_: tuple[int, int], **kwargs: Any) -> bytes:
"""Read a byte range with GetObject unless the read has been stopped.
A range that waited behind other requests checks the stop scopes of
its context when it starts, since the retry stop does not cover the
first attempt.
Args:
path: The object, with its version ID, if any.
range_: The range, with an exclusive end.
**kwargs: The GetObject parameters.
Returns:
The bytes of the range.
Raises:
concurrent.futures.CancelledError: If an enclosing stop scope has
been stopped.
"""
raise_if_stopped(f"read {path.uri}")
return self.fs.core.get_object(path, range_, **kwargs)
def _download(self, outfile: Any, callback: Callback) -> None:
"""Write the whole object into a writable file with parallel range reads.
The object is read in waves of ranges of the block size, or in one
range if the block size is not positive, one wave of at most
``max_workers`` ranges at a time, but no more than the request pool
of the filesystem runs at once, since the others would only wait
with their bodies held. Each wave is read with :meth:`S3Executor.map()
<pyathena.filesystem.s3_executor.S3Executor.map>`, and its bodies are
written in order, each dropped once written, before the next wave is
read, so that the download holds at most one wave of bodies; a thread
of the pool that has just returned a body drops it when its call
returns. The reads carry the version and the conditions of the file,
and each checks the stop scopes when it starts; a body read when a
stop came is not written.
A failure or an interrupt while a wave is read is raised as
:meth:`S3Executor.map()` raises it. One while its bodies are written,
including a failure of the write or of the callback, drops the bodies
first, also from the completed frames of its traceback (see
:meth:`S3MapResult.release()
<pyathena.filesystem.s3_executor.S3MapResult.release>`).
Args:
outfile: The writable file-like object.
callback: Progress callback, updated with the size of each body
once it is written.
Raises:
concurrent.futures.CancelledError: If an enclosing stop scope has
been stopped.
"""
size = cast(int, self.size)
# A block size that is not positive reads the rest of the object,
# as read() does.
block = self.blocksize if self.blocksize > 0 else max(size, 1)
window = min(self.max_workers, self.fs._request_pool.max_workers)
path = S3Path(self.bucket, self.key, self.version_id)
read = functools.partial(self._get_range, path, **self._get_request_kwargs("get_object"))
wave = window * block
result: S3MapResult[bytes] | None = None
try:
for wave_start in range(0, size, wave):
wave_end = min(size, wave_start + wave)
# Assigned within the try block, so that an interrupt right
# after the wave was read still drops its bodies.
result = self._executor.map(
read,
[
(start, min(start + block, wave_end))
for start in range(wave_start, wave_end, block)
],
)
bodies = result.outcomes
for i in range(len(bodies)):
# A body read when the stop came is not written.
raise_if_stopped(f"write the contents of {self.path}")
length = len(cast(bytes, bodies[i]))
outfile.write(bodies[i])
# Not kept while the next one is written or read.
bodies[i] = b""
callback.relative_update(length)
except BaseException as e:
if result is not None:
result.release(e)
raise
@staticmethod
def _get_ranges(
start: int, end: int, max_workers: int, worker_block_size: int
) -> list[tuple[int, int]]:
ranges = []
range_size = end - start
if max_workers > 1 and range_size > worker_block_size:
range_start = start
while True:
range_end = range_start + worker_block_size
if range_end >= end:
# Also when the size is an exact multiple of the block
# size, so that no empty trailing range is generated.
ranges.append((range_start, end))
break
ranges.append((range_start, range_end))
range_start += worker_block_size
else:
ranges.append((start, end))
return ranges
gc.callbacks.append(S3File._track_collection)
atexit.register(S3File._wait_for_handed_over_closes)
if hasattr(os, "register_at_fork"):
# The closes of the parent do not run in a child.
os.register_at_fork(after_in_child=_handed_over_closes.clear)