Source code for pyathena.filesystem.s3_async

# Copyright 2022 The PyAthena authors
#
# Licensed under the MIT License.
# See LICENSE or https://opensource.org/licenses/MIT.
#
# SPDX-License-Identifier: MIT

"""Asynchronous fsspec filesystem for Amazon S3 built on ``S3FileSystem``."""

from __future__ import annotations

import asyncio
import contextlib
import contextvars
import functools
import logging
import mimetypes
import os
import threading
import warnings
from collections.abc import AsyncIterator, Awaitable, Callable, Iterable
from concurrent.futures import CancelledError, Future, wait
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, cast

from fsspec.asyn import (
    AbstractAsyncStreamedFile,
    AsyncFileSystem,
    _run_coros_in_chunks,
    is_exception,
    sync,
)
from fsspec.callbacks import _DEFAULT_CALLBACK
from fsspec.core import get_compression
from fsspec.implementations.local import make_path_posix
from fsspec.utils import stringify_path

from pyathena.aio.util import create_in_thread
from pyathena.filesystem.s3 import (
    _UNPICKLABLE_CLIENT_MESSAGE,
    CompressedBuffer,
    S3File,
    S3FileSystem,
    S3FileTarget,
)
from pyathena.filesystem.s3_core import S3Core
from pyathena.filesystem.s3_executor import ItemT, S3Executor, S3MapResult, S3ThreadPool, T
from pyathena.filesystem.s3_object import (
    S3CompleteMultipartUpload,
    S3Metadata,
    S3MultipartUpload,
    S3Object,
    S3ObjectType,
    S3ObjectVersion,
)
from pyathena.filesystem.s3_path import S3Path
from pyathena.filesystem.s3_path_pairing import S3PathPairing
from pyathena.filesystem.s3_writer import _S3WriteSession, _S3WriteState
from pyathena.util import override, retries_stopped

if TYPE_CHECKING:
    from datetime import datetime

    from botocore.client import BaseClient

    from pyathena.connection import Connection
    from pyathena.util import RetryConfig

_logger = logging.getLogger(__name__)


class _S3BatchTasks:
    """The tasks of the per-object calls of a batch coroutine of fsspec.

    fsspec runs the per-object coroutines of its batch coroutines, such as
    ``_cat()`` and ``_put()``, as tasks, and a cancellation of the batch
    coroutine leaves them running. Each per-object coroutine of
    ``AioS3FileSystem`` adds its task with :meth:`track` to the batch that
    runs in its context, which cancels them and waits for them when the
    batch coroutine fails or is cancelled.
    """

    _current: ClassVar[contextvars.ContextVar[_S3BatchTasks | None]] = contextvars.ContextVar(
        "_S3BatchTasks", default=None
    )

    def __init__(self) -> None:
        """Initialize the batch with no task."""
        self._tasks: set[asyncio.Task[Any]] = set()
        self._owner: asyncio.Task[Any] | None = None

    @classmethod
    def track(cls) -> None:
        """Add the current task to the batch that runs in its context, if any.

        A task that the batch coroutine itself runs in is not added.
        """
        batch = cls._current.get()
        task = asyncio.current_task()
        if batch is not None and task is not None and task is not batch._owner:
            batch._tasks.add(task)
            task.add_done_callback(batch._tasks.discard)

    async def run(self, batch: Awaitable[T]) -> T:
        """Await a batch coroutine, and on its failure or cancellation wait for its tasks.

        A task that is already being cancelled, as the shutdown of the event
        loop with ``asyncio.run()`` cancels every task, is not cancelled
        again, so that the wait for its own calls is not stopped. A repeated
        cancellation stops the wait for the tasks.

        Args:
            batch: The batch coroutine, created and not yet started.

        Returns:
            The return value of the batch coroutine.
        """
        self._owner = asyncio.current_task()
        token = self._current.set(self)
        try:
            return await batch
        except BaseException:
            tasks = [task for task in self._tasks if not task.done()]
            for task in tasks:
                if not task.cancelling():
                    task.cancel()
            if tasks:
                await asyncio.wait(tasks)
            raise
        finally:
            self._current.reset(token)


class _S3TransactionWrite:
    """A write of a file of an ``AioS3FileSystem`` transaction, which a cancellation discards.

    The write runs with :meth:`run` in a thread of the filesystem, through
    :meth:`S3Executor.run() <pyathena.filesystem.s3_executor.S3Executor.run>`,
    whose cancellation stops the stop scope of the write, and opens its file
    with :meth:`open`, which adds the file to the transaction unless the
    write has been stopped. A write that ends stopped drops its file from
    the transaction and discards it, so that the wait of the cancelled
    coroutine for the write includes the discard. The coroutine calls
    :meth:`cancel` once the cancellation has been raised, which drops the
    file from the transaction also when the write had ended before it was
    stopped, and then discards it.

    Args:
        fs: The filesystem, which is in a transaction.
        write: The write, which opens its file with the callable that it
            receives, which takes the arguments of ``open()``.
    """

    def __init__(self, fs: AioS3FileSystem, write: Callable[[Callable[..., S3File]], None]) -> None:
        """Initialize the write.

        Args:
            fs: The filesystem, which is in a transaction.
            write: The write, which opens its file with the callable that it
                receives, which takes the arguments of ``open()``.
        """
        self._fs = fs
        self._transaction = fs.transaction
        self._write = write
        self._lock = threading.Lock()
        self._files: list[S3File] = []
        self._cancelled = False
        self._ended = False

    def run(self) -> None:
        """Run the write, and drop and discard its file if it ends stopped.

        Raises:
            concurrent.futures.CancelledError: If the write was stopped, also
                when the write itself succeeded, since its file is discarded.
        """
        try:
            self._write(self.open)
        finally:
            with self._lock:
                self._ended = True
                stopped = self._cancelled or retries_stopped()
                if stopped:
                    self._cancelled = True
                    # Also here, for a write that a scope other than the
                    # cancellation of the coroutine has stopped.
                    self._detach_all()
            if stopped:
                self.discard()
        if stopped:
            raise CancelledError("Stopped before the transaction could commit the write.")

    def open(self, path: str, mode: str, **kwargs) -> S3File:
        """Open the file of the write, which joins the transaction unless the write is stopped.

        Args:
            path: S3 path (s3://bucket/key) of the file.
            mode: The binary write mode of the file.
            **kwargs: The other arguments of ``open()``, except
                ``compression``. With ``autocommit=True``, as with
                ``open()``, the file commits when it is closed and does not
                join the transaction.

        Returns:
            The file.
        """
        autocommit = kwargs.pop("autocommit", False)
        file = self._fs._open(self._fs._strip_protocol(path), mode, autocommit=autocommit, **kwargs)
        with self._lock:
            self._files.append(file)
            # Under the lock, so that a cancellation either finds the file
            # in the transaction or keeps it out.
            if not autocommit and not self._cancelled and not retries_stopped():
                self._transaction.files.append(file)
        return file

    def cancel(self) -> bool:
        """Drop the file of the write from the transaction.

        Returns:
            Whether the write ended before it was stopped, in which case
            the caller discards the file with :meth:`discard`; otherwise the
            thread of the write discards it when the write ends, or has.
        """
        with self._lock:
            ended_unstopped = self._ended and not self._cancelled
            self._cancelled = True
            self._detach_all()
            return ended_unstopped

    def discard(self) -> None:
        """Discard the file of the write.

        A failed discard is logged instead of raised, so that the
        cancellation is raised.
        """
        with self._lock:
            files = list(self._files)
        for file in files:
            try:
                file.discard()
            except Exception:
                _logger.exception(f"Failed to discard the cancelled write to s3://{file.path}.")

    def _detach_all(self) -> None:
        """Drop the files of the write from the transaction; called under the lock."""
        for file in self._files:
            while True:
                try:
                    self._transaction.files.remove(file)
                except ValueError:
                    break
                except (IndexError, RuntimeError):
                    # Another write changed the transaction during the search,
                    # which compares the files with their __eq__().
                    continue
                break


[docs] class AioS3FileSystem(AsyncFileSystem): """An async filesystem interface for Amazon S3 using fsspec's AsyncFileSystem. This class wraps ``S3FileSystem`` to provide native asyncio support. Its coroutines run the blocking boto3 calls in an ``S3ThreadPool`` with ``max_workers`` threads that the filesystem owns, instead of the default executor of the event loop, and run parallel per-object calls, such as the copies of ``_mv()``, with ``asyncio.gather``. The single requests that these calls run in parallel, such as the parts of a file or a copy, and the DeleteObjects batches of ``_rm()``, run in the request pool of the wrapped ``S3FileSystem``, which its files share, through :meth:`~pyathena.filesystem.s3_executor.S3Executor.map_async` or a write session. The implementation uses composition: an internal ``S3FileSystem`` instance handles all boto3 calls, while this class delegates to it in the threads of its pool. This avoids diamond inheritance issues and keeps all boto3 logic in one place. File handles created by ``_open`` run their parallel operations (range reads, multipart uploads) in that request pool, as ``S3File`` does. :meth:`open_async` returns an ``AioS3StreamedFile``, whose reads and writes are coroutines. :meth:`close`, or the end of a ``with`` or ``async with`` block, releases the threads of the filesystem, which stays usable. Attributes: _sync_fs: The internal synchronous S3FileSystem instance. Example: >>> from pyathena.filesystem.s3_async import AioS3FileSystem >>> fs = AioS3FileSystem(asynchronous=True) >>> >>> # Use in async context >>> files = await fs._ls('s3://my-bucket/data/') >>> >>> # Sync wrappers (auto-generated by fsspec) need an instance created >>> # without asynchronous=True; they block the caller until done >>> files = AioS3FileSystem().ls('s3://my-bucket/data/') """ protocol = ("s3", "s3a") mirror_sync_methods = True async_impl = True _extra_tokenize_attributes = ("default_block_size",)
[docs] def __init__( self, connection: Connection[Any] | None = None, default_block_size: int | None = None, default_cache_type: str | None = None, max_workers: int | None = None, s3_additional_kwargs: dict[str, Any] | None = None, allow_bucket_creation: bool = False, allow_bucket_deletion: bool = False, version_aware: bool = False, asynchronous: bool = False, loop: Any | None = None, batch_size: int | None = None, *, client: BaseClient | None = None, retry_config: RetryConfig | None = None, **kwargs, ) -> None: """Initialize the filesystem and its internal ``S3FileSystem``. Args: connection: Deprecated; use ``client`` and ``retry_config``, or :func:`pyathena.filesystem.s3_filesystem` with ``AioS3FileSystem``. A PyAthena connection whose S3 client and retry policy the filesystem uses, with a ``DeprecationWarning``. default_block_size: Passed to the internal ``S3FileSystem``. default_cache_type: Passed to the internal ``S3FileSystem``. max_workers: The number of threads that run the blocking calls of the filesystem, and, passed to the internal ``S3FileSystem``, the size of its request pool and the default number of parallel requests of its files and copies. If None, the ``max_pool_connections`` of the S3 client (10 unless configured), as for ``S3FileSystem``. s3_additional_kwargs: Passed to the internal ``S3FileSystem``. allow_bucket_creation: Passed to the internal ``S3FileSystem``. allow_bucket_deletion: Passed to the internal ``S3FileSystem``. version_aware: Passed to the internal ``S3FileSystem``. asynchronous: Passed to ``fsspec.asyn.AsyncFileSystem``. loop: Passed to ``fsspec.asyn.AsyncFileSystem``. batch_size: Passed to ``fsspec.asyn.AsyncFileSystem``. client: Passed to the internal ``S3FileSystem``. A filesystem with a given client cannot be pickled. retry_config: Passed to the internal ``S3FileSystem``. **kwargs: Passed to both ``fsspec.asyn.AsyncFileSystem`` and the internal ``S3FileSystem``. Raises: TypeError: If ``connection`` is given with ``client`` or ``retry_config``, or the internal ``S3FileSystem`` rejects the arguments. """ super().__init__( asynchronous=asynchronous, loop=loop, batch_size=batch_size, **kwargs, ) # Checked before the client of the connection is used. client, retry_config = S3FileSystem._resolve_injected_client( type(self), connection, client, retry_config, kwargs ) self._sync_fs = S3FileSystem( client=client, retry_config=retry_config, default_block_size=default_block_size, default_cache_type=default_cache_type, max_workers=max_workers, s3_additional_kwargs=s3_additional_kwargs, allow_bucket_creation=allow_bucket_creation, allow_bucket_deletion=allow_bucket_deletion, version_aware=version_aware, # fsspec caches the AioS3FileSystem itself when caching is wanted. skip_instance_cache=True, **kwargs, ) # The blocking calls of the coroutines run in threads of this pool, # not in the default executor of the event loop, and wait for the # requests that they run in parallel in the request pool of the # internal S3FileSystem, which runs no call that waits. self._thread_pool = S3ThreadPool(self._sync_fs.max_workers) # Share dircache for cache coherence between async and sync instances self.dircache = self._sync_fs.dircache
[docs] @override def __reduce__(self) -> tuple[Any, ...]: """Reduce the filesystem for pickling, as fsspec does. Returns: The reduction of ``fsspec.AbstractFileSystem``, which creates the filesystem again from its arguments. Raises: TypeError: If the filesystem was built with ``client`` or ``connection``, whose boto3 client cannot be pickled. """ if self._sync_fs._client_injected: raise TypeError(_UNPICKLABLE_CLIENT_MESSAGE.format(cls=type(self).__name__)) return cast(tuple[Any, ...], super().__reduce__())
@property def core(self) -> S3Core: """The typed S3 operations of the wrapped ``S3FileSystem``.""" return self._sync_fs.core
[docs] def close(self) -> None: """Release the threads of the filesystem, which stays usable, once they are idle. Releases the threads of the blocking calls and those of the request pool of the internal ``S3FileSystem`` (see :meth:`S3FileSystem.close() <pyathena.filesystem.s3.S3FileSystem.close>` and :meth:`S3ThreadPool.release() <pyathena.filesystem.s3_executor.S3ThreadPool.release>`). The calls in progress, including the requests of open files, run to their end. """ self._thread_pool.release() self._sync_fs.close()
[docs] def __enter__(self) -> AioS3FileSystem: """Return the filesystem, whose threads the end of the block releases.""" return self
[docs] def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Release the threads with :meth:`close`.""" self.close()
[docs] async def __aenter__(self) -> AioS3FileSystem: """Return the filesystem, whose threads the end of the block releases.""" return self
[docs] async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Release the threads with :meth:`close`.""" self.close()
async def _info(self, path: str | os.PathLike[str] | S3Path, **kwargs) -> S3Object: return await self._thread_pool.run(self._sync_fs.info, path, **kwargs) async def _ls( self, path: str | os.PathLike[str], detail: bool = False, **kwargs ) -> list[S3Object] | list[str]: return await self._thread_pool.run(self._sync_fs.ls, path, detail=detail, **kwargs) async def _cat_file( self, path: str | os.PathLike[str] | S3Path, start: int | None = None, end: int | None = None, **kwargs, ) -> bytes: _S3BatchTasks.track() return await self._thread_pool.run( self._sync_fs.cat_file, path, start=start, end=end, **kwargs ) async def _exists(self, path: str | os.PathLike[str], **kwargs) -> bool: return await self._thread_pool.run(self._sync_fs.exists, path, **kwargs) async def _rm_file(self, path: str | os.PathLike[str], **kwargs) -> None: await self._thread_pool.run(self._sync_fs.rm_file, path, **kwargs) async def _pipe_file( self, path: str | os.PathLike[str], value: bytes | bytearray | memoryview, mode: str = "overwrite", **kwargs, ) -> None: _S3BatchTasks.track() if self._intrans: # The transaction belongs to this filesystem, not to the internal # S3FileSystem, so write through open() to defer the commit to it. await self._write_in_transaction( functools.partial(self._pipe_file_in_transaction, path, value, mode, **kwargs) ) return await self._thread_pool.run(self._sync_fs.pipe_file, path, value, mode=mode, **kwargs) async def _write_in_transaction(self, write: Callable[[Callable[..., S3File]], None]) -> None: """Run a write of a file of this filesystem's transaction in a thread of the filesystem. A cancellation keeps the file out of the transaction, which would otherwise commit it, and discards it: a write whose last part was running when it was cancelled has nothing left that its stop would skip. A write that the cancellation stops discards its file before it ends, which the cancellation waits for; the file of a write that ended before is discarded in a thread of the filesystem without being waited for. See :class:`_S3TransactionWrite`. Args: write: The write, which opens its file with the callable that it receives, which takes the arguments of ``open()``. """ transaction_write = _S3TransactionWrite(self, write) try: await self._thread_pool.run(transaction_write.run) except asyncio.CancelledError: if transaction_write.cancel(): self._thread_pool.submit(transaction_write.discard) raise def _pipe_file_in_transaction( self, path: str | os.PathLike[str], value: bytes | bytearray | memoryview, mode: str, open_file: Callable[..., S3File], **kwargs, ) -> None: """Write bytes into the path as a file of this filesystem's transaction. Args: path: S3 path (s3://bucket/key) to write to. value: The bytes to write. mode: "overwrite" or "create". With "create", the file is opened in ``xb`` mode: raise FileExistsError when the object already exists, including one created before the transaction is committed, which is not replaced. open_file: Opens the file, with the arguments of ``open()``. **kwargs: Additional parameters passed to ``open()``, except ``compression``, with which the value is compressed before it is written, as in :meth:`S3FileSystem.pipe_file`. Raises: FileExistsError: If the mode is "create" and the path already exists. ValueError: If the compression is not supported, or if the data takes more than ``S3Core.MULTIPART_UPLOAD_MAX_PARTS`` blocks. """ # See S3FileSystem.pipe_file. compression = get_compression(self._strip_protocol(path), kwargs.pop("compression", None)) if compression is not None: value = CompressedBuffer.compress(value, compression) block_size = kwargs.get("block_size") or self._sync_fs.default_block_size # The size in bytes; the length of a memoryview counts its items. self.core.check_multipart_upload_size(memoryview(value).nbytes, block_size) open_file(path, "xb" if mode == "create" else "wb", **kwargs)._write_and_close(value) async def _put_file( self, lpath: str | os.PathLike[str], rpath: str | os.PathLike[str], callback=_DEFAULT_CALLBACK, mode: str = "overwrite", **kwargs, ) -> None: _S3BatchTasks.track() rpath = stringify_path(rpath) if self._intrans: # See _pipe_file. await self._write_in_transaction( functools.partial( self._put_file_in_transaction, lpath, rpath, callback, mode, **kwargs ) ) return await self._thread_pool.run( self._sync_fs.put_file, lpath, rpath, callback=callback, mode=mode, **kwargs ) def _put_file_in_transaction( self, lpath: str | os.PathLike[str], rpath: str, callback, mode: str, open_file: Callable[..., S3File], **kwargs, ) -> None: """Upload a local file as a file of this filesystem's transaction. Mirrors :meth:`S3FileSystem.put_file`, but writes through ``open()`` of this filesystem. Args: lpath: Local file path to upload. rpath: S3 destination path (s3://bucket/key). callback: Progress callback for tracking upload progress. mode: "overwrite" or "create". With "create", the file is opened in ``xb`` mode: raise FileExistsError when the object already exists, including one created before the transaction is committed, which is not replaced. open_file: Opens the file, with the arguments of ``open()``. **kwargs: Additional S3 parameters (e.g., ContentType, StorageClass). The ``block_size``, ``max_workers``, and ``s3_additional_kwargs`` parameters of ``open()`` are also accepted. Raises: FileExistsError: If the mode is "create" and the path already exists. ValueError: If the file takes more than ``S3Core.MULTIPART_UPLOAD_MAX_PARTS`` blocks. """ if os.path.isdir(lpath): return if not S3Path.parse(rpath).key: return size = os.path.getsize(lpath) block_size = kwargs.pop("block_size", None) or self._sync_fs.default_block_size max_workers = kwargs.pop("max_workers", self._sync_fs.max_workers) # The other parameters are S3 request parameters, as in pipe_file(). s3_additional_kwargs = {**kwargs.pop("s3_additional_kwargs", {}), **kwargs} self.core.check_multipart_upload_size(size, block_size) callback.set_size(size) if "ContentType" not in {**self._sync_fs.s3_additional_kwargs, **s3_additional_kwargs}: content_type, _ = mimetypes.guess_type(lpath) if content_type is not None: s3_additional_kwargs["ContentType"] = content_type # See S3FileSystem.put_file. with open(lpath, "rb") as local: open_file( rpath, "xb" if mode == "create" else "wb", block_size=block_size, max_workers=max_workers, s3_additional_kwargs=s3_additional_kwargs, )._write_file_and_close(local, callback) self.invalidate_cache(rpath) async def _get_file( self, rpath: str | os.PathLike[str] | S3Path, lpath: str | os.PathLike[str], callback=_DEFAULT_CALLBACK, **kwargs, ) -> None: _S3BatchTasks.track() await self._thread_pool.run( self._sync_fs.get_file, rpath, lpath, callback=callback, **kwargs ) async def _mkdir( self, path: str | os.PathLike[str], create_parents: bool = True, **kwargs ) -> None: await self._thread_pool.run( self._sync_fs.mkdir, path, create_parents=create_parents, **kwargs ) async def _makedirs(self, path: str | os.PathLike[str], exist_ok: bool = False) -> None: await self._thread_pool.run(self._sync_fs.makedirs, path, exist_ok=exist_ok) async def _rm( self, path: str | os.PathLike[str] | list[str | os.PathLike[str]], recursive: bool = False, batch_size: int | None = None, maxdepth: int | None = None, **kwargs, ) -> None: """Delete objects with DeleteObjects requests. See :meth:`S3FileSystem.rm`. The DeleteObjects requests run in parallel, at most ``batch_size`` at once, in the request pool of the internal ``S3FileSystem``. A cancellation cancels the requests that have not started and waits for the others. Args: path: S3 path (s3://bucket/key) or list of paths to delete. recursive: Whether to delete all objects below the paths. batch_size: Number of DeleteObjects requests to send at the same time. None or -1 is ``max_workers``; the ``batch_size`` of the filesystem, which is that of fsspec's other batch coroutines, does not apply. ``max_workers`` is a deprecated alias. maxdepth: Maximum depth to expand when ``recursive`` is True. **kwargs: Additional parameters passed to the DeleteObjects API. ``Quiet`` (default True) sets the quiet mode of the requests. Raises: ValueError: If a path is a bucket, or ``batch_size`` is neither positive, -1, nor None. TypeError: If both ``batch_size`` and ``max_workers`` are given. OSError: If S3 could not delete some of the objects. """ path = self._sync_fs._stringify_paths(path) max_workers = self._sync_fs._delete_batch_limit(batch_size, kwargs) paths = await self._thread_pool.run( self._sync_fs._delete_paths, path, recursive=recursive, maxdepth=maxdepth ) await self._delete_objects(paths, max_workers=max_workers, **kwargs) async def _delete_objects( self, paths: list[str | S3Path], max_workers: int | None = None, **kwargs ) -> None: """Delete objects with parallel DeleteObjects requests. See :meth:`S3FileSystem._delete_objects`. Args: paths: Paths of the objects to delete. Bucket paths are skipped. max_workers: Maximum number of parallel requests. Defaults to the ``max_workers`` of the internal ``S3FileSystem``. **kwargs: Additional parameters passed to the DeleteObjects API. ``Quiet`` (default True) sets the quiet mode of the requests. Raises: TypeError: If kwargs has ``Bucket`` or ``Delete``. OSError: If S3 could not delete some of the objects. """ batches, params = self._sync_fs._delete_batches(paths, **kwargs) if not batches: return # At most max_workers requests at once, as S3FileSystem.rm() sends, # so that a large deletion does not queue ahead of the requests of # the files that share the request pool. results = await self._map_async( functools.partial(self._sync_fs._delete_batch, **params), batches, max_workers=max_workers, stop_on_failure=False, ) self._sync_fs._raise_delete_errors(results.outcomes) async def _map_async( self, fn: Callable[[ItemT], T], items: Iterable[ItemT], max_workers: int | None = None, **kwargs: Any, ) -> S3MapResult[T]: """Run the parallel requests of an operation in a scope of their own. As :meth:`S3FileSystem._map`, with :meth:`S3Executor.map_async() <pyathena.filesystem.s3_executor.S3Executor.map_async>`. Args: fn: The request to run for each item. items: The items. max_workers: Maximum number of parallel requests. Defaults to the ``max_workers`` of the internal ``S3FileSystem``. **kwargs: The options of ``S3Executor.map_async()``. Returns: The outcomes of the requests. """ executor = self._sync_fs._create_executor( max_workers=max_workers or self._sync_fs.max_workers ) try: return await executor.map_async(fn, items, **kwargs) finally: executor.shutdown(wait=False) async def _mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs) -> None: """Move files from one S3 location to another. See :meth:`S3FileSystem.mv`. The copies run in parallel with ``asyncio.gather``, whose cancellation cancels them and waits for them, and the deletions in the request pool. Args: path1: Source S3 path, glob pattern, or list of paths. path2: Destination S3 path, or list of paths when ``path1`` is a list. recursive: Whether to move the directories with their contents. maxdepth: Maximum depth of a recursive move. **kwargs: Additional S3 copy parameters, as for ``_cp_file()``. Raises: ValueError: If two sources have the same destination, or a destination is another source, including one left in place, which is checked before anything is copied. A source without an object at its key, such as a directory, writes no destination and does not conflict. FileNotFoundError: If a source does not exist, once the other copies have ended; nothing is deleted. OSError: If bucket versioning cannot be read. """ path1, path2 = self._sync_fs._stringify_paths(path1), self._sync_fs._stringify_paths(path2) if path1 == path2: return source_params = self.core.copy_source_params(kwargs) pairs = await self._thread_pool.run( self._sync_fs._move_pairs, path1, path2, recursive=recursive, maxdepth=maxdepth, source_params=source_params, ) # Every copy finishes before a failure is raised, as in fsspec's # _copy(), and nothing is deleted after a failure. results = await asyncio.gather( *[self._copy_file(p1, p2, **kwargs) for p1, p2 in pairs], return_exceptions=True ) for result in results: if isinstance(result, BaseException): raise result await self._delete_objects( [p1 for (p1, _), copied in zip(pairs, results, strict=True) if copied], **self.core.operation_params("delete_objects", source_params), )
[docs] def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs) -> None: """Move files from one S3 location to another. Runs :meth:`_mv`; fsspec does not provide the synchronous wrapper of ``_mv``. Args: path1: Source S3 path, glob pattern, or list of paths. path2: Destination S3 path, or list of paths when ``path1`` is a list. recursive: Whether to move the directories with their contents. maxdepth: Maximum depth of a recursive move. **kwargs: Additional S3 copy parameters. Raises: ValueError: If the move has conflicting paths. OSError: If bucket versioning cannot be read. """ sync(self.loop, self._mv, path1, path2, recursive=recursive, maxdepth=maxdepth, **kwargs)
async def _copy( self, path1, path2, recursive=False, on_error=None, maxdepth=None, batch_size=None, **kwargs, ) -> None: """Copy files within S3. See :meth:`S3FileSystem.copy`. The paths are paired and checked as there before anything is copied, and the copies run as in fsspec's ``_copy()``, ``batch_size`` of them at once as tasks. A cancellation cancels the copies and waits for them; a copy that is already being cancelled, such as by the shutdown of the event loop, is not cancelled again. Args: path1: Source S3 path, glob pattern, or list of them. path2: Destination S3 path, or list of paths when ``path1`` is a list. recursive: Whether to copy the directories with their contents. on_error: ``"raise"`` or ``"ignore"`` for a missing source. maxdepth: Maximum depth of a recursive copy. batch_size: Number of copies to run at the same time. **kwargs: Additional S3 copy parameters passed to ``_cp_file()``. Raises: ValueError: If two sources have the same destination, or a destination is another source, except for a directory with no object at its key, which is not copied. OSError: If bucket versioning cannot be read for the checks. """ path1, path2 = self._sync_fs._stringify_paths(path1), self._sync_fs._stringify_paths(path2) if on_error is None: on_error = "ignore" if recursive else "raise" pairs = await self._thread_pool.run( self._sync_fs._checked_copy_pairs, path1, path2, recursive=recursive, maxdepth=maxdepth, source_params=self.core.copy_source_params(kwargs), ) if not pairs: return await _S3BatchTasks().run( super()._copy( [p1 for p1, _ in pairs], [p2 for _, p2 in pairs], on_error=on_error, batch_size=batch_size, **kwargs, ) ) async def _get( self, rpath, lpath, recursive=False, callback=_DEFAULT_CALLBACK, maxdepth=None, **kwargs ) -> None: """Copy files from S3 to the local filesystem. See :meth:`S3FileSystem.get`. The paths are paired and checked as there before anything is downloaded, and the downloads run as in fsspec's ``_get()``; a cancellation cancels them and waits for them, as in :meth:`_copy`. As in :meth:`S3FileSystem.get`, each download gets ``max(1, max_workers // n)`` parallel range reads, or the caller's smaller ``max_workers``, where ``n`` is the number of files, or ``batch_size`` when it is positive and smaller. With ``outfile``, the files are written to it one after another, in order, and ``n`` is 1. Args: rpath: Source S3 path, glob pattern, or list of them. lpath: Local destination path, or list of paths when ``rpath`` is a list. recursive: Whether to copy the directories with their contents. callback: Progress callback. maxdepth: Maximum depth of a recursive copy. **kwargs: Additional parameters passed to ``_get_file()``, and ``batch_size``, the number of downloads that fsspec runs at once. Raises: ValueError: If a destination named after a source lies outside ``lpath``, or two downloads write the same local file. """ rpath = self._sync_fs._stringify_paths(rpath) # Local paths are made absolute, as fsspec's _get() does, so that # it can create their parent directories. root = ( make_path_posix(lpath) if isinstance(lpath, (str, os.PathLike)) else [make_path_posix(p) if isinstance(p, (str, os.PathLike)) else p for p in lpath] ) pairs = await self._thread_pool.run( self._sync_fs._checked_get_pairs, S3PathPairing(rpath, root, recursive=recursive, maxdepth=maxdepth), kwargs, ) if not pairs: return if kwargs.get("outfile") is not None: # _get_file() writes to the outfile instead of the local paths, # so the files are downloaded one after another, in order. kwargs = {**kwargs, "batch_size": 1} # fsspec's default batch size, when none is given, is not counted: # at most max_workers downloads run at once in the threads of the # filesystem, and more than max_workers get one range each anyway. concurrency = len(pairs) batch_size = kwargs.get("batch_size", self.batch_size) if batch_size is not None and batch_size > 0: concurrency = min(concurrency, batch_size) await _S3BatchTasks().run( self._get_pairs(pairs, callback, **self._sync_fs._batch_kwargs(concurrency, kwargs)) ) async def _get_pairs( self, pairs: list[tuple[str | S3Path, Any]], callback: Any, **kwargs ) -> None: """Download the pairs of a ``get()`` as fsspec's ``_get()`` downloads lists of paths. Each download gets a branch of the callback named after the string of its source, and the sources that a listing returns are downloaded by their buckets and keys. Args: pairs: The S3 sources and their local destinations. callback: Progress callback. **kwargs: Passed to ``_get_file()``, and ``batch_size``, the number of downloads to run at once. """ for _, lp in pairs: os.makedirs(os.path.dirname(lp), exist_ok=True) batch_size = kwargs.pop("batch_size", self.batch_size) callback.set_size(len(pairs)) async def get_pair(rp: str | S3Path, lp: Any) -> None: """Download a pair with a branch of the callback. Args: rp: The S3 source. lp: The local destination. """ with callback.branched(str(rp), lp, **kwargs) as child: await self._get_file(rp, lp, callback=child, **kwargs) await _run_coros_in_chunks( [get_pair(rp, lp) for rp, lp in pairs], batch_size=batch_size, callback=callback ) async def _put( self, lpath, rpath, recursive=False, callback=_DEFAULT_CALLBACK, batch_size=None, maxdepth=None, **kwargs, ) -> list[Any]: """Copy files from the local filesystem to S3. See :meth:`S3FileSystem.put`. The paths are paired and checked as there before anything is uploaded, and the uploads run as in fsspec's ``_put()``; a cancellation cancels them and waits for them, as in :meth:`_copy`. Unlike fsspec's ``_put()``, which creates a directory for each local directory, it makes no other request: S3 has no directories, and creating one checks or creates the bucket. Args: lpath: Local source path, glob pattern, or list of them, as strings or ``os.PathLike`` objects. rpath: S3 destination path, or list of paths when ``lpath`` is a list, as strings or ``os.PathLike`` objects. recursive: Whether to copy the directories with their contents. callback: Progress callback. batch_size: Number of uploads to run at the same time. maxdepth: Maximum depth of a recursive copy. **kwargs: Additional parameters passed to ``_put_file()``. Returns: The return values of the ``_put_file()`` calls, as in fsspec. Raises: ValueError: If two files have the same destination. """ lpath, rpath = self._sync_fs._stringify_paths(lpath), self._sync_fs._stringify_paths(rpath) pairs = await self._thread_pool.run( self._sync_fs._put_pairs, lpath, rpath, recursive=recursive, maxdepth=maxdepth, **kwargs ) files = [(lp, rp) for lp, rp in pairs if not os.path.isdir(lp)] return await _S3BatchTasks().run( super()._put( [lp for lp, _ in files], [rp for _, rp in files], callback=callback, batch_size=batch_size, **kwargs, ) ) async def _pipe(self, path, value=None, batch_size=None, **kwargs) -> list[Any]: """Write bytes into one or more paths. See :meth:`S3FileSystem.pipe`. The paths of a dictionary are checked as there before anything is written, and the writes run as in fsspec's ``_pipe()``; a cancellation cancels them and waits for them, as in :meth:`_copy`. Args: path: S3 path, or a dictionary of paths and their bytes. value: The bytes to write when ``path`` is a path. batch_size: Number of writes to run at the same time. **kwargs: Additional parameters passed to ``_pipe_file()``. Returns: The return values of the ``_pipe_file()`` calls, as in fsspec. Raises: ValueError: If two paths of a dictionary name the same object. """ if isinstance(path, dict): path = dict(self._sync_fs._pipe_items(path)) else: path = stringify_path(path) return await _S3BatchTasks().run( super()._pipe(path, value, batch_size=batch_size, **kwargs) ) async def _cat(self, path, recursive=False, on_error="raise", batch_size=None, **kwargs): """Read the contents of one or more paths. See :meth:`S3FileSystem.cat`. The reads run as in fsspec's ``_cat()``, and a cancellation cancels them and waits for them, as in :meth:`_copy`. Args: path: S3 path, glob pattern, or list of them. recursive: Whether to read the files below the directories. on_error: ``"raise"``, ``"omit"``, or ``"return"``. batch_size: Number of reads to run at the same time. **kwargs: Additional parameters passed to ``_cat_file()``. Returns: The bytes of a single path, or a dictionary of the paths and their bytes. """ return await _S3BatchTasks().run( self._cat_paths( path, recursive=recursive, on_error=on_error, batch_size=batch_size, **kwargs ) ) async def _cat_paths( self, path, recursive=False, on_error="raise", batch_size=None, **kwargs ) -> Any: """Read the contents of one or more paths as fsspec's ``_cat()`` does. Unlike fsspec, each path that the expansion returns is read by its bucket and key (see :meth:`S3FileSystem._expand_paths`), never by parsing its name. Args: path: S3 path, glob pattern, or list of them. recursive: Whether to read the files below the directories. on_error: ``"raise"``, ``"omit"``, or ``"return"``. batch_size: Number of reads to run at the same time. **kwargs: Additional parameters passed to the reads. Returns: The bytes of a single path, or a dictionary of the paths and their bytes. """ # As in fsspec, the expansion does not receive the parameters of the # reads. targets = self._sync_fs._first_targets( await self._thread_pool.run(self._sync_fs._expand_paths, path, recursive=recursive) ) paths = list(targets) out = await _run_coros_in_chunks( [self._cat_file(target, **kwargs) for target in targets.values()], batch_size=batch_size or self.batch_size, nofiles=True, return_exceptions=True, ) if on_error == "raise": error = next(filter(is_exception, out), None) if error is not None: raise error if len(paths) > 1 or isinstance(path, list) or paths[0] != self._strip_protocol(path): return { k: v for k, v in zip(paths, out, strict=True) if on_error != "omit" or not is_exception(v) } return out[0] async def _du(self, path, total=True, maxdepth=None, **kwargs): """Return the sizes of the objects below a path, as fsspec's ``_du()`` does. As fsspec does, looks up the entries that ``find()`` returns one at a time, each in a thread of the filesystem, so that a cancellation sends no further lookup of an entry. A listed entry is looked up by its bucket and key, never by parsing its name. Args: path: The path to search under. total: Whether to return the sum of the sizes instead of the sizes by name. maxdepth: Maximum number of levels to descend, at least 1 (None for unlimited). **kwargs: Additional arguments passed to ``find()``. Returns: The total size, or the sizes by the names that ``info()`` gives. """ found = await self._thread_pool.run( self._sync_fs._find_paths, path, maxdepth=maxdepth, **kwargs ) sizes: dict[str, int] = {} for _, target in found: info = await self._info(target) sizes[info["name"]] = info["size"] return sum(sizes.values()) if total else sizes async def _cat_ranges( self, paths, starts, ends, max_gap=None, batch_size=None, on_error="return", **kwargs, ): """Read byte ranges of one or more paths. See :meth:`S3FileSystem.cat_ranges`. The reads run as in fsspec's ``_cat_ranges()``, and a cancellation cancels them and waits for them, as in :meth:`_copy`. Args: paths: List of S3 paths. starts: Start offset of each range, or one for all of them. ends: End offset of each range, or one for all of them. max_gap: Not supported; must be None. batch_size: Number of reads to run at the same time. on_error: ``"return"`` to return the error of a read in place of its bytes; otherwise the first error is raised. **kwargs: Additional parameters passed to ``_cat_file()``. Returns: The bytes or the error of each range, in order. """ return await _S3BatchTasks().run( super()._cat_ranges( paths, starts, ends, max_gap=max_gap, batch_size=batch_size, on_error=on_error, **kwargs, ) ) async def _cp_file( self, path1: str | os.PathLike[str] | S3Path, path2: str | os.PathLike[str] | S3Path, recursive=False, maxdepth=None, on_error=None, **kwargs, ) -> None: """Copy an S3 object, using async parallel multipart upload for large files. A directory ``path1``, which recursive ``copy()`` passes along with the files under it, is skipped. Args: path1: Source S3 path (s3://bucket/key), or an ``S3Path``, as for :meth:`S3FileSystem.cp_file`. path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. recursive: Unused parameter for fsspec compatibility. maxdepth: Unused parameter for fsspec compatibility. on_error: Unused parameter for fsspec compatibility. **kwargs: Additional S3 copy parameters, as for :meth:`S3FileSystem.cp_file`. Raises: ValueError: If trying to copy to a versioned file or copy buckets. """ _S3BatchTasks.track() await self._copy_file( path1 if isinstance(path1, S3Path) else stringify_path(path1), path2 if isinstance(path2, S3Path) else stringify_path(path2), **kwargs, ) async def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`_cp_file` does. Args: path1: Source S3 path (s3://bucket/key), or an ``S3Path``. path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. **kwargs: Additional S3 copy parameters, as for :meth:`_cp_file`. Returns: False if ``path1`` is a directory, which is skipped; True if the object was copied. Raises: ValueError: If trying to copy to a versioned file or copy buckets. """ # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) source = S3Path.of(path1) destination = S3Path.of(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: raise ValueError("Cannot copy buckets.") # Looked up with the parameters that authorize the access to the # source; see S3FileSystem._copy_file. info1 = await self._info(path1, **self.core.copy_source_params(kwargs)) if info1.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # Recursive copy() passes the directories too; see # S3FileSystem.cp_file. return False size1 = info1.get("size", 0) try: if size1 <= self.core.MULTIPART_UPLOAD_MAX_PART_SIZE: await self._thread_pool.run(self.core.copy_object, source, destination, **kwargs) else: await self._copy_object_with_multipart_upload( source, destination, max_workers=max_workers, block_size=block_size, **kwargs, ) finally: # See S3FileSystem._copy_file. self._sync_fs._invalidate(destination) return True async def _copy_object_with_multipart_upload( self, source: S3Path, destination: S3Path, max_workers: int | None = None, block_size: int | None = None, **kwargs, ) -> None: """Copy an object with a multipart upload of its byte ranges. See :meth:`S3FileSystem._copy_object_with_multipart_upload`. The part copies run in the request pool of the internal ``S3FileSystem``, which bounds the requests of all its copies and files together, with at most ``max_workers`` of them in flight; each one is sent once an earlier one has finished, and the upload is completed once all of them have. The annotation copies then run in parallel, at most ``max_workers`` at once, in the same pool, and a failure or a cancellation stops them as :meth:`~pyathena.filesystem.s3_executor.S3Executor.map_async` does. On a cancellation, the requests that have not started, including the completion, are cancelled, the running ones are waited for in a thread of the filesystem, the upload is aborted if it was created and has not completed, and the cancellation is re-raised. On a failure or cancellation, the running part copies make no further attempt. This also holds when the shutdown of the event loop, such as at the end of ``asyncio.run()``, cancels the copy, also after its cleanup has started. A repeated cancellation returns without stopping this cleanup. A failed abort is logged with the upload ID. Args: source: Source S3 path, with the version ID to copy, if any. destination: Destination S3 path. max_workers: Maximum number of parallel requests. block_size: Size in bytes of the copied ranges. **kwargs: The CopyObject parameters of the copy; each request receives those that it accepts. Raises: ValueError: If ``block_size`` is out of the part size limits, a directive has an invalid value, or HeadObject reports no size. """ max_workers = max_workers if max_workers else self._sync_fs.max_workers plan = await self._thread_pool.run( self.core.plan_multipart_copy, source, destination, block_size, **kwargs ) if plan.fits_single_request: # See S3FileSystem._copy_object_with_multipart_upload. await self._thread_pool.run( self.core.copy_object, plan.source, plan.destination, **kwargs ) return # The requests run in the request pool, not as tasks, which the # shutdown of the event loop would cancel while the requests keep # running. The completion and the abort, which wait for them, run in # the threads of the filesystem. session = _S3WriteSession( self.core, plan.destination, self._sync_fs._create_executor(max_workers=max_workers), block_size=block_size if block_size else self.core.MULTIPART_UPLOAD_MAX_PART_SIZE, max_workers=max_workers, prepared_params={ "create_multipart_upload": plan.create_params, "upload_part_copy": plan.part_params, "complete_multipart_upload": plan.complete_params, "abort_multipart_upload": plan.abort_params, }, ) # Created before it is scheduled, so that an interrupt while it is # being scheduled still lets the cleanup cancel it or wait for it. completion: Future[S3CompleteMultipartUpload] = Future() def _abort() -> None: """Abort the upload, if any, once its requests have ended. Runs in a thread of the filesystem, so that neither a repeated cancellation nor the shutdown of the event loop stops it. See :meth:`_S3WriteSession.abort`. A failure is logged instead of raised, so that the error that made the copy fail is raised. """ if not completion.cancelled(): # A completion that has started runs to its end, and the # upload is aborted only if it failed. wait([completion]) try: session.abort() except Exception: upload = cast(S3MultipartUpload, session.upload) _logger.exception( f"Failed to abort multipart upload {upload.upload_id} " f"to s3://{upload.bucket}/{upload.key}." ) try: session.start() for part_number, range_ in enumerate(plan.ranges, 1): # A cancellation of this task cancels only its wait. await asyncio.wrap_future(session.ready()) session.submit_copy(part_number, plan.source, range_) # Here, not in the thread of the completion, so that a failed # part stops the others at once. await asyncio.wrap_future(session.settled()) # Reserved before it is queued, so that a cancellation of this # task can still cancel a completion that has not started. session.begin_complete() self._thread_pool.submit_to(completion, session.complete) # A cancellation of this task leaves the completion to the # cleanup, which waits for a started completion. completed = await self._thread_pool.wait_shielded(completion) except BaseException: # Also on cancellation, as S3FileSystem._copy_object_with_multipart_upload # does on an interrupt. A completion that never runs, as it has # not started or was never scheduled, gives back its reservation. if completion.cancel(): session.withdraw_complete() # Here, so that the creation and the running parts stop even # while they hold every thread that the cleanup waits for. session.begin_abort() # A repeated cancellation of this task returns without stopping # the cleanup. await self._thread_pool.run_shielded(_abort) raise # At most max_workers copies at once in the request pool. await self._map_async( lambda name: self.core.copy_object_annotation( name, plan.source, plan.destination, completed.version_id, completed.etag, **kwargs, ), plan.annotations, max_workers=max_workers, ) async def _find( self, path: str | os.PathLike[str], maxdepth: int | None = None, withdirs: bool = False, **kwargs, ) -> dict[str, S3Object] | list[str]: detail = kwargs.pop("detail", False) files = await self._thread_pool.run( self._sync_fs._find, path, maxdepth=maxdepth, withdirs=withdirs, **kwargs ) if detail: return {f.name: f for f in files} return [f.name for f in files] async def _glob(self, path, maxdepth=None, **kwargs): """Find the paths that a glob pattern matches, as fsspec's ``_glob()`` does. Args: path: The glob pattern, as a string or an ``os.PathLike``. maxdepth: Maximum depth of a ``**`` pattern. **kwargs: Additional arguments passed to ``_find()``. Returns: The matching paths, or a dict of their info with ``detail=True``. """ return await super()._glob(stringify_path(path), maxdepth=maxdepth, **kwargs)
[docs] def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): """Walk the directories below a path, as fsspec's ``walk()`` does. Runs the walk of :meth:`S3FileSystem.walk`, which lists a listed directory by its key prefix rather than by its name. As fsspec's ``walk()`` of an asynchronous filesystem, which runs ``_walk()``, it yields a directory whose listing fails with no entries. Args: path: The path to walk. maxdepth: Maximum number of levels to walk, at least 1 (None for unlimited). topdown: Whether to yield a directory before the directories below it. on_error: ``"omit"``, ``"raise"``, or a callable that receives the error of the listing of the path, as in :meth:`S3FileSystem.walk`. **kwargs: Additional arguments passed to ``ls()``. Yields: Tuples of a directory path, its directories, and its files. """ yield from self._sync_fs._walk_tree( self._strip_protocol(path), maxdepth=maxdepth, topdown=topdown, on_error=on_error, yield_failed=True, **kwargs, )
async def _walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): """Walk the directories below a path, as fsspec's ``_walk()`` does. Walks as :meth:`S3FileSystem.walk` does, listing a listed directory by its key prefix rather than by its name. Each listing runs in the threads of the filesystem and is awaited on its own, and ``on_error`` runs in the event loop, as in fsspec. As in fsspec's ``_walk()``, a directory whose listing fails with ``on_error`` other than ``"raise"`` is yielded with no entries. Args: path: The path to walk. maxdepth: Maximum number of levels to walk, at least 1 (None for unlimited). topdown: Whether to yield a directory before the directories below it. on_error: ``"omit"``, ``"raise"``, or a callable that receives the error of the listing of the path, as in :meth:`S3FileSystem.walk`. **kwargs: Additional arguments passed to ``ls()``. Yields: Tuples of a directory path, its directories, and its files. Raises: ValueError: If ``maxdepth`` is less than 1. """ async for step in self._walk_directory( self._strip_protocol(path), None, maxdepth, topdown, on_error, **kwargs ): yield step async def _walk_directory( self, path: str, directory: S3Object | None, maxdepth: int | None, topdown: bool, on_error: Any, **kwargs, ) -> AsyncIterator[tuple[str, Any, Any]]: """Walk a path, or a listed directory by its key prefix, as :meth:`_walk` describes. Args: path: The path to walk, without the protocol, or the name of ``directory``. directory: The listed directory to walk, or None to list ``path`` with ``ls()``. maxdepth: Maximum number of levels to walk, at least 1. topdown: Whether to yield a directory before those below it. on_error: What to do with an error of the listing of the path. **kwargs: Additional arguments passed to ``ls()``. Yields: Tuples of a directory path, its directories, and its files. Raises: ValueError: If ``maxdepth`` is less than 1. """ if maxdepth is not None and maxdepth < 1: raise ValueError("maxdepth must be at least 1") detail = kwargs.pop("detail", False) try: listing = await self._thread_pool.run( self._sync_fs._walk_listing, path, directory, **kwargs ) except (FileNotFoundError, OSError) as e: if on_error == "raise": raise if callable(on_error): on_error(e) yield path, {} if detail else [], {} if detail else [] return dirs, files = self._sync_fs._walk_entries(path, listing) names: Any = dirs if detail else list(dirs) listed: Any = files if detail else list(files) if topdown: yield path, names, listed if maxdepth is not None: maxdepth -= 1 if maxdepth < 1: if not topdown: yield path, names, listed return for d in names: # As in fsspec, on_error applies to the path itself. async for step in self._walk_directory( dirs[d].name, dirs[d], maxdepth, topdown, "omit", detail=detail, **kwargs ): yield step if not topdown: yield path, names, listed
[docs] def tree( self, path: str | os.PathLike[str] = "/", recursion_limit: int = 2, max_display: int = 25, display_size: bool = False, prefix: str = "", is_last: bool = True, first: bool = True, indent_size: int = 4, ) -> str: """Return a tree of the paths below a path, as fsspec's ``tree()`` does. Runs :meth:`S3FileSystem.tree`, which lists a listed directory by its key prefix rather than by its name. Args: path: The root path, as a string or an ``os.PathLike``. recursion_limit: Maximum depth of the tree. max_display: Maximum number of entries shown per directory. display_size: Whether to show the sizes of the files. prefix: The prefix of each line. is_last: Whether the path is the last entry of its directory. first: Whether the path is the root of the tree. indent_size: Number of spaces per level. Returns: The tree as a string. """ return self._sync_fs.tree( path, recursion_limit=recursion_limit, max_display=max_display, display_size=display_size, prefix=prefix, is_last=is_last, first=first, indent_size=indent_size, )
[docs] def get_mapper( self, root: str | os.PathLike[str] = "", check: bool = False, create: bool = False, missing_exceptions=None, ): """Return a key/value store at a path, as fsspec's ``get_mapper()`` does. Args: root: The root path of the store, as a string or an ``os.PathLike``. check: Whether to check that the root is writable. create: Whether to create the root if it does not exist. missing_exceptions: The exceptions that mean a missing key. Returns: The ``fsspec.mapping.FSMap`` of the root. """ return super().get_mapper( stringify_path(root), check=check, create=create, missing_exceptions=missing_exceptions, )
[docs] def unstrip_protocol(self, name: str | os.PathLike[str]) -> str: """Add the protocol to a path, as fsspec's ``unstrip_protocol()`` does. Args: name: The path, as a string or an ``os.PathLike``. Returns: The path with the ``s3://`` protocol. """ return cast(str, super().unstrip_protocol(stringify_path(name)))
async def _expand_path(self, path, recursive=False, maxdepth=None, **kwargs) -> list[str]: """Expand glob patterns and directories into the paths they match. Runs :meth:`S3FileSystem.expand_path`, as :meth:`_find` runs ``find()``. Args: path: S3 path, glob pattern, or list of them. recursive: Whether to include the paths below the directories. maxdepth: Maximum depth of the expansion, at least 1. **kwargs: Additional arguments passed to ``glob`` and ``find``, and ``assume_literal``. Returns: The sorted matching paths. Raises: ValueError: If ``maxdepth`` is less than 1. FileNotFoundError: If nothing matches. """ return await self._thread_pool.run( self._sync_fs.expand_path, path, recursive=recursive, maxdepth=maxdepth, **kwargs ) def _open( self, path: str, mode: str = "rb", block_size: int | None = None, cache_type: str | None = None, autocommit: bool = True, cache_options: dict[Any, Any] | None = None, **kwargs, ) -> AioS3File: if block_size is None: block_size = self._sync_fs.default_block_size if cache_type is None: cache_type = self._sync_fs.default_cache_type max_workers = kwargs.pop("max_workers", None) if max_workers is None: # None takes the default of the filesystem. max_workers = self._sync_fs.max_workers # The parameters of the call take precedence over those of the # filesystem; the caller's dictionary is not modified. s3_additional_kwargs = { **self._sync_fs.s3_additional_kwargs, **kwargs.pop("s3_additional_kwargs", {}), } return AioS3File( self._sync_fs, path, mode, max_workers=max_workers, executor=self._sync_fs._create_executor(max_workers=max_workers), block_size=block_size, cache_type=cache_type, autocommit=autocommit, cache_options=cache_options, s3_additional_kwargs=s3_additional_kwargs, **kwargs, )
[docs] async def open_async( self, path: str | os.PathLike[str], mode: str = "rb", block_size: int | None = None, max_workers: int | None = None, version_id: str | None = None, s3_additional_kwargs: dict[str, Any] | None = None, **kwargs, ) -> AioS3StreamedFile: """Open a file whose reads and writes are coroutines. The lookups of the mode, as in ``open()``, run in the threads of the filesystem. If the caller is cancelled, a file opened after the cancellation is closed without committing anything (see :func:`~pyathena.aio.util.create_in_thread`). See :class:`AioS3StreamedFile` for how the file reads, writes, and commits. Args: path: S3 path (s3://bucket/key) of the file. mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. block_size: The block size for reads and writes. Defaults to the default block size of the filesystem. max_workers: The number of parallel requests of the file, in the request pool of the filesystem. Defaults to the ``max_workers`` of the filesystem. version_id: The version ID to read. s3_additional_kwargs: Additional parameters for the S3 requests of the file, which take precedence over those of the filesystem. **kwargs: Additional parameters for the S3 requests of the file, which take precedence over ``s3_additional_kwargs``. Returns: The opened file. Raises: ValueError: If the mode is not binary, compression is requested, or the arguments are invalid, as in ``open()``. NotImplementedError: If the mode is not supported, or the file is opened for writing in a transaction or with ``autocommit=False``, as the file commits when it is closed. FileExistsError: If an object exists at the path in exclusive-create mode. FileNotFoundError: If no object exists at the path when reading. """ if "b" not in mode or kwargs.pop("compression", None): raise ValueError("open_async() supports only binary modes without compression.") if not kwargs.pop("autocommit", True): raise NotImplementedError("The file of open_async() commits when it is closed.") if mode != "rb" and self._intrans: raise NotImplementedError("open_async() cannot write in a transaction.") path = self._strip_protocol(path) if block_size is None: block_size = self._sync_fs.default_block_size if max_workers is None: max_workers = self._sync_fs.max_workers s3_additional_kwargs = { **self._sync_fs.s3_additional_kwargs, **(s3_additional_kwargs or {}), } return await create_in_thread( functools.partial( AioS3StreamedFile, self, path, mode, version_id=version_id, max_workers=max_workers, block_size=block_size, s3_additional_kwargs=s3_additional_kwargs, **kwargs, ), release=AioS3StreamedFile._close_without_commit_blocking, executor=self._thread_pool, )
async def _rmdir(self, path: str | os.PathLike[str]) -> None: await self._thread_pool.run(self._sync_fs.rmdir, path)
[docs] def rmdir(self, path: str | os.PathLike[str]) -> None: """Remove an S3 bucket, which must be empty. See :meth:`S3FileSystem.rmdir`. Args: path: S3 bucket path (e.g., "s3://bucket"). """ self._sync_fs.rmdir(path)
[docs] def sign(self, path: str | os.PathLike[str], expiration: int = 3600, **kwargs) -> str: """Generate a presigned URL for S3 object access. See :meth:`S3FileSystem.sign`. Args: path: S3 path (s3://bucket/key) to generate the URL for. expiration: URL expiration time in seconds. **kwargs: Additional parameters passed to :meth:`S3FileSystem.sign`. Returns: The presigned URL. """ return cast(str, self._sync_fs.sign(path, expiration=expiration, **kwargs))
[docs] def metadata(self, path: str | os.PathLike[str], **kwargs) -> S3Metadata: """Return the metadata of the path. See :meth:`S3FileSystem.metadata`. Args: path: S3 path (s3://bucket/key) to get metadata for. **kwargs: Additional parameters passed to the HeadObject API. Returns: S3Metadata of the object. """ return self._sync_fs.metadata(path, **kwargs)
[docs] def getxattr(self, path: str | os.PathLike[str], attr_name: str, **kwargs) -> str | None: """Get an attribute from the user-defined metadata of the path. See :meth:`S3FileSystem.getxattr`. Args: path: S3 path (s3://bucket/key) to get the attribute for. attr_name: The name of the attribute. **kwargs: Additional parameters passed to the HeadObject API. Returns: The value of the attribute, or None if the attribute is not set. """ return self._sync_fs.getxattr(path, attr_name, **kwargs)
[docs] def setxattr( self, path: str | os.PathLike[str], copy_kwargs: dict[str, Any] | None = None, **kwargs ) -> None: """Set the user-defined metadata of the path. See :meth:`S3FileSystem.setxattr`. Args: path: S3 path (s3://bucket/key) to set metadata for. copy_kwargs: Additional parameters to use for the underlying CopyObject API call. **kwargs: Key-value pairs of metadata to set; a None value deletes the key. """ self._sync_fs.setxattr(path, copy_kwargs=copy_kwargs, **kwargs)
[docs] def get_tags(self, path: str | os.PathLike[str]) -> dict[str, str]: """Retrieve the tag key/values for the given path. See :meth:`S3FileSystem.get_tags`. Args: path: S3 path (s3://bucket/key) to get tags for. Returns: Dictionary mapping tag keys to tag values. """ return self._sync_fs.get_tags(path)
[docs] def put_tags(self, path: str | os.PathLike[str], tags: dict[str, str], mode: str = "o") -> None: """Set the tags for the given existing key. See :meth:`S3FileSystem.put_tags`. Args: path: S3 path (s3://bucket/key) of the existing key. tags: Tags to apply. mode: ``o`` to overwrite the existing tags or ``m`` to merge with them. """ self._sync_fs.put_tags(path, tags, mode=mode)
[docs] def chmod( self, path: str | os.PathLike[str], acl: str, recursive: bool = False, **kwargs ) -> None: """Set the canned ACL of a bucket or key. See :meth:`S3FileSystem.chmod`. Args: path: S3 path (s3://bucket or s3://bucket/key) to set the ACL on. acl: The canned ACL to apply. recursive: Whether to apply the ACL to all keys below the path too. **kwargs: Additional parameters passed to the PutObjectAcl or PutBucketAcl API. """ self._sync_fs.chmod(path, acl, recursive=recursive, **kwargs)
[docs] def object_version_info( self, path: str | os.PathLike[str], delete_markers: bool = False, **kwargs ) -> list[S3ObjectVersion]: """List the versions of the object or of the objects under the path. See :meth:`S3FileSystem.object_version_info`. Args: path: S3 path (s3://bucket/key or a key prefix) to list the versions for. delete_markers: Whether to include delete markers in the result. **kwargs: Additional parameters passed to the ListObjectVersions API. Returns: List of S3ObjectVersion instances describing the versions. """ return self._sync_fs.object_version_info(path, delete_markers=delete_markers, **kwargs)
[docs] def list_multipart_uploads(self, path: str | os.PathLike[str]) -> list[S3MultipartUpload]: """List in-progress (incomplete) multipart uploads in a bucket. See :meth:`S3FileSystem.list_multipart_uploads`. Args: path: S3 bucket or key path (e.g., "s3://bucket" or "s3://bucket/prefix"). Returns: List of S3MultipartUpload instances describing the uploads. """ return self._sync_fs.list_multipart_uploads(path)
[docs] def clear_multipart_uploads(self, path: str | os.PathLike[str]) -> None: """Abort any incomplete multipart uploads in the bucket. See :meth:`S3FileSystem.clear_multipart_uploads`. Args: path: S3 bucket or key path (e.g., "s3://bucket" or "s3://bucket/prefix"). """ self._sync_fs.clear_multipart_uploads(path)
[docs] def checksum(self, path: str | os.PathLike[str], **kwargs) -> int: """Get the checksum of an S3 object or directory. See :meth:`S3FileSystem.checksum`. Args: path: S3 path (s3://bucket/key) to get the checksum for. **kwargs: Additional arguments passed to :meth:`S3FileSystem.checksum`. Returns: Integer checksum derived from the ETag or the directory token. """ return cast(int, self._sync_fs.checksum(path, **kwargs))
[docs] def created(self, path: str | os.PathLike[str]) -> datetime: """Return the creation time of the path. See :meth:`S3FileSystem.created`. Args: path: S3 path (s3://bucket/key). Returns: The last-modified time of the object. """ return self._sync_fs.created(path)
[docs] def modified(self, path: str | os.PathLike[str]) -> datetime: """Return the last-modified time of the path. See :meth:`S3FileSystem.modified`. Args: path: S3 path (s3://bucket/key). Returns: The last-modified time of the object. """ return self._sync_fs.modified(path)
[docs] def invalidate_cache(self, path: str | os.PathLike[str] | None = None) -> None: """Remove the cached entries of the path and its parent paths. See :meth:`S3FileSystem.invalidate_cache`. Args: path: The path to invalidate. If None, clear the whole cache. """ self._sync_fs.invalidate_cache(path)
def _invalidate(self, path: S3Path) -> None: """Remove the cached entries of a path and of the directories above it. See :meth:`S3FileSystem._invalidate`. Args: path: The path of the bucket or the object. """ self._sync_fs._invalidate(path) async def _touch( self, path: str | os.PathLike[str], truncate: bool = True, **kwargs ) -> dict[str, Any]: return await self._thread_pool.run(self._sync_fs.touch, path, truncate=truncate, **kwargs)
[docs] def touch( self, path: str | os.PathLike[str], truncate: bool = True, **kwargs ) -> dict[str, Any]: """Create an empty object with PutObject. See :meth:`S3FileSystem.touch`. Args: path: S3 path (s3://bucket/key) of the object. truncate: If True, replace an existing object with an empty one; if False, raise if the object exists. **kwargs: Additional parameters passed to the PutObject API. Returns: The PutObject response as a dictionary. """ return self._sync_fs.touch(path, truncate=truncate, **kwargs)
[docs] class AioS3File(S3File): """The file handle that ``AioS3FileSystem.open()`` returns. Functionally identical to ``S3File``, whose parallel operations (range reads, multipart uploads) it runs in the request pool of the filesystem, at most ``max_workers`` of them at once; it exists as a distinct type for ``isinstance`` checks. For reads and writes that are coroutines, see ``AioS3StreamedFile``. """
[docs] class AioS3StreamedFile(AbstractAsyncStreamedFile): """An S3 file whose reads and writes are coroutines. Returned by :meth:`AioS3FileSystem.open_async`. The requests of the file run in the request pool of the filesystem, at most ``max_workers`` of them at once, and the coroutines wait for them without blocking the event loop: - A read splits the requested range into ranges of the block size, read in parallel. Nothing is cached. As with ``S3File``, the reads are conditional on the ETag of the object, and on its version when the filesystem is version-aware. - A write buffers the data, and uploads each full block as a part of a multipart upload, with at most ``max_workers`` parts, including the part copies of an append, in flight. A write that would submit another part waits for one of them to finish first. A request found failed by the next write, or before a part is submitted, stops the write with its error. An append copies an existing object with ``UploadPartCopy``, or rewrites one smaller than a part from the buffer, as ``S3File`` does. - Closing the file commits the written data, as ``S3File.close()`` does: with PutObject if no part was uploaded, otherwise by completing the multipart upload, which is aborted if the completion fails. - Leaving an ``async with`` block with an exception, a failed write, and a cancelled write close the file without committing it: the requests that have started are waited for and the multipart upload is aborted, so that the existing object is left unchanged. The commit and the abort, which wait for the requests, run to their end in the threads of the filesystem, also if the task that waits for them is cancelled, including a commit that still waits for a thread; once the commit has been submitted, nothing else aborts the upload. In a forked child process, a file that the parent opened raises ``RuntimeError`` from its reads, writes, and flushes, and closing it only marks it closed. A file opened for writing must be closed with ``await close()`` or an ``async with`` block; garbage collection cannot await it, so the data of a file that is collected before it is closed or its commit has been submitted is never committed; its multipart upload, if any, is aborted, with a ``ResourceWarning``. """ fs: AioS3FileSystem buffer: BytesIO | None closed: bool forced: bool offset: int | None
[docs] def __init__( self, fs: AioS3FileSystem, path: str, mode: str = "rb", version_id: str | None = None, max_workers: int | None = None, block_size: int = S3FileSystem.DEFAULT_BLOCK_SIZE, size: int | None = None, s3_additional_kwargs: dict[str, Any] | None = None, **kwargs, ) -> None: """Initialize the file for the path and mode. Sends the lookup requests of the mode (see :class:`S3FileTarget`), which block; :meth:`AioS3FileSystem.open_async` runs this in a thread. Args: fs: The filesystem that the file belongs to. path: S3 path (s3://bucket/key) of the file. mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. version_id: The version ID to read. max_workers: The number of parallel requests of the file, in the request pool of the filesystem. If None, the ``max_workers`` of the filesystem. block_size: The block size for reads and writes. Must be between ``S3Core.MULTIPART_UPLOAD_MIN_PART_SIZE`` and ``S3Core.MULTIPART_UPLOAD_MAX_PART_SIZE``, inclusive, unless reading. size: The size of the object to read, if known. s3_additional_kwargs: Additional parameters for the S3 requests of the file. Each request receives those that its operation accepts. **kwargs: Additional parameters for the S3 requests of the file, which take precedence over ``s3_additional_kwargs``. Raises: FileExistsError: If an object exists at the path in exclusive-create mode. FileNotFoundError: If no object exists at the path when reading. NotImplementedError: If the mode is not supported. ValueError: If ``max_workers`` is not positive, or the arguments are invalid as for ``S3FileTarget.resolve()``. """ if mode not in {"rb", "wb", "ab", "xb"}: raise NotImplementedError("File mode not supported") # The requests of the file, at most max_workers at once in the # request pool of the filesystem. Created first, so that an invalid # max_workers raises before the lookups. if max_workers is None: max_workers = fs._sync_fs.max_workers self._requests = fs._sync_fs._create_executor(max_workers=max_workers) # The scheduled commit and abort, which close() waits for, and those of # them whose queuing has returned. self._commit_future: Future[None] | None = None self._discard_future: Future[None] | None = None self._queued: set[Future[None]] = set() self.max_workers = max_workers # The process that opened the file; a forked child rejects it. self._pid = os.getpid() target = S3FileTarget.resolve( fs._sync_fs, path, mode, block_size=block_size, version_id=version_id, size=size, request_kwargs={**(s3_additional_kwargs or {}), **kwargs}, ) self.bucket = target.bucket self.key = target.key self.version_id = target.version_id self.s3_additional_kwargs: dict[str, Any] = target.request_kwargs self._details: S3Object | dict[str, Any] = target.details super().__init__( fs=fs, path=target.path, mode=mode, block_size=block_size, cache_type="none", size=target.size, # Kept as ``self.kwargs``, which fsspec passes to open() when it # reopens the file (e.g., when it is unpickled), as for S3File. **(target.request_kwargs if mode == "rb" else {}), ) self.append_block = False # The size of the appended object, as looked up when opening. self._append_size = 0 # The requests and the state of the write; None when reading. self._session: _S3WriteSession | None = None if mode != "rb": self._session = _S3WriteSession( fs.core, S3Path(self.bucket, self.key), self._requests, block_size=block_size, max_workers=self.max_workers, request_kwargs=self.s3_additional_kwargs, ) if target.appended is not None: if target.append_data is not None: cast(BytesIO, self.buffer).write(target.append_data) else: # Copied with UploadPartCopy as the leading part(s). self.append_block = True self._append_size = self.loc = target.appended.get("size", 0)
def __del__(self) -> None: # close() is a coroutine, which garbage collection cannot await, so # an unclosed file whose commit was not submitted is not committed: its # multipart upload, if any, is aborted. requests = getattr(self, "_requests", None) if requests is None or self._inherited(): # The upload of a file inherited across a fork is the parent's. return session = getattr(self, "_session", None) if ( not getattr(self, "closed", True) and session is not None and session.state not in (_S3WriteState.COMPLETING, _S3WriteState.COMPLETED) ): warnings.warn( f"s3://{self.bucket}/{self.key} was not closed, and the written data " "was not committed.", ResourceWarning, stacklevel=2, ) with contextlib.suppress(RuntimeError): self.fs._thread_pool.submit(self._discard) requests.shutdown(wait=False) def _inherited(self) -> bool: """Whether this process is a forked child of the process that opened the file. Returns: True in a forked child process of the process that opened the file. """ return getattr(self, "_pid", os.getpid()) != os.getpid() def _check_process(self) -> None: """Reject the use of a file that the parent of a forked process opened. Raises: RuntimeError: In a forked child process of the process that opened the file. """ if self._inherited(): raise RuntimeError( f"s3://{self.bucket}/{self.key} was opened by the parent of this forked " "process and cannot be used here; open it again in this process." )
[docs] def __enter__(self) -> AioS3StreamedFile: """Reject a synchronous ``with`` block, whose exit cannot close the file. Raises: TypeError: Always; use ``async with``. """ raise TypeError("AioS3StreamedFile is asynchronous; use 'async with'.")
def _get_request_kwargs(self, method: str) -> dict[str, Any]: """Select the parameters of the file that an S3 operation accepts. Args: method: The name of the client method, such as ``upload_part``. Returns: The parameters in ``s3_additional_kwargs`` that the operation accepts. """ return self.fs.core.operation_params(method, self.s3_additional_kwargs)
[docs] async def read(self, length: int = -1) -> bytes: """Read bytes from the file. Args: length: The number of bytes to read; -1 reads to the end. Returns: The bytes read. Raises: RuntimeError: In a forked child process of the process that opened the file. ValueError: If the file is closed or not open for reading. """ self._check_process() data: bytes = await super().read(length) return data
[docs] async def write(self, data: bytes) -> int: """Write data to the buffer, uploading each full block as a part. Args: data: The bytes to write. Returns: The number of bytes written. Raises: RuntimeError: In a forked child process of the process that opened the file. ValueError: If the file is not open for writing, is closed, or has been force-flushed. BaseException: The error of a request of the write that failed since the last write, or before a part is submitted, after which the file is closed without committing it. """ self._check_process() if self.mode not in {"wb", "ab", "xb"}: raise ValueError("File not in write mode") if self.closed: raise ValueError("I/O operation on closed file.") if self.forced: raise ValueError("This file has been force-flushed, can only close") try: cast(_S3WriteSession, self._session).check() except BaseException: await self._close_without_commit() raise buffer = cast(BytesIO, self.buffer) out = buffer.write(data) self.loc += out if buffer.tell() >= self.blocksize: await self.flush() return out
[docs] async def flush(self, force: bool = False) -> None: """Upload the buffered data if it fills a block, or all of it if forced. Args: force: Whether this is the final flush of the file, after which the written data is committed. Raises: RuntimeError: In a forked child process of the process that opened the file. ValueError: If the file is closed, or force-flushed twice. """ self._check_process() if self.closed: raise ValueError("Flush on closed file") if force and self.forced: raise ValueError("Force flush cannot be called more than once") if force: self.forced = True if self.mode not in {"wb", "ab", "xb"}: return buffer = cast(BytesIO, self.buffer) if not force and buffer.tell() < self.blocksize: return if self.offset is None: self.offset = 0 await self._initiate_upload() if await self._upload_chunk(final=force) is not False: self.offset += cast(BytesIO, self.buffer).seek(0, 2) self.buffer = BytesIO()
[docs] async def close(self) -> None: """Close the file, committing the written data. Returns once the requests of the file and its scheduled commit have ended, also when the close or an earlier read or flush was cancelled; the event loop runs meanwhile. In a forked child process, a file that the parent opened is only marked closed, without committing or aborting anything. """ if self.closed: return if self._inherited(): self.closed = True return try: if self.mode != "rb" and not self.forced: await self.flush(force=True) finally: self.closed = True await self._drain()
async def _drain(self) -> None: """Wait, without blocking the event loop, for the requests, commit, and abort of the file. A cancellation of the waiting task, also a repeated one, does not stop the wait; it is re-raised once they have ended, so that nothing of the file runs after its close returns (I1). A commit or abort whose queuing did not return, as when interrupts stopped it and its cleanup, is cancelled unless it has started, so that the wait does not last forever for one that was never queued. """ self._requests.shutdown(wait=False) pending = self._requests.pending() for future in (self._commit_future, self._discard_future): if future is not None and (future in self._queued or not future.cancel()): pending.append(future) if not pending: return # The outcomes belong to the reads, the commit, and the abort that # awaited them; retrieved, so that asyncio does not log them. waiters = [S3Executor.waiter(future) for future in pending] cancelled = False # The states are also checked every second, since an interrupt of a # cancellation can skip the callbacks that the waiters rely on. while not all(future.done() for future in pending): try: # Unlike gather(), wait() neither raises their outcomes nor # cancels them when this task is cancelled. await asyncio.wait(waiters, timeout=1.0) except asyncio.CancelledError: cancelled = True if cancelled: raise asyncio.CancelledError async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: if self.closed: return if exc_type is not None and self.mode != "rb": await self._close_without_commit() else: await self.close() async def _close_without_commit(self) -> None: """Close the file without uploading the written data. Drops the buffered data, and discards the write (see :meth:`_discard`), unless its commit has been reserved, which runs on. An abort that another call is running is waited for. """ if self.closed: return if self._inherited(): self.closed = True return try: if self._begin_close_without_commit(): # Tracked before it is queued, so that the drain below waits # for it also after a cancellation of this task. discard: Future[None] = Future() self._discard_future = discard self.fs._thread_pool.submit_to(discard, self._discard) self._queued.add(discard) await self.fs._thread_pool.wait_shielded(discard) finally: # A commit that was scheduled before runs on, and is waited for. await self._drain() def _close_without_commit_blocking(self) -> None: """Close the file without committing it; blocks. The blocking counterpart of :meth:`_close_without_commit`, for a file that nobody awaits, such as one opened for a cancelled :meth:`AioS3FileSystem.open_async`. """ if self.closed: return try: if self._begin_close_without_commit(): self._discard() finally: self._requests.shutdown(wait=False) def _begin_close_without_commit(self) -> bool: """Mark the file closed, drop the buffered data, and begin the abort. The part of :meth:`_close_without_commit` and :meth:`_close_without_commit_blocking` that runs before the discard. The abort begins here, so that the creation and the running parts stop even while they hold every thread of the request pool, ahead of :meth:`_discard`. Returns: Whether the caller has to discard the write; see :meth:`_S3WriteSession.begin_abort`. False for a file without a write session. """ self.closed = True self.buffer = None return self._session is not None and self._session.begin_abort() async def _initiate_upload(self) -> None: if not self.append_block and self.tell() < self.blocksize: return session = cast(_S3WriteSession, self._session) try: session.start() await self._wait_ready() # Raises the failure of the creation. session.check() if self.append_block: size = self._append_size whole_source = size <= self.fs.core.MULTIPART_UPLOAD_MAX_PART_SIZE source = S3Path(self.bucket, self.key) for part_number, range_ in session.iter_copy_parts(size): # The part copies count toward max_workers, as the parts do. await self._wait_ready() session.submit_copy(part_number, source, None if whole_source else range_) except BaseException: # A creation that has started may still create an upload, which # the abort waits for. await self._close_without_commit() raise async def _upload_chunk(self, final: bool = False) -> bool: """Upload the buffered data as parts of the multipart upload. See :meth:`S3File._upload_chunk`; a write waits for a part in flight without blocking the event loop. Args: final: Whether this is the last flush of the file, after which the upload is committed. Returns: Whether the buffer is reset afterwards. Raises: BaseException: The error of a request of the write that failed before a part is submitted. ValueError: If the data needs more parts than a multipart upload allows. """ if not self.append_block and self.tell() < self.blocksize: # Smaller than a block: uploaded with PutObject by the commit. if final: await self._commit() return not final session = cast(_S3WriteSession, self._session) buffer = cast(BytesIO, self.buffer) buffer.seek(0) try: for part_number, data in session.iter_parts(buffer): # Otherwise the parts waiting for a thread pile up in memory. await self._wait_ready() session.submit_part(part_number, data) except BaseException: await self._close_without_commit() raise if final: await self._commit() return not final async def _wait_ready(self) -> None: """Wait until another part may be submitted, without blocking the event loop. A cancellation of the waiting task cancels only its own wait. See :meth:`_S3WriteSession.ready`. """ waiter = cast(_S3WriteSession, self._session).ready() if not waiter.done(): await asyncio.wrap_future(waiter) async def _commit(self) -> None: """Commit the written data in a thread of the file. The commit is reserved before it is queued, so that a cancellation of the waiting task, also while the commit waits for a thread, does not abort it. If an interrupt stops the queuing, the reservation is withdrawn unless the commit has taken it, and the file is closed without committing it, so that the multipart upload is aborted instead of left incomplete. See :meth:`_complete_upload`. Raises: BaseException: The error of a request of the write that failed before the commit is reserved, or an error that stopped the queuing of the commit, after which the file is closed without committing it. """ session = cast(_S3WriteSession, self._session) # Tracked before it is queued, so that close() also waits for a # commit that an interrupt of the queuing has let start. commit: Future[None] = Future() self._commit_future = commit try: session.begin_complete() # Passed, as closing the file without committing it drops the buffer. self.fs._thread_pool.submit_to( commit, self._complete_upload, cast(BytesIO, self.buffer) ) self._queued.add(commit) except BaseException: if commit.cancel(): # Not started, and now never: a commit that was queued by a # submission that an interrupt stopped finds itself cancelled. session.withdraw_complete() await self._close_without_commit() raise await self.fs._thread_pool.wait_shielded(commit) def _complete_upload(self, buffer: BytesIO) -> None: """Upload or complete the written data; blocks, and runs in a thread. Creates an empty object if nothing was written, uploads the buffered data with PutObject if no multipart upload was started, and otherwise completes the multipart upload once its parts have been uploaded, aborting it if a part or the completion fails (see :meth:`_discard`). A failed part is raised as soon as it fails, without waiting for the parts before it. Invalidates the cache of the path afterwards. Args: buffer: The buffered data that no part has uploaded. Raises: FileExistsError: If an object was created at the path after the file was opened in exclusive-create mode. """ session = cast(_S3WriteSession, self._session) if not session.multipart: body = None if self.tell(): buffer.seek(0) body = buffer.read() # None writes an empty object. session.put(body) else: try: session.complete() except BaseException: self._discard() raise self.fs._invalidate(S3Path(self.bucket, self.key)) def _discard(self) -> None: """Discard the write, and abort its multipart upload, if any; blocks. Runs in a thread. See :meth:`_S3WriteSession.abort`. A failure is logged instead of raised, so that it does not mask the error that the caller is handling, also when no task waits for the abort any more. """ session = cast(_S3WriteSession, self._session) try: session.abort() except Exception: upload_id = cast(S3MultipartUpload, session.upload).upload_id _logger.exception( f"Failed to abort multipart upload {upload_id} to s3://{self.bucket}/{self.key}." ) async def _fetch_range(self, start: int, end: int) -> bytes: """Read a byte range of the object with parallel range requests. The range is clamped to the size of the object, and split into ranges of the block size, read at most ``max_workers`` at once in the request pool of the filesystem. When a read fails, the reads that have not started are cancelled, the running ones make no further attempt, and its error is raised once they have ended, as :meth:`S3File._fetch_range` does; a cancellation does the same. Args: start: The offset of the first byte to read. end: The offset to stop reading at (exclusive). Returns: The bytes read, empty if the clamped range is empty. Raises: RuntimeError: In a forked child process of the process that opened the file. """ self._check_process() end = min(end, self.size) if start >= end: return b"" ranges = S3File._get_ranges( start, end, max_workers=self.max_workers, worker_block_size=self.blocksize ) path = S3Path(self.bucket, self.key, self.version_id) request_kwargs = self._get_request_kwargs("get_object") # Not kept by a variable of this frame, which the traceback of a # cancellation keeps (F5). return b"".join( cast( list[bytes], ( await self._requests.map_async( functools.partial(self.fs.core.get_object, path, **request_kwargs), ranges, ) ).outcomes, ) )