diff --git a/pyathena/filesystem/s3.py b/pyathena/filesystem/s3.py index eceb61c7..69f8ed98 100644 --- a/pyathena/filesystem/s3.py +++ b/pyathena/filesystem/s3.py @@ -10,6 +10,7 @@ import os.path import re import time +from collections import Counter from collections.abc import Callable, Iterator, Mapping from concurrent.futures import Future, as_completed, wait from copy import deepcopy @@ -899,54 +900,66 @@ def _find( raise ValueError("Cannot traverse all files in S3.") bucket, key, _ = self.parse_path(path) prefix = kwargs.pop("prefix", "") - # Keep refresh in kwargs so that the recursive calls also refresh. - refresh = kwargs.get("refresh", False) + refresh = kwargs.pop("refresh", False) - # When maxdepth is specified, use a recursive approach with delimiter if maxdepth is not None: - result: list[S3Object] = [] - - # List files and directories at current level - current_items = self._ls_dirs(path, prefix=prefix, delimiter="/", refresh=refresh) - - for item in current_items: - if item.type == S3ObjectType.S3_OBJECT_TYPE_FILE: - # Add files - result.append(item) - elif item.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: - # Add directory if withdirs is True - if withdirs: - result.append(item) - - # Recursively explore subdirectory if depth allows - if maxdepth > 1: - sub_path = f"s3://{bucket}/{item.key}" - sub_results = self._find( - sub_path, maxdepth=maxdepth - 1, withdirs=withdirs, **kwargs - ) - result.extend(sub_results) - - return result - - # For unlimited depth, use the original approach (get all files at once) - files = self._ls_dirs(path, prefix=prefix, delimiter="", refresh=refresh) - if not files and key: + # The entries listed with the prefix lie as many levels further + # below the path as the prefix has slashes. + files = self._find_levels( + path, maxdepth - prefix.count("/"), prefix=prefix, refresh=refresh + ) + else: + files = self._ls_dirs(path, prefix=prefix, delimiter="", refresh=refresh) + # S3 doesn't return directory entries without a delimiter, so the + # directories are derived from the listed keys, below the last + # slash of the prefix, as with maxdepth. + if withdirs: + base_key = "/".join(k for k in (key, prefix.rpartition("/")[0]) if k) + # Build a new list; files may be the cached listing. + files = files + self._extract_parent_directories(files, bucket, base_key) + + if files: + # Something is listed below the path, so the path is a directory, + # which fsspec includes with the directories. A bucket is not + # included, since cp_file() cannot copy it in a recursive copy. + if withdirs and key: + files = [self._directory_object(bucket, key), *files] + elif key: + # As in fsspec, the path itself is returned if it is an object, + # or with withdirs if it is a directory. try: files = [self.info(path, refresh=refresh)] except FileNotFoundError: files = [] - # If withdirs is True, we need to derive directories from file paths - if withdirs: - # Build a new list; files may be the cached listing. - files = files + self._extract_parent_directories(files, bucket, key) - - # Filter directories if withdirs is False (default) - if withdirs is False or withdirs is None: + if not withdirs: files = [f for f in files if f.type != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY] - return files + def _find_levels( + self, path: str, maxdepth: int, prefix: str = "", refresh: bool = False + ) -> list[S3Object]: + """List the entries below a path level by level with ``Delimiter="/"``. + + Args: + path: S3 path to search under. + maxdepth: Number of levels to list. + prefix: Key prefix, relative to the path, to filter the first + level by. + refresh: If True, bypass the cache and list from S3. + + Returns: + The objects and directories found. + """ + if maxdepth < 1: + return [] + result: list[S3Object] = [] + for item in self._ls_dirs(path, prefix=prefix, delimiter="/", refresh=refresh): + result.append(item) + if item.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: + result.extend(self._find_levels(item.name, maxdepth - 1, refresh=refresh)) + return result + def find( self, path: str, @@ -959,7 +972,10 @@ def find( Recursively searches for files under the specified path, with optional depth limiting and directory inclusion. Uses efficient S3 list operations - with delimiter handling for performance. + with delimiter handling for performance. As in fsspec, the result + includes the path itself if withdirs is True and objects exist below + it, unless it is a bucket, or if it is an object and nothing is listed + below it. Args: path: S3 path to search under (e.g., "s3://bucket/prefix"). @@ -970,8 +986,9 @@ def find( detail: If True, return dict of {path: S3Object}; if False, return list of paths. **kwargs: Additional arguments including: prefix: Key prefix, relative to the path, to filter the listed keys - by. Without maxdepth, if nothing is listed and the path itself is - an object, that object is returned regardless of the prefix. + by. Each slash in the prefix counts as one level of maxdepth. + With withdirs, the directories above the prefix, such as + ``sub`` for ``sub/deep/``, are not included. refresh: If True, bypass the cache and list from S3. Returns: @@ -1458,8 +1475,9 @@ def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs) -> None: ``AbstractFileSystem.mv()`` instead removes ``path1`` by expanding it again, which also deletes copies placed where ``path1`` matches them and files that ``maxdepth`` kept from being copied. A file whose - destination is the file itself is left in place, and directories, - which S3 does not store as objects, are not copied. + destination is the file itself, or the ``null`` version of a file + moved to the file, is left in place, and directories, which S3 does + not store as objects, are not copied. Args: path1: Source S3 path, glob pattern, or list of paths. @@ -1471,8 +1489,9 @@ def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs) -> None: Raises: ValueError: If two sources have the same destination, or a - destination is another source, which is checked before - anything is copied. + destination is another source, including one left in place, + which is checked before anything is copied. A directory with + no object at its key, which is not copied, does not conflict. """ if path1 == path2: return @@ -1497,11 +1516,14 @@ def _move_paths( Returns: The source and destination paths, except the sources whose - destination is the source itself. + destination is the source itself or, for a ``null`` version, the + key of the source. Raises: ValueError: If two sources have the same destination, or a - destination is another source. + destination is another source, including one left in place, + except for a directory with no object at its key, which is not + copied. """ if isinstance(path1, list) and isinstance(path2, list): paths1, paths2 = path1, path2 @@ -1524,19 +1546,63 @@ def _move_paths( ) ) paths2 = other_paths(paths1, path2, exists=exists, flatten=not source_is_str) - # The paths are copied as given, and compared without the protocol. - pairs = [ - (p1, p2) + # The paths are copied as given, and compared by what they name. + named = [ + (p1, p2, self._move_target(p1), self._move_target(p2)) for p1, p2 in zip(paths1, paths2, strict=False) - if self._strip_protocol(p1) != self._strip_protocol(p2) ] - destinations = [self._strip_protocol(p2) for _, p2 in pairs] - if len(set(destinations)) != len(destinations): - raise ValueError("Cannot move several paths to the same destination.") - if {self._strip_protocol(p1) for p1, _ in pairs}.intersection(destinations): - raise ValueError("Cannot move a path onto another path that is moved.") + pairs = [(p1, p2) for p1, p2, source, dest in named if source != dest] + moved = [(p1, source, dest) for p1, _, source, dest in named if source != dest] + # The sources left in place count too; a copy onto one overwrites it. + sources = {source for _, _, source, _ in named} + counts = Counter(dest for _, _, dest in moved) + # A source with another source below it may be a directory. + directories: set[str] = set() + for source in sources: + parent = source.rpartition("/")[0] + while parent and parent not in directories: + directories.add(parent) + parent = parent.rpartition("/")[0] + # A directory without an object at its key is not copied, so it + # writes no destination and is left out of the checks. A path with a + # version always names an object. + writers = [ + (source, dest) + for p1, source, dest in moved + if not ( + (counts[dest] > 1 or dest in sources) + and source in directories + and not self.parse_path(p1)[2] + and self._head_object(source) is None + ) + ] + counts = Counter(dest for _, dest in writers) + for _, dest in writers: + if counts[dest] > 1: + raise ValueError("Cannot move several paths to the same destination.") + if dest in sources: + raise ValueError("Cannot move a path onto another path that is moved.") return pairs + def _move_target(self, path: str) -> str: + """Return what a path of a move names, for comparing the paths. + + A write to a key replaces its ``null`` version, which the objects of a + bucket without versioning have, so that version names the key itself. + + Args: + path: S3 path, possibly with a version ID. + + Returns: + The path in ``bucket/key`` form, with the version ID unless it is + ``null``. + """ + bucket, key, version_id = self.parse_path(path) + target = f"{bucket}/{key}" if key else bucket + if version_id and version_id != "null": + return f"{target}?versionId={version_id}" + return target + def cp_file( self, path1: str, path2: str, recursive=False, maxdepth=None, on_error=None, **kwargs ): diff --git a/pyathena/filesystem/s3_async.py b/pyathena/filesystem/s3_async.py index a7d933bc..16269606 100644 --- a/pyathena/filesystem/s3_async.py +++ b/pyathena/filesystem/s3_async.py @@ -354,8 +354,9 @@ async def _mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs) -> N Raises: ValueError: If two sources have the same destination, or a - destination is another source, which is checked before - anything is copied. + destination is another source, including one left in place, + which is checked before anything is copied. A directory with + no object at its key, which is not copied, does not conflict. """ if path1 == path2: return diff --git a/tests/pyathena/filesystem/test_s3.py b/tests/pyathena/filesystem/test_s3.py index a0948219..2f2dc87e 100644 --- a/tests/pyathena/filesystem/test_s3.py +++ b/tests/pyathena/filesystem/test_s3.py @@ -26,6 +26,7 @@ from fsspec import Callback from fsspec.dircache import DirCache from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.memory import MemoryFileSystem import pyathena from pyathena.filesystem import register_s3_filesystem @@ -735,7 +736,7 @@ def test_find_withdirs_does_not_modify_cached_listing(self): fs = self._make_fs() fs.dircache[("bucket/dir", "")] = [self._file_object("dir/sub/file")] - expected = ["bucket/dir/sub", "bucket/dir/sub/file"] + expected = ["bucket/dir", "bucket/dir/sub", "bucket/dir/sub/file"] assert sorted(fs.find("s3://bucket/dir", withdirs=True)) == expected assert sorted(fs.find("s3://bucket/dir", withdirs=True)) == expected assert fs.find("s3://bucket/dir") == ["bucket/dir/sub/file"] @@ -759,39 +760,196 @@ def test_find_refresh_bypasses_cached_listings(self): # The subdirectory listings of maxdepth are refreshed as well. assert fs.find("s3://bucket/dir", maxdepth=2, refresh=True) == ["bucket/dir/sub/new"] + FIND_KEYS = ("dir/direct", "dir/sub/nested", "dir/sub/deep/file") + + @staticmethod + def _serve_keys(fs, keys): + # Answer the ListObjectsV2, HeadObject, CopyObject, and DeleteObjects + # requests of "bucket" from a set of the given keys, which is returned. + keys = set(keys) + + def call(method, **kwargs): + if method is fs._client.head_object: + if kwargs["Key"] not in keys: + raise FileNotFoundError(kwargs["Key"]) + return {"ContentLength": 0} + if method is fs._client.copy_object: + if kwargs["CopySource"]["Key"] not in keys: + raise FileNotFoundError(kwargs["CopySource"]["Key"]) + keys.add(kwargs["Key"]) + return {} + if method is fs._client.delete_objects: + keys.difference_update(o["Key"] for o in kwargs["Delete"]["Objects"]) + return {} + prefix, delimiter = kwargs["Prefix"], kwargs["Delimiter"] + contents, prefixes = [], set() + for key in sorted(keys): + if not key.startswith(prefix): + continue + rest = key[len(prefix) :] + if delimiter and delimiter in rest: + prefixes.add(prefix + rest.split(delimiter)[0] + delimiter) + else: + contents.append({"Key": key}) + return { + "Contents": contents, + "CommonPrefixes": [{"Prefix": p} for p in sorted(prefixes)], + "KeyCount": len(contents) + len(prefixes), + } + + fs._call.side_effect = call + return keys + + @staticmethod + def _memory_fs(keys): + # Build an fsspec MemoryFileSystem with the keys under "/bucket". + memory = MemoryFileSystem(skip_instance_cache=True) + memory.store = {} + memory.pseudo_dirs = [""] + for key in keys: + memory.pipe(f"/bucket/{key}", b"") + return memory + def test_find_maxdepth_counts_levels_like_fsspec(self): fs = self._make_fs() - responses = { - "dir/": { - "Contents": [{"Key": "dir/direct"}], - "CommonPrefixes": [{"Prefix": "dir/sub/"}], - }, - "dir/sub/": { - "Contents": [{"Key": "dir/sub/nested"}], - "CommonPrefixes": [{"Prefix": "dir/sub/deep/"}], - }, - "dir/sub/deep/": {"Contents": [{"Key": "dir/sub/deep/file"}]}, - } - fs._call.side_effect = lambda method, **kwargs: responses[kwargs["Prefix"]] + self._serve_keys(fs, self.FIND_KEYS) with pytest.raises(ValueError, match="maxdepth must be at least 1"): fs.find("s3://bucket/dir", maxdepth=0) fs._call.assert_not_called() assert fs.find("s3://bucket/dir", maxdepth=1) == ["bucket/dir/direct"] - assert sorted(fs.find("s3://bucket/dir", maxdepth=1, withdirs=True)) == [ + assert sorted(fs.find("s3://bucket/dir", maxdepth=2)) == [ "bucket/dir/direct", - "bucket/dir/sub", + "bucket/dir/sub/nested", ] + assert sorted(fs.find("s3://bucket/dir", maxdepth=3)) == [ + "bucket/dir/direct", + "bucket/dir/sub/deep/file", + "bucket/dir/sub/nested", + ] + + @pytest.mark.parametrize( + ("path", "maxdepth", "withdirs"), + [ + ("dir", None, True), + ("dir", None, False), + ("dir", 1, True), + ("dir", 2, True), + ("dir", 1, False), + ("dir/sub", 1, True), + ("dir/direct", None, True), + ("dir/direct", 1, True), + ("dir/direct", 1, False), + ("missing", None, True), + ("missing", 1, True), + ], + ) + def test_find_matches_fsspec(self, path, maxdepth, withdirs): + fs = self._make_fs() + self._serve_keys(fs, self.FIND_KEYS) + memory = self._memory_fs(self.FIND_KEYS) + + expected = [p.lstrip("/") for p in memory.find(f"/bucket/{path}", maxdepth, withdirs)] + assert sorted(fs.find(f"s3://bucket/{path}", maxdepth, withdirs)) == expected + + @pytest.mark.parametrize( + ("pattern", "maxdepth"), + [ + ("dir/**", None), + ("dir/**", 1), + ("dir/**", 2), + ("dir/*", None), + ("dir/*/*", None), + ("dir/s*", None), + ("dir/**/file", None), + ("missing/*", None), + ], + ) + def test_glob_matches_fsspec(self, pattern, maxdepth): + fs = self._make_fs() + self._serve_keys(fs, self.FIND_KEYS) + memory = self._memory_fs(self.FIND_KEYS) + + expected = [p.lstrip("/") for p in memory.glob(f"/bucket/{pattern}", maxdepth=maxdepth)] + assert sorted(fs.glob(f"s3://bucket/{pattern}", maxdepth=maxdepth)) == expected + + def test_find_withdirs_omits_bucket(self): + fs = self._make_fs() + self._serve_keys(fs, self.FIND_KEYS) + + # A recursive copy of the expanded paths cannot copy a bucket. + assert "bucket" not in fs.find("s3://bucket", withdirs=True) + assert "bucket" not in fs.find("s3://bucket", maxdepth=1, withdirs=True) + assert "bucket" not in fs.expand_path("s3://bucket/**", recursive=True) + + def test_find_directory_without_extra_requests(self): + fs = self._make_fs() + self._serve_keys(fs, self.FIND_KEYS) + + assert "bucket/dir" in fs.find("s3://bucket/dir", withdirs=True) + assert fs._call.call_count == 1 + fs._call.reset_mock() + assert "bucket/dir" in fs.find("s3://bucket/dir", maxdepth=1, withdirs=True) + assert fs._call.call_count == 1 + # Only a subdirectory is listed; it is dropped without withdirs, but + # the path is a directory, so it is not looked up as an object. + fs._call.reset_mock() + assert fs.find("s3://bucket/dir", maxdepth=1, prefix="s") == [] + assert fs._call.call_count == 1 + + def test_find_maxdepth_listings_follow_invalidation(self): + fs = self._make_fs() + self._serve_keys(fs, ("dir/direct", "dir/sub/nested")) assert sorted(fs.find("s3://bucket/dir", maxdepth=2)) == [ "bucket/dir/direct", "bucket/dir/sub/nested", ] - assert sorted(fs.find("s3://bucket/dir", maxdepth=3)) == [ + + # A write below the subdirectory invalidates its cached listing. + self._serve_keys(fs, ("dir/direct", "dir/sub/nested", "dir/sub/new")) + fs.invalidate_cache("s3://bucket/dir/sub/new") + assert sorted(fs.find("s3://bucket/dir", maxdepth=2)) == [ "bucket/dir/direct", + "bucket/dir/sub/nested", + "bucket/dir/sub/new", + ] + + def test_find_prefix_counts_levels_from_path(self): + fs = self._make_fs() + self._serve_keys(fs, self.FIND_KEYS) + + # Nothing directly under dir/ starts with "sub/deep/". + assert fs.find("s3://bucket/dir", maxdepth=1, prefix="sub/deep/") == [] + assert fs.find("s3://bucket/dir", maxdepth=2, prefix="sub/deep/") == [] + assert fs.find("s3://bucket/dir", maxdepth=3, prefix="sub/deep/") == [ + "bucket/dir/sub/deep/file" + ] + assert sorted(fs.find("s3://bucket/dir", maxdepth=2, prefix="sub/")) == [ + "bucket/dir/sub/nested" + ] + # The directories above the prefix do not start with it, with or + # without maxdepth. + expected = [ + "bucket/dir", + "bucket/dir/sub/deep", "bucket/dir/sub/deep/file", "bucket/dir/sub/nested", ] + assert sorted(fs.find("s3://bucket/dir", maxdepth=3, prefix="sub/", withdirs=True)) == ( + expected + ) + assert sorted(fs.find("s3://bucket/dir", prefix="sub/", withdirs=True)) == expected + + @pytest.mark.parametrize("maxdepth", [None, 1]) + def test_find_object_path_ignores_prefix(self, maxdepth): + fs = self._make_fs() + self._serve_keys(fs, self.FIND_KEYS) + + # As in fsspec, the object itself is returned when nothing is listed. + assert fs.find("s3://bucket/dir/direct", maxdepth=maxdepth, prefix="x") == [ + "bucket/dir/direct" + ] def test_refresh_evicts_cached_object_and_bucket_not_found(self): fs = self._make_fs() @@ -1359,6 +1517,13 @@ def test_cp_file_directory(self): (["s3://bucket/a", "s3://bucket/b"], ["s3://bucket/c", "s3a://bucket/c"]), # A destination that is another source. (["s3://bucket/a", "s3://bucket/b"], ["s3://bucket/b", "s3://bucket/a"]), + # A destination that is a source left in place. + (["s3://bucket/a", "s3://bucket/b"], ["s3://bucket/b", "s3://bucket/b"]), + # A destination whose "null" version is another source. + ( + ["s3://bucket/a", "s3://bucket/b?versionId=null"], + ["s3://bucket/b", "s3://bucket/out"], + ), ], ) def test_mv_conflicting_destinations(self, path1, path2): @@ -1391,6 +1556,94 @@ def test_mv_keeps_given_paths(self): fs._copy_file.assert_called_once_with("s3://bucket/src", "s3://bucket/dst/") fs._delete_objects.assert_called_once_with(["s3://bucket/src"]) + @pytest.mark.parametrize( + ("keys", "path1", "path2", "expected"), + [ + # The directory itself, which find() includes, is not copied. + ({"d/a", "d/b"}, "s3://bucket/d/**", "s3://bucket/out/", {"out/a", "out/b"}), + # The directory moved onto an existing subdirectory is no conflict. + ( + {"src/a", "src/archive/x"}, + "s3://bucket/src/**", + "s3://bucket/src/archive/", + {"src/archive/a", "src/archive/archive/x"}, + ), + # A directory shares its destination with an object. + ( + {"d/x", "e/y"}, + ["s3://bucket/d", "s3://bucket/d/x", "s3://bucket/e/y"], + ["s3://bucket/e", "s3://bucket/e", "s3://bucket/out"], + {"e", "out"}, + ), + ], + ) + def test_mv_glob_with_directories(self, keys, path1, path2, expected): + fs = self._make_fs() + store = self._serve_keys(fs, keys) + + fs.mv(path1, path2, recursive=True) + assert store == expected + + @pytest.mark.parametrize( + ("keys", "path2"), + [ + # Onto another source that is only an object. + ({"d", "d/x", "e"}, ["e", "o/x", "o/e"]), + # Onto another source that also has keys below it. + ({"d", "d/x", "d/x/y"}, ["d/x", "o", "o/y"]), + ], + ) + def test_mv_objects_with_keys_below_conflict(self, keys, path2): + # An object that also has keys below it is copied, so moving it onto + # another source raises before anything is copied. + fs = self._make_fs() + store = self._serve_keys(fs, keys) + + with pytest.raises(ValueError, match="another path that is moved"): + fs.mv( + [f"s3://bucket/{k}" for k in sorted(keys)], + [f"s3://bucket/{k}" for k in path2], + recursive=True, + ) + assert store == keys + methods = {c.args[0] for c in fs._call.call_args_list} + assert fs._client.copy_object not in methods + assert fs._client.delete_objects not in methods + + def test_mv_version_with_keys_below_conflicts(self): + # A version names an object even with keys below its key, so it is + # not taken for a directory when no current object exists at the key. + fs = self._make_fs() + self._serve_keys(fs, {"d/x", "a"}) + + with pytest.raises(ValueError, match="same destination"): + fs.mv( + ["s3://bucket/d?versionId=null", "s3://bucket/d/x", "s3://bucket/a"], + ["s3://bucket/out", "s3://bucket/x", "s3://bucket/out"], + ) + methods = {c.args[0] for c in fs._call.call_args_list} + assert fs._client.copy_object not in methods + assert fs._client.delete_objects not in methods + + def test_mv_versions_onto_their_key(self): + fs = self._make_fs() + self._serve_keys(fs, {"b"}) + + # The "null" version is the object at the key, so it stays in place. + fs.mv(["s3://bucket/b?versionId=null"], ["s3://bucket/b"]) + fs._call.assert_not_called() + + # Another version is copied onto the key, and then deleted. + fs.mv(["s3://bucket/b?versionId=v1"], ["s3://bucket/b"]) + copies = [c.kwargs for c in fs._call.call_args_list if c.args[0] is fs._client.copy_object] + assert [(c["CopySource"].get("VersionId"), c["Key"]) for c in copies] == [("v1", "b")] + deletes = [ + c.kwargs["Delete"]["Objects"] + for c in fs._call.call_args_list + if c.args[0] is fs._client.delete_objects + ] + assert deletes == [[{"Key": "b", "VersionId": "v1"}]] + def test_mv_nothing_within_maxdepth(self): # Only directories within maxdepth: nothing is moved, as with copy(). fs = self._make_fs() @@ -3433,6 +3686,15 @@ def test_find_maxdepth(self, fs): result = fs.find(dir_) assert len(result) == 4 + # Each slash in the prefix counts as one level + assert fs.find(dir_, maxdepth=1, prefix="level1/") == [] + assert fs.find(dir_, maxdepth=2, prefix="level1/") == [ + fs._strip_protocol(f"{dir_}/level1/file1.txt") + ] + + # An object path returns the object itself + assert fs.find(f"{dir_}/file0.txt", maxdepth=1) == [fs._strip_protocol(f"{dir_}/file0.txt")] + def test_find_withdirs(self, fs): dir_ = f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/filesystem/test_find_withdirs" # Create directory structure with files @@ -3450,6 +3712,8 @@ def test_find_withdirs(self, fs): # Test withdirs=True result = fs.find(dir_, withdirs=True) assert len(result) > 4 # Files and directories + assert fs._strip_protocol(dir_) in result + assert fs._strip_protocol(dir_) in fs.find(dir_, maxdepth=1, withdirs=True) # Verify directories are included dirs = [r for r in result if not r.endswith(".txt")] @@ -3478,6 +3742,7 @@ def test_glob(self, fs): assert fs._strip_protocol(path) in fs.glob(f"{dir_}/nested/*") assert fs._strip_protocol(path) in fs.glob(f"{dir_}/nested/test_*") assert fs._strip_protocol(path) in fs.glob(f"{dir_}/*/*") + assert fs._strip_protocol(f"{dir_}/nested") in fs.glob(f"{dir_}/nested/**") with pytest.raises(ValueError): # noqa: PT011 fs.glob("*") diff --git a/tests/pyathena/filesystem/test_s3_async.py b/tests/pyathena/filesystem/test_s3_async.py index a5b2367b..1f5fc684 100644 --- a/tests/pyathena/filesystem/test_s3_async.py +++ b/tests/pyathena/filesystem/test_s3_async.py @@ -1057,6 +1057,7 @@ async def test_glob(self, fs): assert fs._strip_protocol(path) in fs.glob(f"{dir_}/nested/*") assert fs._strip_protocol(path) in fs.glob(f"{dir_}/nested/test_*") assert fs._strip_protocol(path) in fs.glob(f"{dir_}/*/*") + assert fs._strip_protocol(f"{dir_}/nested") in fs.glob(f"{dir_}/nested/**") with pytest.raises(ValueError): # noqa: PT011 fs.glob("*")