From 9cc6d013516cb94011ac159d99eddee851f8ea24 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sat, 3 Oct 2026 16:56:26 +0900 Subject: [PATCH] Use the s3fs profile argument as the boto3 profile name S3FileSystem accepted s3fs's constructor arguments without a connection, but dropped s3fs's profile argument because boto3 names it profile_name, so requests were signed with the default credential chain. Map profile to profile_name when profile_name is not given. Closes #983 Co-Authored-By: Claude Opus 5.5 --- docs/filesystem.md | 3 +++ pyathena/filesystem/s3.py | 7 ++++-- tests/pyathena/filesystem/test_s3.py | 33 ++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/filesystem.md b/docs/filesystem.md index b19f93d43..31fc3d7b2 100644 --- a/docs/filesystem.md +++ b/docs/filesystem.md @@ -54,6 +54,9 @@ fs = S3FileSystem(connect(s3_staging_dir="s3://YOUR_S3_BUCKET/path/to/", # Or with direct credentials (s3fs-compatible arguments). fs = S3FileSystem(key="YOUR_ACCESS_KEY", secret="YOUR_SECRET_KEY") +# Or with a named profile. +fs = S3FileSystem(profile="YOUR_PROFILE") + # Or anonymously for public buckets. fs = S3FileSystem(anon=True) ``` diff --git a/pyathena/filesystem/s3.py b/pyathena/filesystem/s3.py index 3d039832c..e9565cefd 100644 --- a/pyathena/filesystem/s3.py +++ b/pyathena/filesystem/s3.py @@ -199,10 +199,11 @@ def _get_client_compatible_with_s3fs(self, **kwargs) -> BaseClient: Accepts the constructor arguments that s3fs users pass through fsspec storage options — ``key``/``username``, ``secret``/``password``, - ``token``, ``anon``, ``use_ssl``, ``endpoint_url``, + ``token``, ``profile``, ``anon``, ``use_ssl``, ``endpoint_url``, ``connect_timeout``/``read_timeout``, and the ``client_kwargs`` / ``config_kwargs`` dictionaries — in addition to boto3 session - arguments such as ``region_name`` and ``profile_name``. + arguments such as ``region_name`` and ``profile_name``. ``profile`` + is used as ``profile_name`` when ``profile_name`` is not given. Args: **kwargs: The filesystem constructor arguments. @@ -241,6 +242,8 @@ def _get_client_compatible_with_s3fs(self, **kwargs) -> BaseClient: } kwargs.update(creds) client_kwargs.update(creds) + if profile := kwargs.pop("profile", None): + kwargs.setdefault("profile_name", profile) session = Session( **{k: v for k, v in kwargs.items() if k in Connection._SESSION_PASSING_ARGS} diff --git a/tests/pyathena/filesystem/test_s3.py b/tests/pyathena/filesystem/test_s3.py index 2131eb1e3..52fabb9dc 100644 --- a/tests/pyathena/filesystem/test_s3.py +++ b/tests/pyathena/filesystem/test_s3.py @@ -181,6 +181,39 @@ def test_get_client_compatible_with_s3fs(self): ) assert fs._client.meta.endpoint_url == "http://localhost:9000" + @pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({}, "DEFAULTKEY"), + # s3fs names the boto3 profile_name argument "profile". + ({"profile": "other"}, "OTHERKEY"), + ({"profile_name": "other"}, "OTHERKEY"), + ({"profile": "other", "profile_name": "default"}, "DEFAULTKEY"), + ], + ) + def test_get_client_compatible_with_s3fs_profile(self, monkeypatch, tmp_path, kwargs, expected): + # Only constructs a boto3 client from local profile files; no AWS access. + config = tmp_path / "config" + config.write_text("[default]\n[profile other]\n") + credentials = tmp_path / "credentials" + credentials.write_text( + "[default]\naws_access_key_id = DEFAULTKEY\naws_secret_access_key = secret\n" + "[other]\naws_access_key_id = OTHERKEY\naws_secret_access_key = secret\n" + ) + for name in ( + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AWS_CONFIG_FILE", str(config)) + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(credentials)) + + fs = S3FileSystem(region_name="us-east-1", skip_instance_cache=True, **kwargs) + assert fs._client._request_signer._credentials.access_key == expected + def test_ls_from_cache_with_cached_object(self): fs = self._make_fs() obj = S3Object(