diff --git a/docs/filesystem.md b/docs/filesystem.md index fd2439ead..719603257 100644 --- a/docs/filesystem.md +++ b/docs/filesystem.md @@ -54,6 +54,9 @@ fs = S3FileSystem(connect(s3_staging_dir="s3://YOUR_S3_BUCKET/path/to/", # Or with direct credentials (s3fs-compatible arguments). fs = S3FileSystem(key="YOUR_ACCESS_KEY", secret="YOUR_SECRET_KEY") +# Or with a named profile. +fs = S3FileSystem(profile="YOUR_PROFILE") + # Or anonymously for public buckets. fs = S3FileSystem(anon=True) ``` diff --git a/pyathena/filesystem/s3.py b/pyathena/filesystem/s3.py index 2819a99dd..19fe08aad 100644 --- a/pyathena/filesystem/s3.py +++ b/pyathena/filesystem/s3.py @@ -205,10 +205,11 @@ def _get_client_compatible_with_s3fs(self, **kwargs) -> BaseClient: Accepts the constructor arguments that s3fs users pass through fsspec storage options — ``key``/``username``, ``secret``/``password``, - ``token``, ``anon``, ``use_ssl``, ``endpoint_url``, + ``token``, ``profile``, ``anon``, ``use_ssl``, ``endpoint_url``, ``connect_timeout``/``read_timeout``, and the ``client_kwargs`` / ``config_kwargs`` dictionaries — in addition to boto3 session - arguments such as ``region_name`` and ``profile_name``. + arguments such as ``region_name`` and ``profile_name``. ``profile`` + is used as ``profile_name`` when ``profile_name`` is not given. Args: **kwargs: The filesystem constructor arguments. @@ -247,6 +248,8 @@ def _get_client_compatible_with_s3fs(self, **kwargs) -> BaseClient: } kwargs.update(creds) client_kwargs.update(creds) + if profile := kwargs.pop("profile", None): + kwargs.setdefault("profile_name", profile) session = Session( **{k: v for k, v in kwargs.items() if k in Connection._SESSION_PASSING_ARGS} diff --git a/tests/pyathena/filesystem/test_s3.py b/tests/pyathena/filesystem/test_s3.py index 762539297..c04d3a4e4 100644 --- a/tests/pyathena/filesystem/test_s3.py +++ b/tests/pyathena/filesystem/test_s3.py @@ -205,6 +205,39 @@ def test_get_client_compatible_with_s3fs(self): ) assert fs._client.meta.endpoint_url == "http://localhost:9000" + @pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({}, "DEFAULTKEY"), + # s3fs names the boto3 profile_name argument "profile". + ({"profile": "other"}, "OTHERKEY"), + ({"profile_name": "other"}, "OTHERKEY"), + ({"profile": "other", "profile_name": "default"}, "DEFAULTKEY"), + ], + ) + def test_get_client_compatible_with_s3fs_profile(self, monkeypatch, tmp_path, kwargs, expected): + # Only constructs a boto3 client from local profile files; no AWS access. + config = tmp_path / "config" + config.write_text("[default]\n[profile other]\n") + credentials = tmp_path / "credentials" + credentials.write_text( + "[default]\naws_access_key_id = DEFAULTKEY\naws_secret_access_key = secret\n" + "[other]\naws_access_key_id = OTHERKEY\naws_secret_access_key = secret\n" + ) + for name in ( + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AWS_CONFIG_FILE", str(config)) + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(credentials)) + + fs = S3FileSystem(region_name="us-east-1", skip_instance_cache=True, **kwargs) + assert fs._client._request_signer._credentials.access_key == expected + def test_ls_from_cache_with_cached_object(self): fs = self._make_fs() obj = S3Object(