From bda452020c5b7191618f7c2c639fdfebe0f5bc72 Mon Sep 17 00:00:00 2001 From: David Banquet Date: Mon, 16 Nov 2026 21:33:08 +0100 Subject: [PATCH] Use the botocore_session credentials in PyArrowFileIO for S3 The botocore_session catalog property reached the catalog clients but not the FileIO, which fell back to the default credential chain and only failed on the first data file write. When no explicit S3 credentials or role are set, the S3 filesystem now takes the session's frozen credentials, and they are part of the fs_by_scheme cache key so refreshed credentials get a new filesystem. Closes #2657 Co-Authored-By: Claude Opus 5.5 (1M context) --- pyiceberg/catalog/__init__.py | 2 +- pyiceberg/io/__init__.py | 1 + pyiceberg/io/pyarrow.py | 38 ++++++++++++++++++++++----- tests/io/test_pyarrow.py | 48 ++++++++++++++++++++++++++++++++++- 4 files changed, 80 insertions(+), 9 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index ce0902f338..9969a8a35c 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -40,6 +40,7 @@ NotInstalledError, TableAlreadyExistsError, ) +from pyiceberg.io import BOTOCORE_SESSION as BOTOCORE_SESSION from pyiceberg.io import FileIO, load_file_io from pyiceberg.manifest import ManifestFile from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec @@ -96,7 +97,6 @@ URI = "uri" LOCATION = "location" EXTERNAL_TABLE = "EXTERNAL_TABLE" -BOTOCORE_SESSION = "botocore_session" DEFAULT_ROLE_SESSION_NAME = "pyiceberg" diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index e433ed88a5..a024bc6e36 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -54,6 +54,7 @@ def _is_local_path(path: str) -> bool: return drive != "" +BOTOCORE_SESSION = "botocore_session" AWS_PROFILE_NAME = "client.profile-name" AWS_REGION = "client.region" AWS_ACCESS_KEY_ID = "client.access-key-id" diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..756631ff43 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -100,6 +100,7 @@ AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + BOTOCORE_SESSION, GCS_DEFAULT_LOCATION, GCS_SERVICE_HOST, GCS_TOKEN, @@ -405,7 +406,7 @@ def to_input_file(self) -> PyArrowFile: class PyArrowFileIO(_TransientStateMixin, FileIO): - fs_by_scheme: Callable[[str, str | None], FileSystem] + fs_by_scheme: Callable[..., FileSystem] _transient_attrs = ("fs_by_scheme",) def __init__(self, properties: Properties = EMPTY_DICT): @@ -444,13 +445,15 @@ def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[ else: return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}" - def _initialize_fs(self, scheme: str, netloc: str | None = None) -> FileSystem: + def _initialize_fs( + self, scheme: str, netloc: str | None = None, session_credentials: tuple[str, str, str | None] | None = None + ) -> FileSystem: """Initialize FileSystem for different scheme.""" if scheme in {"oss"}: return self._initialize_oss_fs() elif scheme in {"s3", "s3a", "s3n"}: - return self._initialize_s3_fs(netloc) + return self._initialize_s3_fs(netloc, session_credentials) elif scheme in {"hdfs", "viewfs"}: return self._initialize_hdfs_fs(scheme, netloc) @@ -513,7 +516,25 @@ def _initialize_oss_fs(self) -> FileSystem: return S3FileSystem(**client_kwargs) - def _initialize_s3_fs(self, netloc: str | None) -> FileSystem: + def _botocore_session_credentials(self) -> tuple[str, str, str | None] | None: + """Return the frozen credentials of the botocore session when no explicit S3 credentials or role are set.""" + session: Any = self.properties.get(BOTOCORE_SESSION) + if session is None or get_first_property_value(self.properties, S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID) is not None: + return None + if get_first_property_value(self.properties, S3_ROLE_ARN, AWS_ASSUME_ROLE_ARN, AWS_ROLE_ARN) is not None: + return None + if (credentials := session.get_credentials()) is None: + return None + # Refreshes the credentials when they are about to expire, which changes the fs_by_scheme cache key + frozen = credentials.get_frozen_credentials() + return frozen.access_key, frozen.secret_key, frozen.token + + def _filesystem(self, scheme: str, netloc: str | None) -> FileSystem: + if scheme in {"s3", "s3a", "s3n"} and (session_credentials := self._botocore_session_credentials()) is not None: + return self.fs_by_scheme(scheme, netloc, session_credentials) + return self.fs_by_scheme(scheme, netloc) + + def _initialize_s3_fs(self, netloc: str | None, session_credentials: tuple[str, str, str | None] | None = None) -> FileSystem: from pyarrow.fs import S3FileSystem provided_region = get_first_property_value(self.properties, S3_REGION, AWS_REGION) @@ -539,6 +560,9 @@ def _initialize_s3_fs(self, netloc: str | None) -> FileSystem: "region": bucket_region, } + if session_credentials is not None: + client_kwargs["access_key"], client_kwargs["secret_key"], client_kwargs["session_token"] = session_credentials + if proxy_uri := self.properties.get(S3_PROXY_URI): client_kwargs["proxy_options"] = proxy_uri @@ -681,7 +705,7 @@ def new_input(self, location: str) -> PyArrowFile: """ scheme, netloc, path = self.parse_location(location, self.properties) return PyArrowFile( - fs=self.fs_by_scheme(scheme, netloc), + fs=self._filesystem(scheme, netloc), location=location, path=path, buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE)), @@ -699,7 +723,7 @@ def new_output(self, location: str) -> PyArrowFile: """ scheme, netloc, path = self.parse_location(location, self.properties) return PyArrowFile( - fs=self.fs_by_scheme(scheme, netloc), + fs=self._filesystem(scheme, netloc), location=location, path=path, buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE)), @@ -721,7 +745,7 @@ def delete(self, location: str | InputFile | OutputFile) -> None: """ str_location = location.location if isinstance(location, (InputFile, OutputFile)) else location scheme, netloc, path = self.parse_location(str_location, self.properties) - fs = self.fs_by_scheme(scheme, netloc) + fs = self._filesystem(scheme, netloc) try: fs.delete_file(path) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..cb074861d6 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -75,7 +75,18 @@ Or, ) from pyiceberg.expressions.literals import literal -from pyiceberg.io import ADLS_ACCOUNT_NAME, S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io +from pyiceberg.io import ( + ADLS_ACCOUNT_NAME, + BOTOCORE_SESSION, + S3_ACCESS_KEY_ID, + S3_REGION, + S3_RETRY_STRATEGY_IMPL, + S3_SECRET_ACCESS_KEY, + S3_SSL_CA_CERT, + InputStream, + OutputStream, + load_file_io, +) from pyiceberg.io.pyarrow import ( ICEBERG_SCHEMA, PYARROW_PARQUET_FIELD_ID_KEY, @@ -4196,6 +4207,41 @@ def test_pyarrow_file_io_fs_by_scheme_cache() -> None: assert pyarrow_file_io.fs_by_scheme.cache_info().hits == 2 # type: ignore +def test_pyarrow_file_io_botocore_session_credentials() -> None: + import botocore.session + + session = botocore.session.get_session() + session.set_credentials("session-access-key", "session-secret-key", "session-token") + explicit_credentials = {S3_ACCESS_KEY_ID: "explicit-access-key", S3_SECRET_ACCESS_KEY: "explicit-secret-key"} + cases: list[tuple[dict[str, Any], tuple[str | None, str | None, str | None]]] = [ + ({}, ("session-access-key", "session-secret-key", "session-token")), + (explicit_credentials, ("explicit-access-key", "explicit-secret-key", None)), + ] + for properties, expected in cases: + file_io = PyArrowFileIO({**properties, BOTOCORE_SESSION: session, S3_REGION: "us-east-1"}) + with patch("pyarrow.fs.S3FileSystem") as mock_s3_fs: + file_io.new_input("s3://bucket/key") + kwargs = mock_s3_fs.call_args.kwargs + assert (kwargs["access_key"], kwargs["secret_key"], kwargs["session_token"]) == expected + + +def test_pyarrow_file_io_botocore_session_refreshed_credentials() -> None: + import botocore.session + + session = botocore.session.get_session() + session.set_credentials("first-access-key", "first-secret-key", "first-token") + file_io = PyArrowFileIO({BOTOCORE_SESSION: session, S3_REGION: "us-east-1"}) + + with patch("pyarrow.fs.S3FileSystem") as mock_s3_fs: + file_io.new_input("s3://bucket/a") + file_io.new_input("s3://bucket/b") + session.set_credentials("second-access-key", "second-secret-key", "second-token") + file_io.new_input("s3://bucket/c") + + assert [call.kwargs["access_key"] for call in mock_s3_fs.call_args_list] == ["first-access-key", "second-access-key"] + assert file_io.fs_by_scheme.cache_info().hits == 1 # type: ignore + + def test_pyarrow_io_new_input_multi_region(caplog: Any) -> None: # It's better to set up multi-region minio servers for an integration test once `endpoint_url` argument # becomes available for `resolve_s3_region`