diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index ce0902f338..9969a8a35c 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -40,6 +40,7 @@ NotInstalledError, TableAlreadyExistsError, ) +from pyiceberg.io import BOTOCORE_SESSION as BOTOCORE_SESSION from pyiceberg.io import FileIO, load_file_io from pyiceberg.manifest import ManifestFile from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec @@ -96,7 +97,6 @@ URI = "uri" LOCATION = "location" EXTERNAL_TABLE = "EXTERNAL_TABLE" -BOTOCORE_SESSION = "botocore_session" DEFAULT_ROLE_SESSION_NAME = "pyiceberg" diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index e433ed88a5..a024bc6e36 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -54,6 +54,7 @@ def _is_local_path(path: str) -> bool: return drive != "" +BOTOCORE_SESSION = "botocore_session" AWS_PROFILE_NAME = "client.profile-name" AWS_REGION = "client.region" AWS_ACCESS_KEY_ID = "client.access-key-id" diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..756631ff43 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -100,6 +100,7 @@ AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + BOTOCORE_SESSION, GCS_DEFAULT_LOCATION, GCS_SERVICE_HOST, GCS_TOKEN, @@ -405,7 +406,7 @@ def to_input_file(self) -> PyArrowFile: class PyArrowFileIO(_TransientStateMixin, FileIO): - fs_by_scheme: Callable[[str, str | None], FileSystem] + fs_by_scheme: Callable[..., FileSystem] _transient_attrs = ("fs_by_scheme",) def __init__(self, properties: Properties = EMPTY_DICT): @@ -444,13 +445,15 @@ def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[ else: return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}" - def _initialize_fs(self, scheme: str, netloc: str | None = None) -> FileSystem: + def _initialize_fs( + self, scheme: str, netloc: str | None = None, session_credentials: tuple[str, str, str | None] | None = None + ) -> FileSystem: """Initialize FileSystem for different scheme.""" if scheme in {"oss"}: return self._initialize_oss_fs() elif scheme in {"s3", "s3a", "s3n"}: - return self._initialize_s3_fs(netloc) + return self._initialize_s3_fs(netloc, session_credentials) elif scheme in {"hdfs", "viewfs"}: return self._initialize_hdfs_fs(scheme, netloc) @@ -513,7 +516,25 @@ def _initialize_oss_fs(self) -> FileSystem: return S3FileSystem(**client_kwargs) - def _initialize_s3_fs(self, netloc: str | None) -> FileSystem: + def _botocore_session_credentials(self) -> tuple[str, str, str | None] | None: + """Return the frozen credentials of the botocore session when no explicit S3 credentials or role are set.""" + session: Any = self.properties.get(BOTOCORE_SESSION) + if session is None or get_first_property_value(self.properties, S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID) is not None: + return None + if get_first_property_value(self.properties, S3_ROLE_ARN, AWS_ASSUME_ROLE_ARN, AWS_ROLE_ARN) is not None: + return None + if (credentials := session.get_credentials()) is None: + return None + # Refreshes the credentials when they are about to expire, which changes the fs_by_scheme cache key + frozen = credentials.get_frozen_credentials() + return frozen.access_key, frozen.secret_key, frozen.token + + def _filesystem(self, scheme: str, netloc: str | None) -> FileSystem: + if scheme in {"s3", "s3a", "s3n"} and (session_credentials := self._botocore_session_credentials()) is not None: + return self.fs_by_scheme(scheme, netloc, session_credentials) + return self.fs_by_scheme(scheme, netloc) + + def _initialize_s3_fs(self, netloc: str | None, session_credentials: tuple[str, str, str | None] | None = None) -> FileSystem: from pyarrow.fs import S3FileSystem provided_region = get_first_property_value(self.properties, S3_REGION, AWS_REGION) @@ -539,6 +560,9 @@ def _initialize_s3_fs(self, netloc: str | None) -> FileSystem: "region": bucket_region, } + if session_credentials is not None: + client_kwargs["access_key"], client_kwargs["secret_key"], client_kwargs["session_token"] = session_credentials + if proxy_uri := self.properties.get(S3_PROXY_URI): client_kwargs["proxy_options"] = proxy_uri @@ -681,7 +705,7 @@ def new_input(self, location: str) -> PyArrowFile: """ scheme, netloc, path = self.parse_location(location, self.properties) return PyArrowFile( - fs=self.fs_by_scheme(scheme, netloc), + fs=self._filesystem(scheme, netloc), location=location, path=path, buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE)), @@ -699,7 +723,7 @@ def new_output(self, location: str) -> PyArrowFile: """ scheme, netloc, path = self.parse_location(location, self.properties) return PyArrowFile( - fs=self.fs_by_scheme(scheme, netloc), + fs=self._filesystem(scheme, netloc), location=location, path=path, buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE)), @@ -721,7 +745,7 @@ def delete(self, location: str | InputFile | OutputFile) -> None: """ str_location = location.location if isinstance(location, (InputFile, OutputFile)) else location scheme, netloc, path = self.parse_location(str_location, self.properties) - fs = self.fs_by_scheme(scheme, netloc) + fs = self._filesystem(scheme, netloc) try: fs.delete_file(path) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..cb074861d6 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -75,7 +75,18 @@ Or, ) from pyiceberg.expressions.literals import literal -from pyiceberg.io import ADLS_ACCOUNT_NAME, S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io +from pyiceberg.io import ( + ADLS_ACCOUNT_NAME, + BOTOCORE_SESSION, + S3_ACCESS_KEY_ID, + S3_REGION, + S3_RETRY_STRATEGY_IMPL, + S3_SECRET_ACCESS_KEY, + S3_SSL_CA_CERT, + InputStream, + OutputStream, + load_file_io, +) from pyiceberg.io.pyarrow import ( ICEBERG_SCHEMA, PYARROW_PARQUET_FIELD_ID_KEY, @@ -4196,6 +4207,41 @@ def test_pyarrow_file_io_fs_by_scheme_cache() -> None: assert pyarrow_file_io.fs_by_scheme.cache_info().hits == 2 # type: ignore +def test_pyarrow_file_io_botocore_session_credentials() -> None: + import botocore.session + + session = botocore.session.get_session() + session.set_credentials("session-access-key", "session-secret-key", "session-token") + explicit_credentials = {S3_ACCESS_KEY_ID: "explicit-access-key", S3_SECRET_ACCESS_KEY: "explicit-secret-key"} + cases: list[tuple[dict[str, Any], tuple[str | None, str | None, str | None]]] = [ + ({}, ("session-access-key", "session-secret-key", "session-token")), + (explicit_credentials, ("explicit-access-key", "explicit-secret-key", None)), + ] + for properties, expected in cases: + file_io = PyArrowFileIO({**properties, BOTOCORE_SESSION: session, S3_REGION: "us-east-1"}) + with patch("pyarrow.fs.S3FileSystem") as mock_s3_fs: + file_io.new_input("s3://bucket/key") + kwargs = mock_s3_fs.call_args.kwargs + assert (kwargs["access_key"], kwargs["secret_key"], kwargs["session_token"]) == expected + + +def test_pyarrow_file_io_botocore_session_refreshed_credentials() -> None: + import botocore.session + + session = botocore.session.get_session() + session.set_credentials("first-access-key", "first-secret-key", "first-token") + file_io = PyArrowFileIO({BOTOCORE_SESSION: session, S3_REGION: "us-east-1"}) + + with patch("pyarrow.fs.S3FileSystem") as mock_s3_fs: + file_io.new_input("s3://bucket/a") + file_io.new_input("s3://bucket/b") + session.set_credentials("second-access-key", "second-secret-key", "second-token") + file_io.new_input("s3://bucket/c") + + assert [call.kwargs["access_key"] for call in mock_s3_fs.call_args_list] == ["first-access-key", "second-access-key"] + assert file_io.fs_by_scheme.cache_info().hits == 1 # type: ignore + + def test_pyarrow_io_new_input_multi_region(caplog: Any) -> None: # It's better to set up multi-region minio servers for an integration test once `endpoint_url` argument # becomes available for `resolve_s3_region`