Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyiceberg/catalog/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
NotInstalledError,
TableAlreadyExistsError,
)
from pyiceberg.io import BOTOCORE_SESSION as BOTOCORE_SESSION
from pyiceberg.io import FileIO, load_file_io
from pyiceberg.manifest import ManifestFile
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec
Expand Down Expand Up @@ -96,7 +97,6 @@
URI = "uri"
LOCATION = "location"
EXTERNAL_TABLE = "EXTERNAL_TABLE"
BOTOCORE_SESSION = "botocore_session"

DEFAULT_ROLE_SESSION_NAME = "pyiceberg"

Expand Down
1 change: 1 addition & 0 deletions pyiceberg/io/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ def _is_local_path(path: str) -> bool:
return drive != ""


BOTOCORE_SESSION = "botocore_session"
AWS_PROFILE_NAME = "client.profile-name"
AWS_REGION = "client.region"
AWS_ACCESS_KEY_ID = "client.access-key-id"
Expand Down
38 changes: 31 additions & 7 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
AWS_ROLE_SESSION_NAME,
AWS_SECRET_ACCESS_KEY,
AWS_SESSION_TOKEN,
BOTOCORE_SESSION,
GCS_DEFAULT_LOCATION,
GCS_SERVICE_HOST,
GCS_TOKEN,
Expand Down Expand Up @@ -405,7 +406,7 @@ def to_input_file(self) -> PyArrowFile:


class PyArrowFileIO(_TransientStateMixin, FileIO):
fs_by_scheme: Callable[[str, str | None], FileSystem]
fs_by_scheme: Callable[..., FileSystem]
_transient_attrs = ("fs_by_scheme",)

def __init__(self, properties: Properties = EMPTY_DICT):
Expand Down Expand Up @@ -444,13 +445,15 @@ def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[
else:
return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"

def _initialize_fs(self, scheme: str, netloc: str | None = None) -> FileSystem:
def _initialize_fs(
self, scheme: str, netloc: str | None = None, session_credentials: tuple[str, str, str | None] | None = None
) -> FileSystem:
"""Initialize FileSystem for different scheme."""
if scheme in {"oss"}:
return self._initialize_oss_fs()

elif scheme in {"s3", "s3a", "s3n"}:
return self._initialize_s3_fs(netloc)
return self._initialize_s3_fs(netloc, session_credentials)

elif scheme in {"hdfs", "viewfs"}:
return self._initialize_hdfs_fs(scheme, netloc)
Expand Down Expand Up @@ -513,7 +516,25 @@ def _initialize_oss_fs(self) -> FileSystem:

return S3FileSystem(**client_kwargs)

def _initialize_s3_fs(self, netloc: str | None) -> FileSystem:
def _botocore_session_credentials(self) -> tuple[str, str, str | None] | None:
"""Return the frozen credentials of the botocore session when no explicit S3 credentials or role are set."""
session: Any = self.properties.get(BOTOCORE_SESSION)
if session is None or get_first_property_value(self.properties, S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID) is not None:
return None
if get_first_property_value(self.properties, S3_ROLE_ARN, AWS_ASSUME_ROLE_ARN, AWS_ROLE_ARN) is not None:
return None
if (credentials := session.get_credentials()) is None:
return None
# Refreshes the credentials when they are about to expire, which changes the fs_by_scheme cache key
frozen = credentials.get_frozen_credentials()
return frozen.access_key, frozen.secret_key, frozen.token

def _filesystem(self, scheme: str, netloc: str | None) -> FileSystem:
if scheme in {"s3", "s3a", "s3n"} and (session_credentials := self._botocore_session_credentials()) is not None:
return self.fs_by_scheme(scheme, netloc, session_credentials)
return self.fs_by_scheme(scheme, netloc)

def _initialize_s3_fs(self, netloc: str | None, session_credentials: tuple[str, str, str | None] | None = None) -> FileSystem:
from pyarrow.fs import S3FileSystem

provided_region = get_first_property_value(self.properties, S3_REGION, AWS_REGION)
Expand All @@ -539,6 +560,9 @@ def _initialize_s3_fs(self, netloc: str | None) -> FileSystem:
"region": bucket_region,
}

if session_credentials is not None:
client_kwargs["access_key"], client_kwargs["secret_key"], client_kwargs["session_token"] = session_credentials

if proxy_uri := self.properties.get(S3_PROXY_URI):
client_kwargs["proxy_options"] = proxy_uri

Expand Down Expand Up @@ -681,7 +705,7 @@ def new_input(self, location: str) -> PyArrowFile:
"""
scheme, netloc, path = self.parse_location(location, self.properties)
return PyArrowFile(
fs=self.fs_by_scheme(scheme, netloc),
fs=self._filesystem(scheme, netloc),
location=location,
path=path,
buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE)),
Expand All @@ -699,7 +723,7 @@ def new_output(self, location: str) -> PyArrowFile:
"""
scheme, netloc, path = self.parse_location(location, self.properties)
return PyArrowFile(
fs=self.fs_by_scheme(scheme, netloc),
fs=self._filesystem(scheme, netloc),
location=location,
path=path,
buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE)),
Expand All @@ -721,7 +745,7 @@ def delete(self, location: str | InputFile | OutputFile) -> None:
"""
str_location = location.location if isinstance(location, (InputFile, OutputFile)) else location
scheme, netloc, path = self.parse_location(str_location, self.properties)
fs = self.fs_by_scheme(scheme, netloc)
fs = self._filesystem(scheme, netloc)

try:
fs.delete_file(path)
Expand Down
48 changes: 47 additions & 1 deletion tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,18 @@
Or,
)
from pyiceberg.expressions.literals import literal
from pyiceberg.io import ADLS_ACCOUNT_NAME, S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io
from pyiceberg.io import (
ADLS_ACCOUNT_NAME,
BOTOCORE_SESSION,
S3_ACCESS_KEY_ID,
S3_REGION,
S3_RETRY_STRATEGY_IMPL,
S3_SECRET_ACCESS_KEY,
S3_SSL_CA_CERT,
InputStream,
OutputStream,
load_file_io,
)
from pyiceberg.io.pyarrow import (
ICEBERG_SCHEMA,
PYARROW_PARQUET_FIELD_ID_KEY,
Expand Down Expand Up @@ -4196,6 +4207,41 @@ def test_pyarrow_file_io_fs_by_scheme_cache() -> None:
assert pyarrow_file_io.fs_by_scheme.cache_info().hits == 2 # type: ignore


def test_pyarrow_file_io_botocore_session_credentials() -> None:
import botocore.session

session = botocore.session.get_session()
session.set_credentials("session-access-key", "session-secret-key", "session-token")
explicit_credentials = {S3_ACCESS_KEY_ID: "explicit-access-key", S3_SECRET_ACCESS_KEY: "explicit-secret-key"}
cases: list[tuple[dict[str, Any], tuple[str | None, str | None, str | None]]] = [
({}, ("session-access-key", "session-secret-key", "session-token")),
(explicit_credentials, ("explicit-access-key", "explicit-secret-key", None)),
]
for properties, expected in cases:
file_io = PyArrowFileIO({**properties, BOTOCORE_SESSION: session, S3_REGION: "us-east-1"})
with patch("pyarrow.fs.S3FileSystem") as mock_s3_fs:
file_io.new_input("s3://bucket/key")
kwargs = mock_s3_fs.call_args.kwargs
assert (kwargs["access_key"], kwargs["secret_key"], kwargs["session_token"]) == expected


def test_pyarrow_file_io_botocore_session_refreshed_credentials() -> None:
import botocore.session

session = botocore.session.get_session()
session.set_credentials("first-access-key", "first-secret-key", "first-token")
file_io = PyArrowFileIO({BOTOCORE_SESSION: session, S3_REGION: "us-east-1"})

with patch("pyarrow.fs.S3FileSystem") as mock_s3_fs:
file_io.new_input("s3://bucket/a")
file_io.new_input("s3://bucket/b")
session.set_credentials("second-access-key", "second-secret-key", "second-token")
file_io.new_input("s3://bucket/c")

assert [call.kwargs["access_key"] for call in mock_s3_fs.call_args_list] == ["first-access-key", "second-access-key"]
assert file_io.fs_by_scheme.cache_info().hits == 1 # type: ignore


def test_pyarrow_io_new_input_multi_region(caplog: Any) -> None:
# It's better to set up multi-region minio servers for an integration test once `endpoint_url` argument
# becomes available for `resolve_s3_region`
Expand Down
Loading