Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions mkdocs/docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ Iceberg tables support table properties to configure table behavior.
| `write.summary.partition-limit` | Integer | 0 | Maximum number of changed partitions for which per-partition summaries are included in the snapshot summary. The default of 0 disables them |
| `write.metadata.previous-versions-max` | Integer | 100 | The max number of previous version metadata files to keep before deleting after commit. |
| `write.metadata.delete-after-commit.enabled` | Boolean | False | Whether to automatically delete old *tracked* metadata files after each table commit. It will retain a number of the most recent metadata files, which can be set using property `write.metadata.previous-versions-max`. |
| `write.metadata.compression-codec` | `{none,gzip}` | none | Compression for the table metadata files that PyIceberg writes. `gzip` names new metadata files `*.gz.metadata.json` and gzips them. REST catalogs are unaffected, since the server chooses the metadata file location. |
| `write.object-storage.enabled` | Boolean | False | Enables the [`ObjectStoreLocationProvider`](configuration.md#object-store-location-provider) that adds a hash component to file paths. |
| `write.object-storage.partitioned-paths` | Boolean | True | Controls whether [partition values are included in file paths](configuration.md#partition-exclusion) when object storage is enabled |
| `write.py-location-provider.impl` | String of form `module.ClassName` | null | Optional, [custom `LocationProvider`](configuration.md#loading-a-custom-location-provider) implementation |
Expand Down
9 changes: 6 additions & 3 deletions pyiceberg/catalog/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1117,9 +1117,12 @@ def _update_and_stage_table(
metadata_location=current_table.metadata_location if current_table else None,
)

new_metadata_version = self._parse_metadata_version(current_table.metadata_location) + 1 if current_table else 0
provider = load_location_provider(updated_metadata.location, updated_metadata.properties)
new_metadata_location = provider.new_table_metadata_file_location(new_metadata_version)
if current_table and updated_metadata == current_table.metadata:
new_metadata_location = current_table.metadata_location
else:
new_metadata_version = self._parse_metadata_version(current_table.metadata_location) + 1 if current_table else 0
provider = load_location_provider(updated_metadata.location, updated_metadata.properties)
new_metadata_location = provider.new_table_metadata_file_location(new_metadata_version)

return StagedTable(
identifier=table_identifier,
Expand Down
3 changes: 3 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,9 @@ class TableProperties:
MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled"
MANIFEST_MERGE_ENABLED_DEFAULT = False

METADATA_COMPRESSION = "write.metadata.compression-codec"
METADATA_COMPRESSION_DEFAULT = "none"

METADATA_PREVIOUS_VERSIONS_MAX = "write.metadata.previous-versions-max"
METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT = 100

Expand Down
11 changes: 9 additions & 2 deletions pyiceberg/table/locations.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,19 @@ def new_table_metadata_file_location(self, new_version: int = 0) -> str:
str: fully-qualified URI for the new table metadata file.

Raises:
ValueError: If the version is negative.
ValueError: If the version is negative, or the metadata compression codec is not supported.
"""
from pyiceberg.table import TableProperties

if new_version < 0:
raise ValueError(f"Table metadata version: `{new_version}` must be a non-negative integer")

file_name = f"{new_version:05d}-{uuid.uuid4()}.metadata.json"
codec = self.table_properties.get(TableProperties.METADATA_COMPRESSION, TableProperties.METADATA_COMPRESSION_DEFAULT)
extension = {"none": "", "gzip": ".gz"}.get(codec.lower())
if extension is None:
raise ValueError(f"Invalid {TableProperties.METADATA_COMPRESSION}: {codec}, expected none or gzip")

file_name = f"{new_version:05d}-{uuid.uuid4()}{extension}.metadata.json"
return self.new_metadata_location(file_name)

def new_metadata_location(self, metadata_file_name: str) -> str:
Expand Down
100 changes: 100 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -834,6 +834,106 @@ def test_table_metadata_writes_reflect_latest_path(
assert table.location_provider().new_metadata_location("metadata.json") == f"{new_metadata_path}/metadata.json"


def test_table_writes_gzip_metadata_when_compression_codec_is_set(
catalog: Catalog,
test_table_identifier: Identifier,
table_schema_simple: Schema,
) -> None:
namespace = Catalog.namespace_from(test_table_identifier)
catalog.create_namespace(namespace)
table = catalog.create_table(
identifier=test_table_identifier,
schema=table_schema_simple,
properties={TableProperties.METADATA_COMPRESSION: "gzip"},
)
assert table.metadata_location.endswith(".gz.metadata.json")
with table.io.new_input(table.metadata_location).open() as stream:
assert stream.read(2) == b"\x1f\x8b" # gzip magic number

df = pa.Table.from_pydict(
{"foo": ["a"], "bar": [1], "baz": [True]},
schema=schema_to_pyarrow(table_schema_simple),
)
table.append(df)

# a later commit keeps the codec, and the version is parsed from the gzip file name
assert table.metadata_location.endswith(".gz.metadata.json")
assert table.metadata_location.split("/")[-1].startswith("00001-")
loaded_table = catalog.load_table(test_table_identifier)
assert loaded_table.metadata_location == table.metadata_location
assert loaded_table.metadata == table.metadata


def test_table_metadata_compression_codec_default_and_set_on_commit(
catalog: Catalog,
test_table_identifier: Identifier,
table_schema_simple: Schema,
) -> None:
namespace = Catalog.namespace_from(test_table_identifier)
catalog.create_namespace(namespace)
table = catalog.create_table(identifier=test_table_identifier, schema=table_schema_simple)
assert table.metadata_location.endswith(".metadata.json")
assert not table.metadata_location.endswith(".gz.metadata.json")

# the codec set in a commit applies to the metadata file that the commit writes
table.transaction().set_properties({TableProperties.METADATA_COMPRESSION: "gzip"}).commit_transaction()
assert table.metadata_location.endswith(".gz.metadata.json")
assert catalog.load_table(test_table_identifier).metadata_location == table.metadata_location


def test_table_metadata_invalid_compression_codec_raises(
catalog: Catalog,
test_table_identifier: Identifier,
table_schema_simple: Schema,
) -> None:
namespace = Catalog.namespace_from(test_table_identifier)
catalog.create_namespace(namespace)
with pytest.raises(ValueError, match="snappy"):
catalog.create_table(
identifier=test_table_identifier,
schema=table_schema_simple,
properties={TableProperties.METADATA_COMPRESSION: "snappy"},
)
assert not catalog.table_exists(test_table_identifier)

table = catalog.create_table(identifier=test_table_identifier, schema=table_schema_simple)
with pytest.raises(ValueError, match="snappy"):
table.transaction().set_properties({TableProperties.METADATA_COMPRESSION: "snappy"}).commit_transaction()
assert catalog.load_table(test_table_identifier).metadata_location == table.metadata_location


def test_table_metadata_legacy_invalid_compression_codec_on_commit(
catalog: Catalog,
test_table_identifier: Identifier,
metadata_location: str,
tmp_path: Path,
) -> None:
# metadata registered with a codec this version rejects: commits without changes still work
# the table location is local, so the commit that removes the codec can write its metadata file
metadata = json.loads(Path(metadata_location).read_text())
metadata["location"] = f"file://{tmp_path}/table"
metadata.setdefault("properties", {})[TableProperties.METADATA_COMPRESSION] = "snappy"
legacy_location = str(tmp_path / "table" / "metadata" / "00000-6c97e413-d51b-4538-ac70-12fe2a85cb83.metadata.json")
Path(legacy_location).parent.mkdir(parents=True)
Path(legacy_location).write_text(json.dumps(metadata))
namespace = Catalog.namespace_from(test_table_identifier)
catalog.create_namespace(namespace)
table = catalog.register_table(test_table_identifier, legacy_location)

table.transaction().set_properties({}).commit_transaction()
assert table.metadata_location == legacy_location

# a change writes new metadata, so the unsupported codec raises
with pytest.raises(ValueError, match="snappy"):
table.transaction().set_properties({"owner": "test"}).commit_transaction()
assert catalog.load_table(test_table_identifier).metadata_location == legacy_location

# removing the property writes new metadata with the default codec
table.transaction().remove_properties(TableProperties.METADATA_COMPRESSION).commit_transaction()
assert table.metadata_location != legacy_location
assert table.metadata_location.endswith(".metadata.json")


@pytest.mark.parametrize("format_version", [1, 2])
def test_write_and_evolve(catalog: Catalog, format_version: int) -> None:
identifier = f"default.arrow_write_data_and_evolve_schema_v{format_version}"
Expand Down
26 changes: 26 additions & 0 deletions tests/table/test_locations.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,32 @@ def test_location_provider_metadata_location_with_custom_path() -> None:
assert provider.new_metadata_location("metadata.json") == "s3://table-location/custom/path/metadata.json"


def test_new_table_metadata_file_location_compression_codec() -> None:
for codec, extension in [("none", ""), ("NONE", ""), ("gzip", ".gz"), ("GZIP", ".gz")]:
provider = load_location_provider(
table_location="table_location",
table_properties={TableProperties.METADATA_COMPRESSION: codec},
)

location = provider.new_table_metadata_file_location(1)

assert location.startswith("table_location/metadata/00001-")
assert location.endswith(f"{extension}.metadata.json")
# what remains after the version and the codec extension is exactly the uuid
file_uuid = location.removeprefix("table_location/metadata/00001-").removesuffix(f"{extension}.metadata.json")
assert len(file_uuid) == 36


def test_new_table_metadata_file_location_invalid_compression_codec() -> None:
provider = load_location_provider(
table_location="table_location",
table_properties={TableProperties.METADATA_COMPRESSION: "snappy"},
)

with pytest.raises(ValueError, match="snappy"):
provider.new_table_metadata_file_location()


def test_metadata_location_with_trailing_slash() -> None:
provider = load_location_provider(
table_location="table_location",
Expand Down
Loading