diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 14cb9c7467..8a9f644092 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -93,6 +93,7 @@ Iceberg tables support table properties to configure table behavior. | `write.summary.partition-limit` | Integer | 0 | Maximum number of changed partitions for which per-partition summaries are included in the snapshot summary. The default of 0 disables them | | `write.metadata.previous-versions-max` | Integer | 100 | The max number of previous version metadata files to keep before deleting after commit. | | `write.metadata.delete-after-commit.enabled` | Boolean | False | Whether to automatically delete old *tracked* metadata files after each table commit. It will retain a number of the most recent metadata files, which can be set using property `write.metadata.previous-versions-max`. | +| `write.metadata.compression-codec` | `{none,gzip}` | none | Compression for the table metadata files that PyIceberg writes. `gzip` names new metadata files `*.gz.metadata.json` and gzips them. REST catalogs are unaffected, since the server chooses the metadata file location. | | `write.object-storage.enabled` | Boolean | False | Enables the [`ObjectStoreLocationProvider`](configuration.md#object-store-location-provider) that adds a hash component to file paths. | | `write.object-storage.partitioned-paths` | Boolean | True | Controls whether [partition values are included in file paths](configuration.md#partition-exclusion) when object storage is enabled | | `write.py-location-provider.impl` | String of form `module.ClassName` | null | Optional, [custom `LocationProvider`](configuration.md#loading-a-custom-location-provider) implementation | diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index ce0902f338..1c063cb526 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -1117,9 +1117,12 @@ def _update_and_stage_table( metadata_location=current_table.metadata_location if current_table else None, ) - new_metadata_version = self._parse_metadata_version(current_table.metadata_location) + 1 if current_table else 0 - provider = load_location_provider(updated_metadata.location, updated_metadata.properties) - new_metadata_location = provider.new_table_metadata_file_location(new_metadata_version) + if current_table and updated_metadata == current_table.metadata: + new_metadata_location = current_table.metadata_location + else: + new_metadata_version = self._parse_metadata_version(current_table.metadata_location) + 1 if current_table else 0 + provider = load_location_provider(updated_metadata.location, updated_metadata.properties) + new_metadata_location = provider.new_table_metadata_file_location(new_metadata_version) return StagedTable( identifier=table_identifier, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..f056e5ae6b 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -227,6 +227,9 @@ class TableProperties: MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled" MANIFEST_MERGE_ENABLED_DEFAULT = False + METADATA_COMPRESSION = "write.metadata.compression-codec" + METADATA_COMPRESSION_DEFAULT = "none" + METADATA_PREVIOUS_VERSIONS_MAX = "write.metadata.previous-versions-max" METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT = 100 diff --git a/pyiceberg/table/locations.py b/pyiceberg/table/locations.py index 771c6b5a0f..5fac1aa62c 100644 --- a/pyiceberg/table/locations.py +++ b/pyiceberg/table/locations.py @@ -80,12 +80,19 @@ def new_table_metadata_file_location(self, new_version: int = 0) -> str: str: fully-qualified URI for the new table metadata file. Raises: - ValueError: If the version is negative. + ValueError: If the version is negative, or the metadata compression codec is not supported. """ + from pyiceberg.table import TableProperties + if new_version < 0: raise ValueError(f"Table metadata version: `{new_version}` must be a non-negative integer") - file_name = f"{new_version:05d}-{uuid.uuid4()}.metadata.json" + codec = self.table_properties.get(TableProperties.METADATA_COMPRESSION, TableProperties.METADATA_COMPRESSION_DEFAULT) + extension = {"none": "", "gzip": ".gz"}.get(codec.lower()) + if extension is None: + raise ValueError(f"Invalid {TableProperties.METADATA_COMPRESSION}: {codec}, expected none or gzip") + + file_name = f"{new_version:05d}-{uuid.uuid4()}{extension}.metadata.json" return self.new_metadata_location(file_name) def new_metadata_location(self, metadata_file_name: str) -> str: diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index cd817512b4..7f99dcacab 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -834,6 +834,106 @@ def test_table_metadata_writes_reflect_latest_path( assert table.location_provider().new_metadata_location("metadata.json") == f"{new_metadata_path}/metadata.json" +def test_table_writes_gzip_metadata_when_compression_codec_is_set( + catalog: Catalog, + test_table_identifier: Identifier, + table_schema_simple: Schema, +) -> None: + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + table = catalog.create_table( + identifier=test_table_identifier, + schema=table_schema_simple, + properties={TableProperties.METADATA_COMPRESSION: "gzip"}, + ) + assert table.metadata_location.endswith(".gz.metadata.json") + with table.io.new_input(table.metadata_location).open() as stream: + assert stream.read(2) == b"\x1f\x8b" # gzip magic number + + df = pa.Table.from_pydict( + {"foo": ["a"], "bar": [1], "baz": [True]}, + schema=schema_to_pyarrow(table_schema_simple), + ) + table.append(df) + + # a later commit keeps the codec, and the version is parsed from the gzip file name + assert table.metadata_location.endswith(".gz.metadata.json") + assert table.metadata_location.split("/")[-1].startswith("00001-") + loaded_table = catalog.load_table(test_table_identifier) + assert loaded_table.metadata_location == table.metadata_location + assert loaded_table.metadata == table.metadata + + +def test_table_metadata_compression_codec_default_and_set_on_commit( + catalog: Catalog, + test_table_identifier: Identifier, + table_schema_simple: Schema, +) -> None: + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + table = catalog.create_table(identifier=test_table_identifier, schema=table_schema_simple) + assert table.metadata_location.endswith(".metadata.json") + assert not table.metadata_location.endswith(".gz.metadata.json") + + # the codec set in a commit applies to the metadata file that the commit writes + table.transaction().set_properties({TableProperties.METADATA_COMPRESSION: "gzip"}).commit_transaction() + assert table.metadata_location.endswith(".gz.metadata.json") + assert catalog.load_table(test_table_identifier).metadata_location == table.metadata_location + + +def test_table_metadata_invalid_compression_codec_raises( + catalog: Catalog, + test_table_identifier: Identifier, + table_schema_simple: Schema, +) -> None: + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + with pytest.raises(ValueError, match="snappy"): + catalog.create_table( + identifier=test_table_identifier, + schema=table_schema_simple, + properties={TableProperties.METADATA_COMPRESSION: "snappy"}, + ) + assert not catalog.table_exists(test_table_identifier) + + table = catalog.create_table(identifier=test_table_identifier, schema=table_schema_simple) + with pytest.raises(ValueError, match="snappy"): + table.transaction().set_properties({TableProperties.METADATA_COMPRESSION: "snappy"}).commit_transaction() + assert catalog.load_table(test_table_identifier).metadata_location == table.metadata_location + + +def test_table_metadata_legacy_invalid_compression_codec_on_commit( + catalog: Catalog, + test_table_identifier: Identifier, + metadata_location: str, + tmp_path: Path, +) -> None: + # metadata registered with a codec this version rejects: commits without changes still work + # the table location is local, so the commit that removes the codec can write its metadata file + metadata = json.loads(Path(metadata_location).read_text()) + metadata["location"] = f"file://{tmp_path}/table" + metadata.setdefault("properties", {})[TableProperties.METADATA_COMPRESSION] = "snappy" + legacy_location = str(tmp_path / "table" / "metadata" / "00000-6c97e413-d51b-4538-ac70-12fe2a85cb83.metadata.json") + Path(legacy_location).parent.mkdir(parents=True) + Path(legacy_location).write_text(json.dumps(metadata)) + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + table = catalog.register_table(test_table_identifier, legacy_location) + + table.transaction().set_properties({}).commit_transaction() + assert table.metadata_location == legacy_location + + # a change writes new metadata, so the unsupported codec raises + with pytest.raises(ValueError, match="snappy"): + table.transaction().set_properties({"owner": "test"}).commit_transaction() + assert catalog.load_table(test_table_identifier).metadata_location == legacy_location + + # removing the property writes new metadata with the default codec + table.transaction().remove_properties(TableProperties.METADATA_COMPRESSION).commit_transaction() + assert table.metadata_location != legacy_location + assert table.metadata_location.endswith(".metadata.json") + + @pytest.mark.parametrize("format_version", [1, 2]) def test_write_and_evolve(catalog: Catalog, format_version: int) -> None: identifier = f"default.arrow_write_data_and_evolve_schema_v{format_version}" diff --git a/tests/table/test_locations.py b/tests/table/test_locations.py index 6dd7520f3c..9dd465ad50 100644 --- a/tests/table/test_locations.py +++ b/tests/table/test_locations.py @@ -182,6 +182,32 @@ def test_location_provider_metadata_location_with_custom_path() -> None: assert provider.new_metadata_location("metadata.json") == "s3://table-location/custom/path/metadata.json" +def test_new_table_metadata_file_location_compression_codec() -> None: + for codec, extension in [("none", ""), ("NONE", ""), ("gzip", ".gz"), ("GZIP", ".gz")]: + provider = load_location_provider( + table_location="table_location", + table_properties={TableProperties.METADATA_COMPRESSION: codec}, + ) + + location = provider.new_table_metadata_file_location(1) + + assert location.startswith("table_location/metadata/00001-") + assert location.endswith(f"{extension}.metadata.json") + # what remains after the version and the codec extension is exactly the uuid + file_uuid = location.removeprefix("table_location/metadata/00001-").removesuffix(f"{extension}.metadata.json") + assert len(file_uuid) == 36 + + +def test_new_table_metadata_file_location_invalid_compression_codec() -> None: + provider = load_location_provider( + table_location="table_location", + table_properties={TableProperties.METADATA_COMPRESSION: "snappy"}, + ) + + with pytest.raises(ValueError, match="snappy"): + provider.new_table_metadata_file_location() + + def test_metadata_location_with_trailing_slash() -> None: provider = load_location_provider( table_location="table_location",