From 4dc3a63aad355bdfafcfc7d0e7cb71894ae200ff Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Mon, 16 Nov 2026 18:19:09 +0000 Subject: [PATCH] Add update_partition_statistics() to Table and Transaction Add an UpdatePartitionStatistics builder with set_partition_statistics() and remove_partition_statistics(), so callers no longer build update objects by hand. Sets apply before removals and absent removals are no-ops, as in Java. The RemovePartitionStatisticsUpdate handler therefore no longer raises. Co-authored-by: Isaac --- mkdocs/docs/api.md | 20 +++ pyiceberg/table/__init__.py | 27 ++++ pyiceberg/table/update/__init__.py | 3 +- .../table/update/partition_statistics.py | 75 ++++++++++ tests/table/test_init.py | 21 +-- tests/table/test_statistics.py | 139 ++++++++++++++++++ 6 files changed, 274 insertions(+), 11 deletions(-) create mode 100644 pyiceberg/table/update/partition_statistics.py diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index a732d2c0a3..e2ba808167 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -1788,6 +1788,26 @@ with table.update_statistics() as update: update.remove_statistics(snapshot_id2) ``` +## Table Partition Statistics Management + +Manage partition statistics files with operations through the `Table` API: + +```python +# Setting a file for a snapshot replaces the file already set for that snapshot +table.update_partition_statistics().set_partition_statistics(partition_statistics_file).commit() +table.update_partition_statistics().remove_partition_statistics(snapshot_id).commit() +``` + +You can also use context managers to make more changes: + +```python +with table.update_partition_statistics() as update: + update.set_partition_statistics(partition_statistics_file) + update.remove_partition_statistics(snapshot_id) +``` + +Within one update, sets are applied before removals, as in Java, so a removal wins over a set for the same snapshot regardless of the order of the calls. Removing partition statistics for a snapshot that has no file is a no-op. + ## Query the data To query a table, a table scan is needed. A table scan accepts a filter, columns, optionally a limit and a snapshot ID: diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..02312b1833 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -97,6 +97,7 @@ UpgradeFormatVersionUpdate, update_table_metadata, ) +from pyiceberg.table.update.partition_statistics import UpdatePartitionStatistics from pyiceberg.table.update.schema import UpdateSchema from pyiceberg.table.update.snapshot import ManageSnapshots, UpdateSnapshot, _FastAppendFiles from pyiceberg.table.update.sorting import UpdateSortOrder @@ -507,6 +508,15 @@ def update_statistics(self) -> UpdateStatistics: """ return UpdateStatistics(transaction=self) + def update_partition_statistics(self) -> UpdatePartitionStatistics: + """ + Create a new UpdatePartitionStatistics to update the partition statistics of the table. + + Returns: + A new UpdatePartitionStatistics + """ + return UpdatePartitionStatistics(transaction=self) + def append( self, df: pa.Table | pa.RecordBatchReader, @@ -1701,6 +1711,23 @@ def update_statistics(self) -> UpdateStatistics: """ return UpdateStatistics(transaction=Transaction(self, autocommit=True)) + def update_partition_statistics(self) -> UpdatePartitionStatistics: + """ + Shorthand to run partition statistics management operations like set and remove partition statistics. + + Use table.update_partition_statistics().().commit() to run a specific operation. + Use table.update_partition_statistics().().().commit() to run multiple operations. + + Pending changes are applied on commit. + + We can also use context managers to make more changes. For example: + + with table.update_partition_statistics() as update: + update.set_partition_statistics(partition_statistics_file=partition_statistics_file) + update.remove_partition_statistics(snapshot_id=2) + """ + return UpdatePartitionStatistics(transaction=Transaction(self, autocommit=True)) + def update_schema(self, allow_incompatible_changes: bool = False, case_sensitive: bool = True) -> UpdateSchema: """Create a new UpdateSchema to alter the columns of this table. diff --git a/pyiceberg/table/update/__init__.py b/pyiceberg/table/update/__init__.py index fc1d2347d9..ae637eeaa2 100644 --- a/pyiceberg/table/update/__init__.py +++ b/pyiceberg/table/update/__init__.py @@ -692,8 +692,9 @@ def _(update: SetPartitionStatisticsUpdate, base_metadata: TableMetadata, contex def _( update: RemovePartitionStatisticsUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext ) -> TableMetadata: + # Removing an absent entry is a no-op, so the metadata is not changed if not any(part_stat.snapshot_id == update.snapshot_id for part_stat in base_metadata.partition_statistics): - raise ValueError(f"Partition Statistics with snapshot id {update.snapshot_id} does not exist") + return base_metadata statistics = filter_statistics_by_snapshot_id(base_metadata.partition_statistics, update.snapshot_id) context.add_update(update) diff --git a/pyiceberg/table/update/partition_statistics.py b/pyiceberg/table/update/partition_statistics.py new file mode 100644 index 0000000000..bccf2bd1d5 --- /dev/null +++ b/pyiceberg/table/update/partition_statistics.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import TYPE_CHECKING + +from pyiceberg.table.statistics import PartitionStatisticsFile +from pyiceberg.table.update import ( + RemovePartitionStatisticsUpdate, + SetPartitionStatisticsUpdate, + TableUpdate, + UpdatesAndRequirements, + UpdateTableMetadata, +) + +if TYPE_CHECKING: + from pyiceberg.table import Transaction + + +class UpdatePartitionStatistics(UpdateTableMetadata["UpdatePartitionStatistics"]): + """ + Run partition statistics management operations using APIs. + + APIs include set_partition_statistics and remove_partition_statistics operations. + + Use table.update_partition_statistics().().commit() to run a specific operation. + Use table.update_partition_statistics().().().commit() to run multiple operations. + + Pending changes are applied on commit. Sets are applied before removals, so a removal wins over a set + for the same snapshot, regardless of the order of the calls. + + We can also use context managers to make more changes. For example: + + with table.update_partition_statistics() as update: + update.set_partition_statistics(partition_statistics_file=partition_statistics_file) + update.remove_partition_statistics(snapshot_id=2) + """ + + _sets: dict[int, SetPartitionStatisticsUpdate] + _removals: dict[int, RemovePartitionStatisticsUpdate] + + def __init__(self, transaction: "Transaction") -> None: + super().__init__(transaction) + self._sets = {} + self._removals = {} + + def set_partition_statistics(self, partition_statistics_file: PartitionStatisticsFile | None) -> "UpdatePartitionStatistics": + """Set the partition statistics file of its snapshot, replacing any existing one. None is ignored.""" + if partition_statistics_file is not None: + set_update = SetPartitionStatisticsUpdate(partition_statistics=partition_statistics_file) + self._sets[partition_statistics_file.snapshot_id] = set_update + + return self + + def remove_partition_statistics(self, snapshot_id: int) -> "UpdatePartitionStatistics": + self._removals[snapshot_id] = RemovePartitionStatisticsUpdate(snapshot_id=snapshot_id) + + return self + + def _commit(self) -> UpdatesAndRequirements: + # Sets come before removals, as in Java's SetPartitionStatistics, so a removal wins over a set + updates: tuple[TableUpdate, ...] = (*self._sets.values(), *self._removals.values()) + return updates, () diff --git a/tests/table/test_init.py b/tests/table/test_init.py index af9a11a452..3b18c5963e 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -2144,16 +2144,17 @@ def test_remove_partition_statistics_update(table_v2_with_statistics: Table) -> assert len(remove_metadata.partition_statistics) == 0 -def test_remove_partition_statistics_update_with_invalid_snapshot_id(table_v2_with_statistics: Table) -> None: - # Remove the same partition statistics file. - with pytest.raises( - ValueError, - match="Partition Statistics with snapshot id 123456789 does not exist", - ): - update_table_metadata( - table_v2_with_statistics.metadata, - (RemovePartitionStatisticsUpdate(snapshot_id=123456789),), - ) +def test_remove_partition_statistics_update_without_file_is_noop(table_v2_with_statistics: Table) -> None: + # Removing a partition statistics file that does not exist leaves the metadata unchanged, also when repeated + new_metadata = update_table_metadata( + table_v2_with_statistics.metadata, + ( + RemovePartitionStatisticsUpdate(snapshot_id=123456789), + RemovePartitionStatisticsUpdate(snapshot_id=123456789), + ), + ) + + assert new_metadata == table_v2_with_statistics.metadata def test_add_encryption_key_update(table_v3: Table) -> None: diff --git a/tests/table/test_statistics.py b/tests/table/test_statistics.py index 6c91990ea3..36f0d7eee3 100644 --- a/tests/table/test_statistics.py +++ b/tests/table/test_statistics.py @@ -14,6 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import pyarrow as pa +import pytest + +from pyiceberg.catalog import Catalog +from pyiceberg.table import Table from pyiceberg.table.statistics import BlobMetadata, PartitionStatisticsFile, StatisticsFile @@ -52,3 +57,137 @@ def test_statistics_file() -> None: ) assert statistics_file.model_dump_json() == statistics_file_json + + +def _create_table_with_two_snapshots(catalog: Catalog) -> tuple[Table, int, int]: + arrow_table = pa.table({"id": [1, 2, 3]}) + catalog.create_namespace("default") + table = catalog.create_table("default.test_partition_statistics", schema=arrow_table.schema) + table.append(arrow_table) + table.append(arrow_table) + snapshot_id_1, snapshot_id_2 = [snapshot.snapshot_id for snapshot in table.snapshots()] + return table, snapshot_id_1, snapshot_id_2 + + +def _partition_statistics_file(snapshot_id: int, statistics_path: str) -> PartitionStatisticsFile: + return PartitionStatisticsFile(snapshot_id=snapshot_id, statistics_path=statistics_path, file_size_in_bytes=124) + + +def test_update_partition_statistics(catalog: Catalog) -> None: + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) + stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet") + stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet") + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") + + table.update_partition_statistics().set_partition_statistics(stats_1).commit() + assert table.metadata.partition_statistics == [stats_1] + + # A file set for a snapshot that already has one replaces it + table.update_partition_statistics().set_partition_statistics(stats_1_replaced).commit() + assert table.metadata.partition_statistics == [stats_1_replaced] + + with table.update_partition_statistics() as update: + update.set_partition_statistics(stats_2) + assert table.metadata.partition_statistics == [stats_1_replaced, stats_2] + + table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() + assert table.metadata.partition_statistics == [stats_2] + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2] + + +def test_update_partition_statistics_set_and_remove_in_one_update(catalog: Catalog) -> None: + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) + stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet") + stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet") + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") + + # Operations on different snapshots are both applied, in either call order + table.update_partition_statistics().set_partition_statistics(stats_1).commit() + table.update_partition_statistics().set_partition_statistics(stats_2).remove_partition_statistics(snapshot_id_1).commit() + assert table.metadata.partition_statistics == [stats_2] + + with table.update_partition_statistics() as update: + update.set_partition_statistics(stats_1) + update.remove_partition_statistics(snapshot_id_2) + assert table.metadata.partition_statistics == [stats_1] + + # On the same snapshot, the removal wins in either call order, with or without an existing file + for has_existing_file in (False, True): + for remove_first in (False, True): + if has_existing_file: + table.update_partition_statistics().set_partition_statistics(stats_1).commit() + else: + table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() + with table.update_partition_statistics() as update: + if remove_first: + update.remove_partition_statistics(snapshot_id_1).set_partition_statistics(stats_1_replaced) + else: + update.set_partition_statistics(stats_1_replaced).remove_partition_statistics(snapshot_id_1) + assert table.metadata.partition_statistics == [] + + # Repeated operations: the last file set wins, and repeated removals are no-ops + with table.update_partition_statistics() as update: + update.set_partition_statistics(stats_1).set_partition_statistics(stats_1_replaced) + assert table.metadata.partition_statistics == [stats_1_replaced] + + with table.update_partition_statistics() as update: + update.remove_partition_statistics(snapshot_id_1) + update.remove_partition_statistics(snapshot_id_1) + assert table.metadata.partition_statistics == [] + + +def test_update_partition_statistics_in_transaction(catalog: Catalog) -> None: + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) + stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet") + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") + + with table.transaction() as transaction: + transaction.update_partition_statistics().set_partition_statistics(stats_1).commit() + with transaction.update_partition_statistics() as update: + update.set_partition_statistics(stats_2) + + # The pending changes are visible in the transaction only, until it commits + assert transaction.table_metadata.partition_statistics == [stats_1, stats_2] + assert table.metadata.partition_statistics == [] + assert catalog.load_table(table.name()).metadata.partition_statistics == [] + + assert table.metadata.partition_statistics == [stats_1, stats_2] + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2] + + # An exception raised inside the transaction rolls back its pending changes + def abort_transaction() -> None: + with table.transaction() as transaction: + transaction.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() + raise RuntimeError("abort") + + with pytest.raises(RuntimeError, match="abort"): + abort_transaction() + + assert table.metadata.partition_statistics == [stats_1, stats_2] + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2] + + +def test_remove_partition_statistics_without_file_is_noop(catalog: Catalog) -> None: + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") + table.update_partition_statistics().set_partition_statistics(stats_2).commit() + metadata_before = table.metadata + + # Removing a snapshot without a file commits without error, once or repeated, and leaves the metadata unchanged + table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() + with table.update_partition_statistics() as update: + update.remove_partition_statistics(snapshot_id_1) + update.remove_partition_statistics(snapshot_id_1) + + assert table.metadata == metadata_before + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2] + + +def test_set_partition_statistics_none_is_noop(catalog: Catalog) -> None: + table, snapshot_id_1, _ = _create_table_with_two_snapshots(catalog) + stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet") + table.update_partition_statistics().set_partition_statistics(stats_1).commit() + + table.update_partition_statistics().set_partition_statistics(None).commit() + + assert table.metadata.partition_statistics == [stats_1]