Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -1788,6 +1788,26 @@ with table.update_statistics() as update:
update.remove_statistics(snapshot_id2)
```

## Table Partition Statistics Management

Manage partition statistics files with operations through the `Table` API:

```python
# Setting a file for a snapshot replaces the file already set for that snapshot
table.update_partition_statistics().set_partition_statistics(partition_statistics_file).commit()
table.update_partition_statistics().remove_partition_statistics(snapshot_id).commit()
```

You can also use context managers to make more changes:

```python
with table.update_partition_statistics() as update:
update.set_partition_statistics(partition_statistics_file)
update.remove_partition_statistics(snapshot_id)
```

Within one update, sets are applied before removals, as in Java, so a removal wins over a set for the same snapshot regardless of the order of the calls. Removing partition statistics for a snapshot that has no file is a no-op.

## Query the data

To query a table, a table scan is needed. A table scan accepts a filter, columns, optionally a limit and a snapshot ID:
Expand Down
27 changes: 27 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@
UpgradeFormatVersionUpdate,
update_table_metadata,
)
from pyiceberg.table.update.partition_statistics import UpdatePartitionStatistics
from pyiceberg.table.update.schema import UpdateSchema
from pyiceberg.table.update.snapshot import ManageSnapshots, UpdateSnapshot, _FastAppendFiles
from pyiceberg.table.update.sorting import UpdateSortOrder
Expand Down Expand Up @@ -507,6 +508,15 @@ def update_statistics(self) -> UpdateStatistics:
"""
return UpdateStatistics(transaction=self)

def update_partition_statistics(self) -> UpdatePartitionStatistics:
"""
Create a new UpdatePartitionStatistics to update the partition statistics of the table.

Returns:
A new UpdatePartitionStatistics
"""
return UpdatePartitionStatistics(transaction=self)

def append(
self,
df: pa.Table | pa.RecordBatchReader,
Expand Down Expand Up @@ -1701,6 +1711,23 @@ def update_statistics(self) -> UpdateStatistics:
"""
return UpdateStatistics(transaction=Transaction(self, autocommit=True))

def update_partition_statistics(self) -> UpdatePartitionStatistics:
"""
Shorthand to run partition statistics management operations like set and remove partition statistics.

Use table.update_partition_statistics().<operation>().commit() to run a specific operation.
Use table.update_partition_statistics().<operation-one>().<operation-two>().commit() to run multiple operations.

Pending changes are applied on commit.

We can also use context managers to make more changes. For example:

with table.update_partition_statistics() as update:
update.set_partition_statistics(partition_statistics_file=partition_statistics_file)
update.remove_partition_statistics(snapshot_id=2)
"""
return UpdatePartitionStatistics(transaction=Transaction(self, autocommit=True))

def update_schema(self, allow_incompatible_changes: bool = False, case_sensitive: bool = True) -> UpdateSchema:
"""Create a new UpdateSchema to alter the columns of this table.

Expand Down
3 changes: 2 additions & 1 deletion pyiceberg/table/update/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -692,8 +692,9 @@ def _(update: SetPartitionStatisticsUpdate, base_metadata: TableMetadata, contex
def _(
update: RemovePartitionStatisticsUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext
) -> TableMetadata:
# Removing an absent entry is a no-op, so the metadata is not changed
if not any(part_stat.snapshot_id == update.snapshot_id for part_stat in base_metadata.partition_statistics):
raise ValueError(f"Partition Statistics with snapshot id {update.snapshot_id} does not exist")
return base_metadata

statistics = filter_statistics_by_snapshot_id(base_metadata.partition_statistics, update.snapshot_id)
context.add_update(update)
Expand Down
75 changes: 75 additions & 0 deletions pyiceberg/table/update/partition_statistics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from typing import TYPE_CHECKING

from pyiceberg.table.statistics import PartitionStatisticsFile
from pyiceberg.table.update import (
RemovePartitionStatisticsUpdate,
SetPartitionStatisticsUpdate,
TableUpdate,
UpdatesAndRequirements,
UpdateTableMetadata,
)

if TYPE_CHECKING:
from pyiceberg.table import Transaction


class UpdatePartitionStatistics(UpdateTableMetadata["UpdatePartitionStatistics"]):
"""
Run partition statistics management operations using APIs.

APIs include set_partition_statistics and remove_partition_statistics operations.

Use table.update_partition_statistics().<operation>().commit() to run a specific operation.
Use table.update_partition_statistics().<operation-one>().<operation-two>().commit() to run multiple operations.

Pending changes are applied on commit. Sets are applied before removals, so a removal wins over a set
for the same snapshot, regardless of the order of the calls.

We can also use context managers to make more changes. For example:

with table.update_partition_statistics() as update:
update.set_partition_statistics(partition_statistics_file=partition_statistics_file)
update.remove_partition_statistics(snapshot_id=2)
"""

_sets: dict[int, SetPartitionStatisticsUpdate]
_removals: dict[int, RemovePartitionStatisticsUpdate]

def __init__(self, transaction: "Transaction") -> None:
super().__init__(transaction)
self._sets = {}
self._removals = {}

def set_partition_statistics(self, partition_statistics_file: PartitionStatisticsFile | None) -> "UpdatePartitionStatistics":
"""Set the partition statistics file of its snapshot, replacing any existing one. None is ignored."""
if partition_statistics_file is not None:
set_update = SetPartitionStatisticsUpdate(partition_statistics=partition_statistics_file)
self._sets[partition_statistics_file.snapshot_id] = set_update

return self

def remove_partition_statistics(self, snapshot_id: int) -> "UpdatePartitionStatistics":
self._removals[snapshot_id] = RemovePartitionStatisticsUpdate(snapshot_id=snapshot_id)

return self

def _commit(self) -> UpdatesAndRequirements:
# Sets come before removals, as in Java's SetPartitionStatistics, so a removal wins over a set
updates: tuple[TableUpdate, ...] = (*self._sets.values(), *self._removals.values())
return updates, ()
21 changes: 11 additions & 10 deletions tests/table/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -2144,16 +2144,17 @@ def test_remove_partition_statistics_update(table_v2_with_statistics: Table) ->
assert len(remove_metadata.partition_statistics) == 0


def test_remove_partition_statistics_update_with_invalid_snapshot_id(table_v2_with_statistics: Table) -> None:
# Remove the same partition statistics file.
with pytest.raises(
ValueError,
match="Partition Statistics with snapshot id 123456789 does not exist",
):
update_table_metadata(
table_v2_with_statistics.metadata,
(RemovePartitionStatisticsUpdate(snapshot_id=123456789),),
)
def test_remove_partition_statistics_update_without_file_is_noop(table_v2_with_statistics: Table) -> None:
# Removing a partition statistics file that does not exist leaves the metadata unchanged, also when repeated
new_metadata = update_table_metadata(
table_v2_with_statistics.metadata,
(
RemovePartitionStatisticsUpdate(snapshot_id=123456789),
RemovePartitionStatisticsUpdate(snapshot_id=123456789),
),
)

assert new_metadata == table_v2_with_statistics.metadata


def test_add_encryption_key_update(table_v3: Table) -> None:
Expand Down
139 changes: 139 additions & 0 deletions tests/table/test_statistics.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import pyarrow as pa
import pytest

from pyiceberg.catalog import Catalog
from pyiceberg.table import Table
from pyiceberg.table.statistics import BlobMetadata, PartitionStatisticsFile, StatisticsFile


Expand Down Expand Up @@ -52,3 +57,137 @@ def test_statistics_file() -> None:
)

assert statistics_file.model_dump_json() == statistics_file_json


def _create_table_with_two_snapshots(catalog: Catalog) -> tuple[Table, int, int]:
arrow_table = pa.table({"id": [1, 2, 3]})
catalog.create_namespace("default")
table = catalog.create_table("default.test_partition_statistics", schema=arrow_table.schema)
table.append(arrow_table)
table.append(arrow_table)
snapshot_id_1, snapshot_id_2 = [snapshot.snapshot_id for snapshot in table.snapshots()]
return table, snapshot_id_1, snapshot_id_2


def _partition_statistics_file(snapshot_id: int, statistics_path: str) -> PartitionStatisticsFile:
return PartitionStatisticsFile(snapshot_id=snapshot_id, statistics_path=statistics_path, file_size_in_bytes=124)


def test_update_partition_statistics(catalog: Catalog) -> None:
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet")
stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet")
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")

table.update_partition_statistics().set_partition_statistics(stats_1).commit()
assert table.metadata.partition_statistics == [stats_1]

# A file set for a snapshot that already has one replaces it
table.update_partition_statistics().set_partition_statistics(stats_1_replaced).commit()
assert table.metadata.partition_statistics == [stats_1_replaced]

with table.update_partition_statistics() as update:
update.set_partition_statistics(stats_2)
assert table.metadata.partition_statistics == [stats_1_replaced, stats_2]

table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
assert table.metadata.partition_statistics == [stats_2]
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2]


def test_update_partition_statistics_set_and_remove_in_one_update(catalog: Catalog) -> None:
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet")
stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet")
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")

# Operations on different snapshots are both applied, in either call order
table.update_partition_statistics().set_partition_statistics(stats_1).commit()
table.update_partition_statistics().set_partition_statistics(stats_2).remove_partition_statistics(snapshot_id_1).commit()
assert table.metadata.partition_statistics == [stats_2]

with table.update_partition_statistics() as update:
update.set_partition_statistics(stats_1)
update.remove_partition_statistics(snapshot_id_2)
assert table.metadata.partition_statistics == [stats_1]

# On the same snapshot, the removal wins in either call order, with or without an existing file
for has_existing_file in (False, True):
for remove_first in (False, True):
if has_existing_file:
table.update_partition_statistics().set_partition_statistics(stats_1).commit()
else:
table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
with table.update_partition_statistics() as update:
if remove_first:
update.remove_partition_statistics(snapshot_id_1).set_partition_statistics(stats_1_replaced)
else:
update.set_partition_statistics(stats_1_replaced).remove_partition_statistics(snapshot_id_1)
assert table.metadata.partition_statistics == []

# Repeated operations: the last file set wins, and repeated removals are no-ops
with table.update_partition_statistics() as update:
update.set_partition_statistics(stats_1).set_partition_statistics(stats_1_replaced)
assert table.metadata.partition_statistics == [stats_1_replaced]

with table.update_partition_statistics() as update:
update.remove_partition_statistics(snapshot_id_1)
update.remove_partition_statistics(snapshot_id_1)
assert table.metadata.partition_statistics == []


def test_update_partition_statistics_in_transaction(catalog: Catalog) -> None:
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet")
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")

with table.transaction() as transaction:
transaction.update_partition_statistics().set_partition_statistics(stats_1).commit()
with transaction.update_partition_statistics() as update:
update.set_partition_statistics(stats_2)

# The pending changes are visible in the transaction only, until it commits
assert transaction.table_metadata.partition_statistics == [stats_1, stats_2]
assert table.metadata.partition_statistics == []
assert catalog.load_table(table.name()).metadata.partition_statistics == []

assert table.metadata.partition_statistics == [stats_1, stats_2]
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2]

# An exception raised inside the transaction rolls back its pending changes
def abort_transaction() -> None:
with table.transaction() as transaction:
transaction.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
raise RuntimeError("abort")

with pytest.raises(RuntimeError, match="abort"):
abort_transaction()

assert table.metadata.partition_statistics == [stats_1, stats_2]
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2]


def test_remove_partition_statistics_without_file_is_noop(catalog: Catalog) -> None:
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")
table.update_partition_statistics().set_partition_statistics(stats_2).commit()
metadata_before = table.metadata

# Removing a snapshot without a file commits without error, once or repeated, and leaves the metadata unchanged
table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
with table.update_partition_statistics() as update:
update.remove_partition_statistics(snapshot_id_1)
update.remove_partition_statistics(snapshot_id_1)

assert table.metadata == metadata_before
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2]


def test_set_partition_statistics_none_is_noop(catalog: Catalog) -> None:
table, snapshot_id_1, _ = _create_table_with_two_snapshots(catalog)
stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet")
table.update_partition_statistics().set_partition_statistics(stats_1).commit()

table.update_partition_statistics().set_partition_statistics(None).commit()

assert table.metadata.partition_statistics == [stats_1]
Loading