From 158fa32ba8d210177f23679a62b365927c0f3b27 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Thu, 12 Nov 2026 22:00:20 +0000 Subject: [PATCH 1/2] Fix inspect file and partition tables for UUID columns inspect.files(), entries() and partitions() raised ArrowNotImplementedError on tables with a uuid column: pa.Table.from_pylist cannot build the arrow.uuid type nested in readable_metrics or a partition struct. Build these tables from the storage types and cast them to the schema, so the output types are unchanged. UUID partition values are passed as their 16 bytes, which pyarrow 18 requires. Co-authored-by: Isaac --- pyiceberg/table/inspect.py | 60 ++++++++++++++++++++++++------------- tests/table/test_inspect.py | 48 ++++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 21 deletions(-) diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index d51079fc9d..fdcf36c66b 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -17,9 +17,10 @@ from __future__ import annotations import itertools -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from uuid import UUID from pyiceberg.conversions import from_bytes from pyiceberg.expressions import AlwaysTrue, BooleanExpression @@ -42,6 +43,35 @@ def _readable_bound(field_type: PrimitiveType, bound: bytes | None) -> Any | Non return from_bytes(field_type, bound) if bound is not None else None +def _storage_type(data_type: pa.DataType) -> pa.DataType: + """Return the storage type of a data type, replacing extension types inside structs.""" + import pyarrow as pa + + if isinstance(data_type, pa.BaseExtensionType): + return data_type.storage_type + if pa.types.is_struct(data_type): + return pa.struct([pa.field(field.name, _storage_type(field.type), field.nullable) for field in data_type]) + return data_type + + +def _from_pylist(rows: Iterable[dict[str, Any]], schema: pa.Schema) -> pa.Table: + """Build a table from Python rows, typed by the given schema. + + pyarrow cannot convert Python values into an extension type nested in a struct, + such as the ``arrow.uuid`` bounds in ``readable_metrics``. The rows are built with + the storage types and cast to ``schema``, so the output types are unchanged. + """ + import pyarrow as pa + + storage_schema = pa.schema([pa.field(field.name, _storage_type(field.type), field.nullable) for field in schema]) + return pa.Table.from_pylist(rows, schema=storage_schema).cast(schema) + + +def _storage_value(value: Any) -> Any: + """Return a partition value in its storage form, where a UUID is its 16 bytes.""" + return value.bytes if isinstance(value, UUID) else value + + class InspectTable: """Provides metadata inspection for Apache Iceberg tables. @@ -255,7 +285,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: partition = entry.data_file.partition partition_record_dict = { - field.name: partition[pos] + field.name: _storage_value(partition[pos]) for pos, field in enumerate(self.tbl.metadata.specs()[manifest.partition_spec_id].fields) } @@ -288,10 +318,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: } ) - return pa.Table.from_pylist( - entries, - schema=entries_schema, - ) + return _from_pylist(entries, entries_schema) def refs(self) -> pa.Table: """Return all snapshot references (branches and tags) as a PyArrow Table. @@ -413,17 +440,15 @@ def partitions( for entry in itertools.chain.from_iterable(scan._plan_manifest_entries()): partition = entry.data_file.partition partition_record_dict = { - field.name: partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields) + field.name: _storage_value(partition[pos]) + for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields) } entry_snapshot = snapshots_by_id.get(entry.snapshot_id) if entry.snapshot_id is not None else None self._update_partitions_map_from_manifest_entry( partitions_map, entry.data_file, partition_record_dict, entry_snapshot ) - return pa.Table.from_pylist( - partitions_map.values(), - schema=table_schema, - ) + return _from_pylist(partitions_map.values(), table_schema) def _update_partitions_map_from_manifest_entry( self, @@ -733,8 +758,6 @@ def _get_files_from_manifest( pa.Table: File metadata rows conforming to the files schema. See ``_get_files_schema`` for the full column list. """ - import pyarrow as pa - files: list[dict[str, Any]] = [] schema = self.tbl.metadata.schema() io = self.tbl.io @@ -762,7 +785,7 @@ def _get_files_from_manifest( } partition = data_file.partition partition_record_dict = { - field.name: partition[pos] + field.name: _storage_value(partition[pos]) for pos, field in enumerate(self.tbl.metadata.specs()[manifest_list.partition_spec_id].fields) } files.append( @@ -787,10 +810,7 @@ def _get_files_from_manifest( "readable_metrics": readable_metrics, } ) - return pa.Table.from_pylist( - files, - schema=self._get_files_schema(), - ) + return _from_pylist(files, self._get_files_schema()) def _get_files_schema(self) -> pa.Schema: """Return the PyArrow schema for file-level metadata tables. @@ -875,7 +895,7 @@ def _files(self, snapshot_id: int | None = None, data_file_filter: set[DataFileC import pyarrow as pa if not snapshot_id and not self.tbl.metadata.current_snapshot(): - return self._get_files_schema().empty_table() + return _from_pylist([], self._get_files_schema()) snapshot = self._get_snapshot(snapshot_id) io = self.tbl.io @@ -969,7 +989,7 @@ def _all_files(self, data_file_filter: set[DataFileContent] | None = None) -> pa snapshots = self.tbl.snapshots() if not snapshots: - return pa.Table.from_pylist([], schema=self._get_files_schema()) + return _from_pylist([], self._get_files_schema()) executor = ExecutorFactory.get_or_create() encryption_manager = self.tbl.encryption_manager diff --git a/tests/table/test_inspect.py b/tests/table/test_inspect.py index 306579754a..08f5180865 100644 --- a/tests/table/test_inspect.py +++ b/tests/table/test_inspect.py @@ -15,10 +15,12 @@ # specific language governing permissions and limitations # under the License. +import uuid from pathlib import PosixPath from typing import Any import pyarrow as pa +import pyarrow.parquet as pq import pytest from pyiceberg.conversions import to_bytes @@ -29,7 +31,7 @@ from pyiceberg.table.snapshots import Snapshot from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import Record -from pyiceberg.types import DoubleType, FloatType, IntegerType, LongType, NestedField, PrimitiveType, StringType +from pyiceberg.types import DoubleType, FloatType, IntegerType, LongType, NestedField, PrimitiveType, StringType, UUIDType from tests.catalog.test_base import InMemoryCatalog @@ -108,6 +110,50 @@ def test_inspect_manifests_preserves_empty_string_bounds(catalog: InMemoryCatalo assert partition_summary["upper_bound"] == "" +def test_inspect_files_and_entries_render_uuid_bounds(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(1, "u", UUIDType(), required=False)) + tbl = catalog.create_table("default.uuid_bounds", schema) + value = uuid.UUID(int=1) + tbl.append(pa.table({"u": pa.array([value.bytes], pa.uuid())})) + + entries = tbl.inspect.entries() + assert entries.schema.field("readable_metrics").type.field("u").type.field("lower_bound").type == pa.uuid() + entries_metrics = entries.to_pydict()["readable_metrics"][0]["u"] + assert entries_metrics["lower_bound"] == value + assert entries_metrics["upper_bound"] == value + + for files_table in [tbl.inspect.files(), tbl.inspect.data_files(), tbl.inspect.all_files(), tbl.inspect.all_data_files()]: + files_metrics = files_table.to_pydict()["readable_metrics"][0]["u"] + assert files_metrics["lower_bound"] == value + assert files_metrics["upper_bound"] == value + + assert tbl.inspect.delete_files().num_rows == 0 + assert tbl.inspect.all_delete_files().num_rows == 0 + + +def test_inspect_files_of_empty_table_with_uuid_column(catalog: InMemoryCatalog) -> None: + tbl = catalog.create_table("default.empty_uuid", Schema(NestedField(1, "u", UUIDType(), required=False))) + + assert tbl.inspect.files().num_rows == 0 + assert tbl.inspect.all_files().num_rows == 0 + + +def test_inspect_partitions_and_files_render_uuid_partition(catalog: InMemoryCatalog, tmp_path: PosixPath) -> None: + schema = Schema(NestedField(1, "u", UUIDType(), required=False)) + spec = PartitionSpec(PartitionField(1, 1000, IdentityTransform(), "u")) + tbl = catalog.create_table("default.uuid_partition", schema, partition_spec=spec) + value = uuid.UUID(int=2) + data_file = f"{tmp_path}/uuid-partition.parquet" + pq.write_table(pa.table({"u": pa.array([value.bytes], pa.uuid())}), data_file) + tbl.add_files([data_file]) + + partitions = tbl.inspect.partitions() + assert partitions.schema.field("partition").type.field("u").type == pa.uuid() + assert partitions.to_pydict()["partition"] == [{"u": value}] + assert tbl.inspect.files().to_pydict()["partition"] == [{"u": value}] + assert tbl.inspect.entries().to_pydict()["data_file"][0]["partition"] == {"u": value} + + def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None: # Promotion does not rewrite existing files, so their bounds keep the narrower 4-byte encoding # while the current schema type is the 8-byte one. From 41ed9c8a8147810bf0ddc298c7595b566816139c Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Mon, 16 Nov 2026 15:51:21 +0000 Subject: [PATCH 2/2] Keep readable_metrics bounds in their storage form Pass the decoded lower and upper bounds through _storage_value, like the partition values, so a uuid bound is built from its 16 bytes even when from_bytes returns a uuid.UUID (proposed in #4370). Co-authored-by: Isaac --- pyiceberg/table/inspect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index fdcf36c66b..0932d01a6e 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -40,7 +40,7 @@ def _readable_bound(field_type: PrimitiveType, bound: bytes | None) -> Any | None: - return from_bytes(field_type, bound) if bound is not None else None + return _storage_value(from_bytes(field_type, bound)) if bound is not None else None def _storage_type(data_type: pa.DataType) -> pa.DataType: @@ -68,7 +68,7 @@ def _from_pylist(rows: Iterable[dict[str, Any]], schema: pa.Schema) -> pa.Table: def _storage_value(value: Any) -> Any: - """Return a partition value in its storage form, where a UUID is its 16 bytes.""" + """Return a value in its storage form, where a UUID is its 16 bytes.""" return value.bytes if isinstance(value, UUID) else value