diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index d51079fc9d..0932d01a6e 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -17,9 +17,10 @@ from __future__ import annotations import itertools -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from uuid import UUID from pyiceberg.conversions import from_bytes from pyiceberg.expressions import AlwaysTrue, BooleanExpression @@ -39,7 +40,36 @@ def _readable_bound(field_type: PrimitiveType, bound: bytes | None) -> Any | None: - return from_bytes(field_type, bound) if bound is not None else None + return _storage_value(from_bytes(field_type, bound)) if bound is not None else None + + +def _storage_type(data_type: pa.DataType) -> pa.DataType: + """Return the storage type of a data type, replacing extension types inside structs.""" + import pyarrow as pa + + if isinstance(data_type, pa.BaseExtensionType): + return data_type.storage_type + if pa.types.is_struct(data_type): + return pa.struct([pa.field(field.name, _storage_type(field.type), field.nullable) for field in data_type]) + return data_type + + +def _from_pylist(rows: Iterable[dict[str, Any]], schema: pa.Schema) -> pa.Table: + """Build a table from Python rows, typed by the given schema. + + pyarrow cannot convert Python values into an extension type nested in a struct, + such as the ``arrow.uuid`` bounds in ``readable_metrics``. The rows are built with + the storage types and cast to ``schema``, so the output types are unchanged. + """ + import pyarrow as pa + + storage_schema = pa.schema([pa.field(field.name, _storage_type(field.type), field.nullable) for field in schema]) + return pa.Table.from_pylist(rows, schema=storage_schema).cast(schema) + + +def _storage_value(value: Any) -> Any: + """Return a value in its storage form, where a UUID is its 16 bytes.""" + return value.bytes if isinstance(value, UUID) else value class InspectTable: @@ -255,7 +285,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: partition = entry.data_file.partition partition_record_dict = { - field.name: partition[pos] + field.name: _storage_value(partition[pos]) for pos, field in enumerate(self.tbl.metadata.specs()[manifest.partition_spec_id].fields) } @@ -288,10 +318,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: } ) - return pa.Table.from_pylist( - entries, - schema=entries_schema, - ) + return _from_pylist(entries, entries_schema) def refs(self) -> pa.Table: """Return all snapshot references (branches and tags) as a PyArrow Table. @@ -413,17 +440,15 @@ def partitions( for entry in itertools.chain.from_iterable(scan._plan_manifest_entries()): partition = entry.data_file.partition partition_record_dict = { - field.name: partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields) + field.name: _storage_value(partition[pos]) + for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields) } entry_snapshot = snapshots_by_id.get(entry.snapshot_id) if entry.snapshot_id is not None else None self._update_partitions_map_from_manifest_entry( partitions_map, entry.data_file, partition_record_dict, entry_snapshot ) - return pa.Table.from_pylist( - partitions_map.values(), - schema=table_schema, - ) + return _from_pylist(partitions_map.values(), table_schema) def _update_partitions_map_from_manifest_entry( self, @@ -733,8 +758,6 @@ def _get_files_from_manifest( pa.Table: File metadata rows conforming to the files schema. See ``_get_files_schema`` for the full column list. """ - import pyarrow as pa - files: list[dict[str, Any]] = [] schema = self.tbl.metadata.schema() io = self.tbl.io @@ -762,7 +785,7 @@ def _get_files_from_manifest( } partition = data_file.partition partition_record_dict = { - field.name: partition[pos] + field.name: _storage_value(partition[pos]) for pos, field in enumerate(self.tbl.metadata.specs()[manifest_list.partition_spec_id].fields) } files.append( @@ -787,10 +810,7 @@ def _get_files_from_manifest( "readable_metrics": readable_metrics, } ) - return pa.Table.from_pylist( - files, - schema=self._get_files_schema(), - ) + return _from_pylist(files, self._get_files_schema()) def _get_files_schema(self) -> pa.Schema: """Return the PyArrow schema for file-level metadata tables. @@ -875,7 +895,7 @@ def _files(self, snapshot_id: int | None = None, data_file_filter: set[DataFileC import pyarrow as pa if not snapshot_id and not self.tbl.metadata.current_snapshot(): - return self._get_files_schema().empty_table() + return _from_pylist([], self._get_files_schema()) snapshot = self._get_snapshot(snapshot_id) io = self.tbl.io @@ -969,7 +989,7 @@ def _all_files(self, data_file_filter: set[DataFileContent] | None = None) -> pa snapshots = self.tbl.snapshots() if not snapshots: - return pa.Table.from_pylist([], schema=self._get_files_schema()) + return _from_pylist([], self._get_files_schema()) executor = ExecutorFactory.get_or_create() encryption_manager = self.tbl.encryption_manager diff --git a/tests/table/test_inspect.py b/tests/table/test_inspect.py index 306579754a..08f5180865 100644 --- a/tests/table/test_inspect.py +++ b/tests/table/test_inspect.py @@ -15,10 +15,12 @@ # specific language governing permissions and limitations # under the License. +import uuid from pathlib import PosixPath from typing import Any import pyarrow as pa +import pyarrow.parquet as pq import pytest from pyiceberg.conversions import to_bytes @@ -29,7 +31,7 @@ from pyiceberg.table.snapshots import Snapshot from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import Record -from pyiceberg.types import DoubleType, FloatType, IntegerType, LongType, NestedField, PrimitiveType, StringType +from pyiceberg.types import DoubleType, FloatType, IntegerType, LongType, NestedField, PrimitiveType, StringType, UUIDType from tests.catalog.test_base import InMemoryCatalog @@ -108,6 +110,50 @@ def test_inspect_manifests_preserves_empty_string_bounds(catalog: InMemoryCatalo assert partition_summary["upper_bound"] == "" +def test_inspect_files_and_entries_render_uuid_bounds(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(1, "u", UUIDType(), required=False)) + tbl = catalog.create_table("default.uuid_bounds", schema) + value = uuid.UUID(int=1) + tbl.append(pa.table({"u": pa.array([value.bytes], pa.uuid())})) + + entries = tbl.inspect.entries() + assert entries.schema.field("readable_metrics").type.field("u").type.field("lower_bound").type == pa.uuid() + entries_metrics = entries.to_pydict()["readable_metrics"][0]["u"] + assert entries_metrics["lower_bound"] == value + assert entries_metrics["upper_bound"] == value + + for files_table in [tbl.inspect.files(), tbl.inspect.data_files(), tbl.inspect.all_files(), tbl.inspect.all_data_files()]: + files_metrics = files_table.to_pydict()["readable_metrics"][0]["u"] + assert files_metrics["lower_bound"] == value + assert files_metrics["upper_bound"] == value + + assert tbl.inspect.delete_files().num_rows == 0 + assert tbl.inspect.all_delete_files().num_rows == 0 + + +def test_inspect_files_of_empty_table_with_uuid_column(catalog: InMemoryCatalog) -> None: + tbl = catalog.create_table("default.empty_uuid", Schema(NestedField(1, "u", UUIDType(), required=False))) + + assert tbl.inspect.files().num_rows == 0 + assert tbl.inspect.all_files().num_rows == 0 + + +def test_inspect_partitions_and_files_render_uuid_partition(catalog: InMemoryCatalog, tmp_path: PosixPath) -> None: + schema = Schema(NestedField(1, "u", UUIDType(), required=False)) + spec = PartitionSpec(PartitionField(1, 1000, IdentityTransform(), "u")) + tbl = catalog.create_table("default.uuid_partition", schema, partition_spec=spec) + value = uuid.UUID(int=2) + data_file = f"{tmp_path}/uuid-partition.parquet" + pq.write_table(pa.table({"u": pa.array([value.bytes], pa.uuid())}), data_file) + tbl.add_files([data_file]) + + partitions = tbl.inspect.partitions() + assert partitions.schema.field("partition").type.field("u").type == pa.uuid() + assert partitions.to_pydict()["partition"] == [{"u": value}] + assert tbl.inspect.files().to_pydict()["partition"] == [{"u": value}] + assert tbl.inspect.entries().to_pydict()["data_file"][0]["partition"] == {"u": value} + + def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None: # Promotion does not rewrite existing files, so their bounds keep the narrower 4-byte encoding # while the current schema type is the 8-byte one.