Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 41 additions & 21 deletions pyiceberg/table/inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@
from __future__ import annotations

import itertools
from collections.abc import Iterator
from collections.abc import Iterable, Iterator
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from uuid import UUID

from pyiceberg.conversions import from_bytes
from pyiceberg.expressions import AlwaysTrue, BooleanExpression
Expand All @@ -39,7 +40,36 @@


def _readable_bound(field_type: PrimitiveType, bound: bytes | None) -> Any | None:
return from_bytes(field_type, bound) if bound is not None else None
return _storage_value(from_bytes(field_type, bound)) if bound is not None else None


def _storage_type(data_type: pa.DataType) -> pa.DataType:
"""Return the storage type of a data type, replacing extension types inside structs."""
import pyarrow as pa

if isinstance(data_type, pa.BaseExtensionType):
return data_type.storage_type
if pa.types.is_struct(data_type):
return pa.struct([pa.field(field.name, _storage_type(field.type), field.nullable) for field in data_type])
return data_type


def _from_pylist(rows: Iterable[dict[str, Any]], schema: pa.Schema) -> pa.Table:
"""Build a table from Python rows, typed by the given schema.

pyarrow cannot convert Python values into an extension type nested in a struct,
such as the ``arrow.uuid`` bounds in ``readable_metrics``. The rows are built with
the storage types and cast to ``schema``, so the output types are unchanged.
"""
import pyarrow as pa

storage_schema = pa.schema([pa.field(field.name, _storage_type(field.type), field.nullable) for field in schema])
return pa.Table.from_pylist(rows, schema=storage_schema).cast(schema)


def _storage_value(value: Any) -> Any:
"""Return a value in its storage form, where a UUID is its 16 bytes."""
return value.bytes if isinstance(value, UUID) else value


class InspectTable:
Expand Down Expand Up @@ -255,7 +285,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:

partition = entry.data_file.partition
partition_record_dict = {
field.name: partition[pos]
field.name: _storage_value(partition[pos])
for pos, field in enumerate(self.tbl.metadata.specs()[manifest.partition_spec_id].fields)
}

Expand Down Expand Up @@ -288,10 +318,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:
}
)

return pa.Table.from_pylist(
entries,
schema=entries_schema,
)
return _from_pylist(entries, entries_schema)

def refs(self) -> pa.Table:
"""Return all snapshot references (branches and tags) as a PyArrow Table.
Expand Down Expand Up @@ -413,17 +440,15 @@ def partitions(
for entry in itertools.chain.from_iterable(scan._plan_manifest_entries()):
partition = entry.data_file.partition
partition_record_dict = {
field.name: partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields)
field.name: _storage_value(partition[pos])
for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields)
}
entry_snapshot = snapshots_by_id.get(entry.snapshot_id) if entry.snapshot_id is not None else None
self._update_partitions_map_from_manifest_entry(
partitions_map, entry.data_file, partition_record_dict, entry_snapshot
)

return pa.Table.from_pylist(
partitions_map.values(),
schema=table_schema,
)
return _from_pylist(partitions_map.values(), table_schema)

def _update_partitions_map_from_manifest_entry(
self,
Expand Down Expand Up @@ -733,8 +758,6 @@ def _get_files_from_manifest(
pa.Table: File metadata rows conforming to the files schema.
See ``_get_files_schema`` for the full column list.
"""
import pyarrow as pa

files: list[dict[str, Any]] = []
schema = self.tbl.metadata.schema()
io = self.tbl.io
Expand Down Expand Up @@ -762,7 +785,7 @@ def _get_files_from_manifest(
}
partition = data_file.partition
partition_record_dict = {
field.name: partition[pos]
field.name: _storage_value(partition[pos])
for pos, field in enumerate(self.tbl.metadata.specs()[manifest_list.partition_spec_id].fields)
}
files.append(
Expand All @@ -787,10 +810,7 @@ def _get_files_from_manifest(
"readable_metrics": readable_metrics,
}
)
return pa.Table.from_pylist(
files,
schema=self._get_files_schema(),
)
return _from_pylist(files, self._get_files_schema())

def _get_files_schema(self) -> pa.Schema:
"""Return the PyArrow schema for file-level metadata tables.
Expand Down Expand Up @@ -875,7 +895,7 @@ def _files(self, snapshot_id: int | None = None, data_file_filter: set[DataFileC
import pyarrow as pa

if not snapshot_id and not self.tbl.metadata.current_snapshot():
return self._get_files_schema().empty_table()
return _from_pylist([], self._get_files_schema())

snapshot = self._get_snapshot(snapshot_id)
io = self.tbl.io
Expand Down Expand Up @@ -969,7 +989,7 @@ def _all_files(self, data_file_filter: set[DataFileContent] | None = None) -> pa

snapshots = self.tbl.snapshots()
if not snapshots:
return pa.Table.from_pylist([], schema=self._get_files_schema())
return _from_pylist([], self._get_files_schema())

executor = ExecutorFactory.get_or_create()
encryption_manager = self.tbl.encryption_manager
Expand Down
48 changes: 47 additions & 1 deletion tests/table/test_inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@
# specific language governing permissions and limitations
# under the License.

import uuid
from pathlib import PosixPath
from typing import Any

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from pyiceberg.conversions import to_bytes
Expand All @@ -29,7 +31,7 @@
from pyiceberg.table.snapshots import Snapshot
from pyiceberg.transforms import IdentityTransform
from pyiceberg.typedef import Record
from pyiceberg.types import DoubleType, FloatType, IntegerType, LongType, NestedField, PrimitiveType, StringType
from pyiceberg.types import DoubleType, FloatType, IntegerType, LongType, NestedField, PrimitiveType, StringType, UUIDType
from tests.catalog.test_base import InMemoryCatalog


Expand Down Expand Up @@ -108,6 +110,50 @@ def test_inspect_manifests_preserves_empty_string_bounds(catalog: InMemoryCatalo
assert partition_summary["upper_bound"] == ""


def test_inspect_files_and_entries_render_uuid_bounds(catalog: InMemoryCatalog) -> None:
schema = Schema(NestedField(1, "u", UUIDType(), required=False))
tbl = catalog.create_table("default.uuid_bounds", schema)
value = uuid.UUID(int=1)
tbl.append(pa.table({"u": pa.array([value.bytes], pa.uuid())}))

entries = tbl.inspect.entries()
assert entries.schema.field("readable_metrics").type.field("u").type.field("lower_bound").type == pa.uuid()
entries_metrics = entries.to_pydict()["readable_metrics"][0]["u"]
assert entries_metrics["lower_bound"] == value
assert entries_metrics["upper_bound"] == value

for files_table in [tbl.inspect.files(), tbl.inspect.data_files(), tbl.inspect.all_files(), tbl.inspect.all_data_files()]:
files_metrics = files_table.to_pydict()["readable_metrics"][0]["u"]
assert files_metrics["lower_bound"] == value
assert files_metrics["upper_bound"] == value

assert tbl.inspect.delete_files().num_rows == 0
assert tbl.inspect.all_delete_files().num_rows == 0


def test_inspect_files_of_empty_table_with_uuid_column(catalog: InMemoryCatalog) -> None:
tbl = catalog.create_table("default.empty_uuid", Schema(NestedField(1, "u", UUIDType(), required=False)))

assert tbl.inspect.files().num_rows == 0
assert tbl.inspect.all_files().num_rows == 0


def test_inspect_partitions_and_files_render_uuid_partition(catalog: InMemoryCatalog, tmp_path: PosixPath) -> None:
schema = Schema(NestedField(1, "u", UUIDType(), required=False))
spec = PartitionSpec(PartitionField(1, 1000, IdentityTransform(), "u"))
tbl = catalog.create_table("default.uuid_partition", schema, partition_spec=spec)
value = uuid.UUID(int=2)
data_file = f"{tmp_path}/uuid-partition.parquet"
pq.write_table(pa.table({"u": pa.array([value.bytes], pa.uuid())}), data_file)
tbl.add_files([data_file])

partitions = tbl.inspect.partitions()
assert partitions.schema.field("partition").type.field("u").type == pa.uuid()
assert partitions.to_pydict()["partition"] == [{"u": value}]
assert tbl.inspect.files().to_pydict()["partition"] == [{"u": value}]
assert tbl.inspect.entries().to_pydict()["data_file"][0]["partition"] == {"u": value}


def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None:
# Promotion does not rewrite existing files, so their bounds keep the narrower 4-byte encoding
# while the current schema type is the 8-byte one.
Expand Down
Loading