From 1a86d8b6184001b774911426d9721665daf074b6 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Fri, 13 Nov 2026 17:08:02 +0000 Subject: [PATCH 1/2] Plan uuid identity partitions with the literal's uuid representation Manifests are read with uuid partition values as uuid.UUID, while UUID literals hold the 16 bytes, so partition filters and residuals on a uuid identity partition never matched: EqualTo and In planned no files, and NotEqualTo and NotIn planned every file. Give partition values the literal representation before the partition and residual evaluators compare them. Co-authored-by: Isaac --- pyiceberg/table/__init__.py | 32 +++++++++++++++++++++++++++++--- tests/io/test_pyarrow.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index ce365ae623..9e13e9bf2d 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -112,7 +112,7 @@ Record, TableVersion, ) -from pyiceberg.types import strtobool +from pyiceberg.types import StructType, UUIDType, strtobool from pyiceberg.utils.concurrent import ExecutorFactory from pyiceberg.utils.config import Config from pyiceberg.utils.properties import property_as_bool, property_as_int @@ -2506,6 +2506,26 @@ def _to_arrow_batch_reader_via_file_scan_tasks( return pa.RecordBatchReader.from_batches(target_schema, batches).cast(target_schema) +def _partition_with_literal_values(partition_type: StructType) -> Callable[[Record], Record]: + """Return a function that gives partition values the representation that literals use. + + Manifests are read with uuid partition values as ``uuid.UUID``, while a UUID literal holds the 16 bytes, + so the values are converted before partition filters and residuals compare them with literals. + """ + uuid_positions = [pos for pos, field in enumerate(partition_type.fields) if isinstance(field.field_type, UUIDType)] + if not uuid_positions: + return lambda partition: partition + + def convert(partition: Record) -> Record: + values = [partition[pos] for pos in range(len(partition))] + for pos in uuid_positions: + if isinstance(values[pos], uuid.UUID): + values[pos] = values[pos].bytes + return Record(*values) + + return convert + + class DataScan(TableScan): @cached_property def _manifest_planner(self) -> ManifestGroupPlanner: @@ -2952,6 +2972,9 @@ def plan_files( if residual_cache_max_size is None or residual_cache_max_size <= 0: raise ValueError(f"{TableProperties.RESIDUAL_CACHE_MAX_SIZE} must be a positive integer") residual_cache: LRUCache[tuple[int, tuple[Any, ...]], BooleanExpression] = LRUCache(maxsize=residual_cache_max_size) + partition_literal_values: dict[int, Callable[[Record], Record]] = KeyDefaultDict( + lambda spec_id: _partition_with_literal_values(partition_specs[spec_id].partition_type(self.table_metadata.schema())) + ) def residual_for(data_file: DataFile) -> BooleanExpression: partition = data_file.partition @@ -2960,7 +2983,9 @@ def residual_for(data_file: DataFile) -> BooleanExpression: try: return residual_cache[cache_key] except KeyError: - residual = residual_evaluators[data_file.spec_id].residual_for(partition) + residual = residual_evaluators[data_file.spec_id].residual_for( + partition_literal_values[data_file.spec_id](partition) + ) residual_cache[cache_key] = residual return residual @@ -3005,10 +3030,11 @@ def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool] partition_schema = Schema(*partition_type.fields) partition_expr = self.partition_filters[spec_id] evaluator = expression_evaluator(partition_schema, partition_expr, self.case_sensitive) + literal_values = _partition_with_literal_values(partition_type) # Expression evaluators keep input-specific state local to each call, so the # prepared evaluator can be shared by every manifest using this spec. - return lambda data_file: evaluator(data_file.partition) + return lambda data_file: evaluator(literal_values(data_file.partition)) def _build_metrics_evaluator(self) -> Callable[[DataFile], bool]: schema = self.table_metadata.schema() diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b663f84727..094d4dd7e3 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3539,6 +3539,36 @@ def test_append_uuid_to_table_partitioned_on_uuid(catalog: InMemoryCatalog) -> N assert {task.file.partition[0] for task in table.scan().plan_files()} == {uuid.UUID(int=1), uuid.UUID(int=2), None} +def test_scan_plans_uuid_identity_partitions(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=False), + NestedField(id=2, name="uuid", field_type=UUIDType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="uuid")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_uuid_partitioned", schema=schema, partition_spec=spec) + first, second, third = uuid.UUID(int=1), uuid.UUID(int=2), uuid.UUID(int=3) + table.append( + pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int64()), + "uuid": pa.array([first.bytes, second.bytes, third.bytes], type=pa.uuid()), + } + ) + ) + + # Manifests are read with uuid partition values as uuid.UUID, while a UUID literal holds the 16 bytes + for row_filter, expected in [ + (f"uuid = '{first}'", {first}), + (f"uuid in ('{first}', '{second}')", {first, second}), + (f"uuid != '{first}'", {second, third}), + (f"uuid not in ('{first}', '{second}')", {third}), + ]: + scan = table.scan(row_filter=row_filter) + assert {task.file.partition[0] for task in scan.plan_files()} == expected, row_filter + assert scan.count() == len(expected), row_filter + + def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="id", field_type=LongType(), required=False), From a4f286a929b33dcaeb658c322177121b239708db Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Mon, 16 Nov 2026 15:44:03 +0000 Subject: [PATCH 2/2] Represent uuid values as uuid.UUID in literals and conversions The Avro reader returns uuid.UUID for uuid values, while UUIDLiteral and from_bytes(UUIDType) used the 16 bytes, so the partition and residual evaluators, and the projected identity partition values, compared a UUID with bytes. Standardize on uuid.UUID, as Java does (Literal, Conversions.fromByteBuffer): UUIDLiteral wraps a UUID, the string, fixed and binary conversions build one, from_bytes(UUIDType) returns one, and identity partition values stay UUID. PyArrow gets the 16 bytes. This replaces the partition value conversion in the scan planner. Co-authored-by: Isaac --- pyiceberg/conversions.py | 6 +- pyiceberg/expressions/literals.py | 28 ++++----- pyiceberg/io/pyarrow.py | 15 ++++- pyiceberg/partitioning.py | 10 ++-- pyiceberg/table/__init__.py | 32 +--------- tests/expressions/test_literals.py | 12 ++-- tests/io/test_pyarrow.py | 93 +++++++++++++++++++++++++++++- tests/table/test_partitioning.py | 12 +++- tests/test_conversions.py | 8 +-- 9 files changed, 150 insertions(+), 66 deletions(-) diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index fde1885443..6b3d84135b 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -388,13 +388,17 @@ def _(_: StringType, b: bytes) -> str: @from_bytes.register(BinaryType) @from_bytes.register(FixedType) -@from_bytes.register(UUIDType) @from_bytes.register(GeometryType) @from_bytes.register(GeographyType) def _(_: PrimitiveType, b: bytes) -> bytes: return b +@from_bytes.register(UUIDType) +def _(_: UUIDType, b: bytes) -> uuid.UUID: + return uuid.UUID(bytes=b) + + @from_bytes.register(DecimalType) def _(primitive_type: DecimalType, buf: bytes) -> Decimal: unscaled = int.from_bytes(buf, "big", signed=True) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index b502b43539..c9e6eade70 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -165,7 +165,7 @@ def literal(value: L) -> Literal[L]: elif isinstance(value, str): return StringLiteral(value) elif isinstance(value, UUID): - return UUIDLiteral(value.bytes) + return UUIDLiteral(value) elif isinstance(value, bytes): return BinaryLiteral(value) elif isinstance(value, Decimal): @@ -635,8 +635,8 @@ def _(self, _: TimestamptzType) -> Literal[int]: return TimestampLiteral(timestamptz_to_micros(self.value)) @to.register(UUIDType) - def _(self, _: UUIDType) -> Literal[bytes]: - return UUIDLiteral(UUID(self.value).bytes) + def _(self, _: UUIDType) -> Literal[UUID]: + return UUIDLiteral(UUID(self.value)) @to.register(DecimalType) def _(self, type_var: DecimalType) -> Literal[Decimal]: @@ -680,26 +680,26 @@ def __repr__(self) -> str: return f"literal({repr(self.value)})" -class UUIDLiteral(Literal[bytes]): - def __init__(self, value: bytes) -> None: - super().__init__(value, bytes) +class UUIDLiteral(Literal[UUID]): + def __init__(self, value: UUID) -> None: + super().__init__(value, UUID) @model_serializer def ser_model(self) -> UUID: - return UUID(bytes=self.root) + return self.root @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert UUIDLiteral into {type_var}") @to.register(UUIDType) - def _(self, _: UUIDType) -> Literal[bytes]: + def _(self, _: UUIDType) -> Literal[UUID]: return self @to.register(FixedType) def _(self, type_var: FixedType) -> Literal[bytes]: if len(type_var) == UUID_BYTES_LENGTH: - return FixedLiteral(self.value) + return FixedLiteral(self.value.bytes) else: raise TypeError( f"Cannot convert UUIDLiteral into {type_var}, different length: {len(type_var)} <> {UUID_BYTES_LENGTH}" @@ -707,7 +707,7 @@ def _(self, type_var: FixedType) -> Literal[bytes]: @to.register(BinaryType) def _(self, _: BinaryType) -> Literal[bytes]: - return BinaryLiteral(self.value) + return BinaryLiteral(self.value.bytes) class FixedLiteral(Literal[bytes]): @@ -736,9 +736,9 @@ def _(self, _: BinaryType) -> Literal[bytes]: return BinaryLiteral(self.value) @to.register(UUIDType) - def _(self, type_var: UUIDType) -> Literal[bytes]: + def _(self, type_var: UUIDType) -> Literal[UUID]: if len(self.value) == UUID_BYTES_LENGTH: - return UUIDLiteral(self.value) + return UUIDLiteral(UUID(bytes=self.value)) else: raise TypeError( f"Could not convert {self.value!r} into a {type_var}, lengths differ {len(self.value)} <> {UUID_BYTES_LENGTH}" @@ -771,9 +771,9 @@ def _(self, type_var: FixedType) -> Literal[bytes]: ) @to.register(UUIDType) - def _(self, type_var: UUIDType) -> Literal[bytes]: + def _(self, type_var: UUIDType) -> Literal[UUID]: if len(self.value) == UUID_BYTES_LENGTH: - return UUIDLiteral(self.value) + return UUIDLiteral(UUID(bytes=self.value)) else: raise TypeError( f"Cannot convert BinaryLiteral into {type_var}, different length: {UUID_BYTES_LENGTH} <> {len(self.value)}" diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..8e99389de4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -897,10 +897,15 @@ def _literal_arrow_type(iceberg_type: IcebergType, values: Iterable[Any]) -> pa. return visit(iceberg_type, _ConvertToArrowSchema()) +def _to_arrow_value(value: Any) -> Any: + # PyArrow builds uuid values from their 16 bytes; older PyArrow releases reject a uuid.UUID + return value.bytes if isinstance(value, uuid.UUID) else value + + def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: if not isinstance(iceberg_type, PrimitiveType): raise ValueError(f"Expected primitive type, got: {iceberg_type}") - return pa.scalar(value=value, type=_literal_arrow_type(iceberg_type, [value])) + return pa.scalar(value=_to_arrow_value(value), type=_literal_arrow_type(iceberg_type, [value])) class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): @@ -937,11 +942,15 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]: return term.ref().field.name def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) + pyarrow_literals = pa.array( + [_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals) + ) return pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) + pyarrow_literals = pa.array( + [_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals) + ) return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm) -> pc.Expression: diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 34c6f6aaaa..6d79dce18a 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -487,7 +487,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema: Return the Partition Record representation of the value. The value is first converted to internal partition representation. - For example, UUID is converted to bytes[16], DateType to days since epoch, etc. + For example, UUID values become uuid.UUID, and DateType values become days since epoch. Then the corresponding PartitionField's transform is applied to return the final partition record value. @@ -498,7 +498,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema: @singledispatch def _to_partition_representation(type: IcebergType, value: Any) -> Any: - """Strip the logical type into the physical type. + """Return the internal partition representation of a value. It can be that the value is already transformed into its physical type, in this case it will return the original value. Keep in mind that the @@ -560,13 +560,13 @@ def _(type: IcebergType, value: int | time | None) -> int | None: @_to_partition_representation.register(UUIDType) -def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> bytes | int | None: +def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> uuid.UUID | int | None: if value is None: return None elif isinstance(value, bytes): - return value # IdentityTransform + return uuid.UUID(bytes=value) # IdentityTransform elif isinstance(value, uuid.UUID): - return value.bytes # IdentityTransform + return value # IdentityTransform elif isinstance(value, int): return value # BucketTransform else: diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index a8c5dc7cef..9ef7519865 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -113,7 +113,7 @@ Record, TableVersion, ) -from pyiceberg.types import StructType, UUIDType, strtobool +from pyiceberg.types import strtobool from pyiceberg.utils.concurrent import ExecutorFactory from pyiceberg.utils.config import Config from pyiceberg.utils.properties import property_as_bool, property_as_int @@ -2505,26 +2505,6 @@ def _to_arrow_batch_reader_via_file_scan_tasks( return pa.RecordBatchReader.from_batches(target_schema, batches).cast(target_schema) -def _partition_with_literal_values(partition_type: StructType) -> Callable[[Record], Record]: - """Return a function that gives partition values the representation that literals use. - - Manifests are read with uuid partition values as ``uuid.UUID``, while a UUID literal holds the 16 bytes, - so the values are converted before partition filters and residuals compare them with literals. - """ - uuid_positions = [pos for pos, field in enumerate(partition_type.fields) if isinstance(field.field_type, UUIDType)] - if not uuid_positions: - return lambda partition: partition - - def convert(partition: Record) -> Record: - values = [partition[pos] for pos in range(len(partition))] - for pos in uuid_positions: - if isinstance(values[pos], uuid.UUID): - values[pos] = values[pos].bytes - return Record(*values) - - return convert - - class DataScan(TableScan): @cached_property def _manifest_planner(self) -> ManifestGroupPlanner: @@ -2978,9 +2958,6 @@ def plan_files( if residual_cache_max_size is None or residual_cache_max_size <= 0: raise ValueError(f"{TableProperties.RESIDUAL_CACHE_MAX_SIZE} must be a positive integer") residual_cache: LRUCache[tuple[int, tuple[Any, ...]], BooleanExpression] = LRUCache(maxsize=residual_cache_max_size) - partition_literal_values: dict[int, Callable[[Record], Record]] = KeyDefaultDict( - lambda spec_id: _partition_with_literal_values(partition_specs[spec_id].partition_type(self.table_metadata.schema())) - ) def residual_for(data_file: DataFile) -> BooleanExpression: partition = data_file.partition @@ -2989,9 +2966,7 @@ def residual_for(data_file: DataFile) -> BooleanExpression: try: return residual_cache[cache_key] except KeyError: - residual = residual_evaluators[data_file.spec_id].residual_for( - partition_literal_values[data_file.spec_id](partition) - ) + residual = residual_evaluators[data_file.spec_id].residual_for(partition) residual_cache[cache_key] = residual return residual @@ -3036,11 +3011,10 @@ def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool] partition_schema = Schema(*partition_type.fields) partition_expr = self.partition_filters[spec_id] evaluator = expression_evaluator(partition_schema, partition_expr, self.case_sensitive) - literal_values = _partition_with_literal_values(partition_type) # Expression evaluators keep input-specific state local to each call, so the # prepared evaluator can be shared by every manifest using this spec. - return lambda data_file: evaluator(literal_values(data_file.partition)) + return lambda data_file: evaluator(data_file.partition) def _build_metrics_evaluator(self) -> Callable[[DataFile], bool]: schema = self.table_metadata.schema() diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 9c4252e57d..0a16932cfe 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -411,7 +411,7 @@ def test_string_to_uuid_literal() -> None: uuid_str = literal(str(expected)) uuid_lit = uuid_str.to(UUIDType()) - assert expected.bytes == uuid_lit.value + assert expected == uuid_lit.value def test_string_to_decimal_literal() -> None: @@ -568,8 +568,8 @@ def test_binary_to_uuid() -> None: lit = literal(test_uuid.bytes) uuid_lit = lit.to(UUIDType()) assert uuid_lit is not None - assert lit.value == uuid_lit.value - assert uuid_lit.value == test_uuid.bytes + assert lit.value == test_uuid.bytes + assert uuid_lit.value == test_uuid def test_incompatible_binary_to_uuid() -> None: @@ -598,8 +598,8 @@ def test_fixed_to_uuid() -> None: lit = literal(test_uuid.bytes).to(FixedType(16)) uuid_lit = lit.to(UUIDType()) assert uuid_lit is not None - assert lit.value == uuid_lit.value - assert uuid_lit.value == test_uuid.bytes + assert lit.value == test_uuid.bytes + assert uuid_lit.value == test_uuid def test_incompatible_fixed_to_uuid() -> None: @@ -1038,7 +1038,7 @@ def test_uuid_literal_initialization() -> None: test_uuid = uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7") uuid_literal = literal(test_uuid) assert isinstance(uuid_literal, Literal) - assert test_uuid.bytes == uuid_literal.value + assert test_uuid == uuid_literal.value def test_uuid_to_fixed() -> None: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 906aeb2ceb..a9c418e50a 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -915,6 +915,25 @@ def test_expr_not_starts_with_to_pyarrow(bound_reference: BoundReference) -> Non ) +def test_uuid_expressions_to_pyarrow() -> None: + schema = Schema(NestedField(1, "u", UUIDType(), required=False)) + ref = BoundReference(schema.find_field(1), schema.accessor_for_field(1)) + first, second = uuid.UUID(int=1), uuid.UUID(int=2) + literals = {literal(first), literal(second)} + # PyArrow builds uuid values from their 16 bytes, so compare with expressions built from bytes in either set order + expected_in = [ + ds.field("u").isin(pa.array(values, type=pa.uuid())) + for values in ([first.bytes, second.bytes], [second.bytes, first.bytes]) + ] + for expression, expected in [ + (BoundEqualTo(ref, literal(first)), [ds.field("u") == pa.scalar(first.bytes, type=pa.uuid())]), + (BoundIn(ref, literals), expected_in), + (BoundNotIn(ref, literals), [~candidate for candidate in expected_in]), + ]: + actual = expression_to_pyarrow(expression) + assert any(actual.equals(candidate) for candidate in expected), expression + + def test_and_to_pyarrow(bound_reference: BoundReference) -> None: assert ( repr(expression_to_pyarrow(And(BoundEqualTo(bound_reference, literal("hello")), BoundIsNull(bound_reference)))) @@ -1571,6 +1590,53 @@ def test_identity_transform_column_projection_with_falsy_value( ) +def test_identity_transform_column_projection_uuid(tmp_path: str, catalog: InMemoryCatalog) -> None: + """A filter on a uuid identity partition column matches the files that do not contain the column.""" + schema = Schema( + NestedField(1, "other_field", StringType(), required=False), + NestedField(2, "partition_col", UUIDType(), required=False), + ) + partition_spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "partition_col")) + catalog.create_namespace("default") + table = catalog.create_table( + "default.test_projection_partition_uuid", + schema=schema, + partition_spec=partition_spec, + properties={TableProperties.DEFAULT_NAME_MAPPING: create_mapping_from_schema(schema).model_dump_json()}, + ) + + file_loc = f"{tmp_path}/test.parquet" + pq.write_table(pa.table([pa.array(["foo", "bar"], type=pa.string())], names=["other_field"]), file_loc) + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=pq.read_metadata(file_loc), + stats_columns=compute_statistics_plan(table.schema(), table.metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(table.schema()), + ) + partition_value = uuid.UUID(int=1) + data_file = DataFile.from_args( + content=DataFileContent.DATA, + file_path=file_loc, + file_format=FileFormat.PARQUET, + partition=Record(partition_value), + file_size_in_bytes=os.path.getsize(file_loc), + sort_order_id=None, + spec_id=table.metadata.default_spec_id, + equality_ids=None, + key_metadata=None, + **statistics.to_serialized_dict(), + ) + with table.transaction() as transaction: + with transaction.update_snapshot().overwrite() as update: + update.append_data_file(data_file) + + for row_filter, expected in [ + (f"partition_col = '{partition_value}'", ["foo", "bar"]), + (f"partition_col != '{partition_value}'", []), + ]: + result = table.scan(row_filter=row_filter, selected_fields=("other_field",)).to_arrow() + assert result["other_field"].to_pylist() == expected, row_filter + + def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None: """A null identity partition value must project as null, not as the field's initial-default.""" table_schema = Schema( @@ -3518,9 +3584,9 @@ def test_determine_partitions_with_uuid_column() -> None: partitions = list(_determine_partitions(spec, schema, arrow_table)) - # The partition values are the 16 bytes of each UUID + # The partition values are uuid.UUID, as they are read back from the manifests rows_by_partition = {p.partition_key.partition[0]: sorted(p.arrow_table_partition["id"].to_pylist()) for p in partitions} - assert rows_by_partition == {uuid.UUID(int=1).bytes: [1, 3], uuid.UUID(int=2).bytes: [2], None: [4]} + assert rows_by_partition == {uuid.UUID(int=1): [1, 3], uuid.UUID(int=2): [2], None: [4]} # The written data keeps the uuid type assert all(p.arrow_table_partition.schema.field("uuid").type == pa.uuid() for p in partitions) @@ -3564,7 +3630,7 @@ def test_scan_plans_uuid_identity_partitions(catalog: InMemoryCatalog) -> None: ) ) - # Manifests are read with uuid partition values as uuid.UUID, while a UUID literal holds the 16 bytes + # UUID partition values read from manifests must compare equal to bound UUID literals for row_filter, expected in [ (f"uuid = '{first}'", {first}), (f"uuid in ('{first}', '{second}')", {first, second}), @@ -3576,6 +3642,27 @@ def test_scan_plans_uuid_identity_partitions(catalog: InMemoryCatalog) -> None: assert scan.count() == len(expected), row_filter +def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=False), + NestedField(id=2, name="uuid", field_type=UUIDType(), required=False), + ) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_uuid_bounds", schema=schema) + for ids in ([1, 2], [3, 4]): + values = [uuid.UUID(int=value).bytes for value in ids] + table.append(pa.table({"id": pa.array(ids, type=pa.int64()), "uuid": pa.array(values, type=pa.uuid())})) + + # The lower and upper bounds of the uuid column are compared with the literal to prune files + for row_filter, expected_files in [ + (f"uuid = '{uuid.UUID(int=1)}'", 1), + (f"uuid > '{uuid.UUID(int=2)}'", 1), + (f"uuid < '{uuid.UUID(int=5)}'", 2), + (f"uuid in ('{uuid.UUID(int=1)}', '{uuid.UUID(int=4)}')", 2), + ]: + assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter + + def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="id", field_type=LongType(), required=False), diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 645cb184e1..17b465e7e4 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -22,7 +22,7 @@ import pytest from pyiceberg.exceptions import ValidationError -from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec, partition_record_value from pyiceberg.schema import Schema from pyiceberg.transforms import ( BucketTransform, @@ -220,6 +220,16 @@ def test_partition_type_missing_source_field(table_schema_simple: Schema) -> Non ) +def test_uuid_identity_partition_record_value_and_path() -> None: + schema = Schema(NestedField(field_id=1, name="u", field_type=UUIDType(), required=False)) + spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="u")) + value = UUID(int=1) + for raw in (value, value.bytes): + assert partition_record_value(spec.fields[0], raw, schema) == value, raw + # The partition path holds the uuid string, as in Java + assert spec.partition_to_path(Record(value), schema) == "u=00000000-0000-0000-0000-000000000001" + + @pytest.mark.parametrize( "source_type, value", [ diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ec14e87d52..8a6a8d3c84 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -284,9 +284,9 @@ def test_partition_to_py_raise_on_incorrect_precision_or_scale( ( UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", - b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", + uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), ), - (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7"), + (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), (FixedType(3), b"foo", b"foo"), (BinaryType(), b"foo", b"foo"), (DecimalType(5, 2), b"\x30\x39", Decimal("123.45")), @@ -322,9 +322,9 @@ def test_from_bytes(primitive_type: PrimitiveType, b: bytes, result: Any) -> Non ( UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", - b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", + uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), ), - (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7"), + (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), (FixedType(3), b"foo", b"foo"), (BinaryType(), b"foo", b"foo"), (DecimalType(5, 2), b"\x30\x39", Decimal("123.45")),