diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index fde1885443..6b3d84135b 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -388,13 +388,17 @@ def _(_: StringType, b: bytes) -> str: @from_bytes.register(BinaryType) @from_bytes.register(FixedType) -@from_bytes.register(UUIDType) @from_bytes.register(GeometryType) @from_bytes.register(GeographyType) def _(_: PrimitiveType, b: bytes) -> bytes: return b +@from_bytes.register(UUIDType) +def _(_: UUIDType, b: bytes) -> uuid.UUID: + return uuid.UUID(bytes=b) + + @from_bytes.register(DecimalType) def _(primitive_type: DecimalType, buf: bytes) -> Decimal: unscaled = int.from_bytes(buf, "big", signed=True) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index b502b43539..c9e6eade70 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -165,7 +165,7 @@ def literal(value: L) -> Literal[L]: elif isinstance(value, str): return StringLiteral(value) elif isinstance(value, UUID): - return UUIDLiteral(value.bytes) + return UUIDLiteral(value) elif isinstance(value, bytes): return BinaryLiteral(value) elif isinstance(value, Decimal): @@ -635,8 +635,8 @@ def _(self, _: TimestamptzType) -> Literal[int]: return TimestampLiteral(timestamptz_to_micros(self.value)) @to.register(UUIDType) - def _(self, _: UUIDType) -> Literal[bytes]: - return UUIDLiteral(UUID(self.value).bytes) + def _(self, _: UUIDType) -> Literal[UUID]: + return UUIDLiteral(UUID(self.value)) @to.register(DecimalType) def _(self, type_var: DecimalType) -> Literal[Decimal]: @@ -680,26 +680,26 @@ def __repr__(self) -> str: return f"literal({repr(self.value)})" -class UUIDLiteral(Literal[bytes]): - def __init__(self, value: bytes) -> None: - super().__init__(value, bytes) +class UUIDLiteral(Literal[UUID]): + def __init__(self, value: UUID) -> None: + super().__init__(value, UUID) @model_serializer def ser_model(self) -> UUID: - return UUID(bytes=self.root) + return self.root @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert UUIDLiteral into {type_var}") @to.register(UUIDType) - def _(self, _: UUIDType) -> Literal[bytes]: + def _(self, _: UUIDType) -> Literal[UUID]: return self @to.register(FixedType) def _(self, type_var: FixedType) -> Literal[bytes]: if len(type_var) == UUID_BYTES_LENGTH: - return FixedLiteral(self.value) + return FixedLiteral(self.value.bytes) else: raise TypeError( f"Cannot convert UUIDLiteral into {type_var}, different length: {len(type_var)} <> {UUID_BYTES_LENGTH}" @@ -707,7 +707,7 @@ def _(self, type_var: FixedType) -> Literal[bytes]: @to.register(BinaryType) def _(self, _: BinaryType) -> Literal[bytes]: - return BinaryLiteral(self.value) + return BinaryLiteral(self.value.bytes) class FixedLiteral(Literal[bytes]): @@ -736,9 +736,9 @@ def _(self, _: BinaryType) -> Literal[bytes]: return BinaryLiteral(self.value) @to.register(UUIDType) - def _(self, type_var: UUIDType) -> Literal[bytes]: + def _(self, type_var: UUIDType) -> Literal[UUID]: if len(self.value) == UUID_BYTES_LENGTH: - return UUIDLiteral(self.value) + return UUIDLiteral(UUID(bytes=self.value)) else: raise TypeError( f"Could not convert {self.value!r} into a {type_var}, lengths differ {len(self.value)} <> {UUID_BYTES_LENGTH}" @@ -771,9 +771,9 @@ def _(self, type_var: FixedType) -> Literal[bytes]: ) @to.register(UUIDType) - def _(self, type_var: UUIDType) -> Literal[bytes]: + def _(self, type_var: UUIDType) -> Literal[UUID]: if len(self.value) == UUID_BYTES_LENGTH: - return UUIDLiteral(self.value) + return UUIDLiteral(UUID(bytes=self.value)) else: raise TypeError( f"Cannot convert BinaryLiteral into {type_var}, different length: {UUID_BYTES_LENGTH} <> {len(self.value)}" diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..8e99389de4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -897,10 +897,15 @@ def _literal_arrow_type(iceberg_type: IcebergType, values: Iterable[Any]) -> pa. return visit(iceberg_type, _ConvertToArrowSchema()) +def _to_arrow_value(value: Any) -> Any: + # PyArrow builds uuid values from their 16 bytes; older PyArrow releases reject a uuid.UUID + return value.bytes if isinstance(value, uuid.UUID) else value + + def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: if not isinstance(iceberg_type, PrimitiveType): raise ValueError(f"Expected primitive type, got: {iceberg_type}") - return pa.scalar(value=value, type=_literal_arrow_type(iceberg_type, [value])) + return pa.scalar(value=_to_arrow_value(value), type=_literal_arrow_type(iceberg_type, [value])) class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): @@ -937,11 +942,15 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]: return term.ref().field.name def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) + pyarrow_literals = pa.array( + [_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals) + ) return pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) + pyarrow_literals = pa.array( + [_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals) + ) return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm) -> pc.Expression: diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 34c6f6aaaa..6d79dce18a 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -487,7 +487,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema: Return the Partition Record representation of the value. The value is first converted to internal partition representation. - For example, UUID is converted to bytes[16], DateType to days since epoch, etc. + For example, UUID values become uuid.UUID, and DateType values become days since epoch. Then the corresponding PartitionField's transform is applied to return the final partition record value. @@ -498,7 +498,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema: @singledispatch def _to_partition_representation(type: IcebergType, value: Any) -> Any: - """Strip the logical type into the physical type. + """Return the internal partition representation of a value. It can be that the value is already transformed into its physical type, in this case it will return the original value. Keep in mind that the @@ -560,13 +560,13 @@ def _(type: IcebergType, value: int | time | None) -> int | None: @_to_partition_representation.register(UUIDType) -def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> bytes | int | None: +def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> uuid.UUID | int | None: if value is None: return None elif isinstance(value, bytes): - return value # IdentityTransform + return uuid.UUID(bytes=value) # IdentityTransform elif isinstance(value, uuid.UUID): - return value.bytes # IdentityTransform + return value # IdentityTransform elif isinstance(value, int): return value # BucketTransform else: diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 9c4252e57d..0a16932cfe 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -411,7 +411,7 @@ def test_string_to_uuid_literal() -> None: uuid_str = literal(str(expected)) uuid_lit = uuid_str.to(UUIDType()) - assert expected.bytes == uuid_lit.value + assert expected == uuid_lit.value def test_string_to_decimal_literal() -> None: @@ -568,8 +568,8 @@ def test_binary_to_uuid() -> None: lit = literal(test_uuid.bytes) uuid_lit = lit.to(UUIDType()) assert uuid_lit is not None - assert lit.value == uuid_lit.value - assert uuid_lit.value == test_uuid.bytes + assert lit.value == test_uuid.bytes + assert uuid_lit.value == test_uuid def test_incompatible_binary_to_uuid() -> None: @@ -598,8 +598,8 @@ def test_fixed_to_uuid() -> None: lit = literal(test_uuid.bytes).to(FixedType(16)) uuid_lit = lit.to(UUIDType()) assert uuid_lit is not None - assert lit.value == uuid_lit.value - assert uuid_lit.value == test_uuid.bytes + assert lit.value == test_uuid.bytes + assert uuid_lit.value == test_uuid def test_incompatible_fixed_to_uuid() -> None: @@ -1038,7 +1038,7 @@ def test_uuid_literal_initialization() -> None: test_uuid = uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7") uuid_literal = literal(test_uuid) assert isinstance(uuid_literal, Literal) - assert test_uuid.bytes == uuid_literal.value + assert test_uuid == uuid_literal.value def test_uuid_to_fixed() -> None: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..a9c418e50a 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -915,6 +915,25 @@ def test_expr_not_starts_with_to_pyarrow(bound_reference: BoundReference) -> Non ) +def test_uuid_expressions_to_pyarrow() -> None: + schema = Schema(NestedField(1, "u", UUIDType(), required=False)) + ref = BoundReference(schema.find_field(1), schema.accessor_for_field(1)) + first, second = uuid.UUID(int=1), uuid.UUID(int=2) + literals = {literal(first), literal(second)} + # PyArrow builds uuid values from their 16 bytes, so compare with expressions built from bytes in either set order + expected_in = [ + ds.field("u").isin(pa.array(values, type=pa.uuid())) + for values in ([first.bytes, second.bytes], [second.bytes, first.bytes]) + ] + for expression, expected in [ + (BoundEqualTo(ref, literal(first)), [ds.field("u") == pa.scalar(first.bytes, type=pa.uuid())]), + (BoundIn(ref, literals), expected_in), + (BoundNotIn(ref, literals), [~candidate for candidate in expected_in]), + ]: + actual = expression_to_pyarrow(expression) + assert any(actual.equals(candidate) for candidate in expected), expression + + def test_and_to_pyarrow(bound_reference: BoundReference) -> None: assert ( repr(expression_to_pyarrow(And(BoundEqualTo(bound_reference, literal("hello")), BoundIsNull(bound_reference)))) @@ -1571,6 +1590,53 @@ def test_identity_transform_column_projection_with_falsy_value( ) +def test_identity_transform_column_projection_uuid(tmp_path: str, catalog: InMemoryCatalog) -> None: + """A filter on a uuid identity partition column matches the files that do not contain the column.""" + schema = Schema( + NestedField(1, "other_field", StringType(), required=False), + NestedField(2, "partition_col", UUIDType(), required=False), + ) + partition_spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "partition_col")) + catalog.create_namespace("default") + table = catalog.create_table( + "default.test_projection_partition_uuid", + schema=schema, + partition_spec=partition_spec, + properties={TableProperties.DEFAULT_NAME_MAPPING: create_mapping_from_schema(schema).model_dump_json()}, + ) + + file_loc = f"{tmp_path}/test.parquet" + pq.write_table(pa.table([pa.array(["foo", "bar"], type=pa.string())], names=["other_field"]), file_loc) + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=pq.read_metadata(file_loc), + stats_columns=compute_statistics_plan(table.schema(), table.metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(table.schema()), + ) + partition_value = uuid.UUID(int=1) + data_file = DataFile.from_args( + content=DataFileContent.DATA, + file_path=file_loc, + file_format=FileFormat.PARQUET, + partition=Record(partition_value), + file_size_in_bytes=os.path.getsize(file_loc), + sort_order_id=None, + spec_id=table.metadata.default_spec_id, + equality_ids=None, + key_metadata=None, + **statistics.to_serialized_dict(), + ) + with table.transaction() as transaction: + with transaction.update_snapshot().overwrite() as update: + update.append_data_file(data_file) + + for row_filter, expected in [ + (f"partition_col = '{partition_value}'", ["foo", "bar"]), + (f"partition_col != '{partition_value}'", []), + ]: + result = table.scan(row_filter=row_filter, selected_fields=("other_field",)).to_arrow() + assert result["other_field"].to_pylist() == expected, row_filter + + def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None: """A null identity partition value must project as null, not as the field's initial-default.""" table_schema = Schema( @@ -3518,9 +3584,9 @@ def test_determine_partitions_with_uuid_column() -> None: partitions = list(_determine_partitions(spec, schema, arrow_table)) - # The partition values are the 16 bytes of each UUID + # The partition values are uuid.UUID, as they are read back from the manifests rows_by_partition = {p.partition_key.partition[0]: sorted(p.arrow_table_partition["id"].to_pylist()) for p in partitions} - assert rows_by_partition == {uuid.UUID(int=1).bytes: [1, 3], uuid.UUID(int=2).bytes: [2], None: [4]} + assert rows_by_partition == {uuid.UUID(int=1): [1, 3], uuid.UUID(int=2): [2], None: [4]} # The written data keeps the uuid type assert all(p.arrow_table_partition.schema.field("uuid").type == pa.uuid() for p in partitions) @@ -3546,6 +3612,57 @@ def test_append_uuid_to_table_partitioned_on_uuid(catalog: InMemoryCatalog) -> N assert {task.file.partition[0] for task in table.scan().plan_files()} == {uuid.UUID(int=1), uuid.UUID(int=2), None} +def test_scan_plans_uuid_identity_partitions(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=False), + NestedField(id=2, name="uuid", field_type=UUIDType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="uuid")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_uuid_partitioned", schema=schema, partition_spec=spec) + first, second, third = uuid.UUID(int=1), uuid.UUID(int=2), uuid.UUID(int=3) + table.append( + pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int64()), + "uuid": pa.array([first.bytes, second.bytes, third.bytes], type=pa.uuid()), + } + ) + ) + + # UUID partition values read from manifests must compare equal to bound UUID literals + for row_filter, expected in [ + (f"uuid = '{first}'", {first}), + (f"uuid in ('{first}', '{second}')", {first, second}), + (f"uuid != '{first}'", {second, third}), + (f"uuid not in ('{first}', '{second}')", {third}), + ]: + scan = table.scan(row_filter=row_filter) + assert {task.file.partition[0] for task in scan.plan_files()} == expected, row_filter + assert scan.count() == len(expected), row_filter + + +def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=False), + NestedField(id=2, name="uuid", field_type=UUIDType(), required=False), + ) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_uuid_bounds", schema=schema) + for ids in ([1, 2], [3, 4]): + values = [uuid.UUID(int=value).bytes for value in ids] + table.append(pa.table({"id": pa.array(ids, type=pa.int64()), "uuid": pa.array(values, type=pa.uuid())})) + + # The lower and upper bounds of the uuid column are compared with the literal to prune files + for row_filter, expected_files in [ + (f"uuid = '{uuid.UUID(int=1)}'", 1), + (f"uuid > '{uuid.UUID(int=2)}'", 1), + (f"uuid < '{uuid.UUID(int=5)}'", 2), + (f"uuid in ('{uuid.UUID(int=1)}', '{uuid.UUID(int=4)}')", 2), + ]: + assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter + + def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="id", field_type=LongType(), required=False), diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 645cb184e1..17b465e7e4 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -22,7 +22,7 @@ import pytest from pyiceberg.exceptions import ValidationError -from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec, partition_record_value from pyiceberg.schema import Schema from pyiceberg.transforms import ( BucketTransform, @@ -220,6 +220,16 @@ def test_partition_type_missing_source_field(table_schema_simple: Schema) -> Non ) +def test_uuid_identity_partition_record_value_and_path() -> None: + schema = Schema(NestedField(field_id=1, name="u", field_type=UUIDType(), required=False)) + spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="u")) + value = UUID(int=1) + for raw in (value, value.bytes): + assert partition_record_value(spec.fields[0], raw, schema) == value, raw + # The partition path holds the uuid string, as in Java + assert spec.partition_to_path(Record(value), schema) == "u=00000000-0000-0000-0000-000000000001" + + @pytest.mark.parametrize( "source_type, value", [ diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ec14e87d52..8a6a8d3c84 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -284,9 +284,9 @@ def test_partition_to_py_raise_on_incorrect_precision_or_scale( ( UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", - b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", + uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), ), - (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7"), + (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), (FixedType(3), b"foo", b"foo"), (BinaryType(), b"foo", b"foo"), (DecimalType(5, 2), b"\x30\x39", Decimal("123.45")), @@ -322,9 +322,9 @@ def test_from_bytes(primitive_type: PrimitiveType, b: bytes, result: Any) -> Non ( UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", - b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", + uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), ), - (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7"), + (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), (FixedType(3), b"foo", b"foo"), (BinaryType(), b"foo", b"foo"), (DecimalType(5, 2), b"\x30\x39", Decimal("123.45")),