diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index 828f1e877a..a0312745d2 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -77,6 +77,7 @@ class _Move: class UpdateSchema(UpdateTableMetadata["UpdateSchema"]): _schema: Schema _last_column_id: itertools.count[int] + _identifier_field_ids: set[int] _identifier_field_names: set[str] _adds: dict[int, list[NestedField]] = {} @@ -108,7 +109,8 @@ def __init__( self._last_column_id = itertools.count(1 + self._transaction.table_metadata.last_column_id) self._name_mapping = name_mapping - self._identifier_field_names = self._schema.identifier_field_names() + self._identifier_field_ids = set(self._schema.identifier_field_ids) + self._identifier_field_names = set() self._adds = {} self._updates = {} @@ -331,13 +333,6 @@ def rename_column(self, path_from: str | tuple[str, ...], new_name: str) -> Upda write_default=field_from.write_default, ) - # Lookup the field because of casing - from_field_correct_casing = self._schema.find_column_name(field_from.field_id) - if from_field_correct_casing in self._identifier_field_names: - self._identifier_field_names.remove(from_field_correct_casing) - new_identifier_path = f"{from_field_correct_casing[: -len(field_from.name)]}{new_name}" - self._identifier_field_names.add(new_identifier_path) - return self def make_column_optional(self, path: str | tuple[str, ...]) -> UpdateSchema: @@ -353,7 +348,13 @@ def make_column_optional(self, path: str | tuple[str, ...]) -> UpdateSchema: return self def set_identifier_fields(self, *fields: str) -> None: - self._identifier_field_names = set(fields) + self._identifier_field_ids = set() + self._identifier_field_names = set() + for field_name in fields: + try: + self._identifier_field_ids.add(self._schema.find_field(field_name, self._case_sensitive).field_id) + except ValueError: + self._identifier_field_names.add(field_name) def _set_column_requirement(self, path: str | tuple[str, ...], required: bool) -> None: path = (path,) if isinstance(path, str) else path @@ -670,7 +671,12 @@ def _apply(self) -> Schema: validate_partition_name( partition_field.name, partition_field.transform, partition_field.source_id, new_schema, set() ) - field_ids = set() + field_ids = set(self._identifier_field_ids) + for field_id in self._identifier_field_ids: + if new_schema.find_column_name(field_id) is None: + name = self._schema.find_column_name(field_id) + raise ValueError(f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first.") + for name in self._identifier_field_names: try: field = new_schema.find_field(name, case_sensitive=self._case_sensitive) diff --git a/tests/test_schema.py b/tests/test_schema.py index 94931277b4..15bac3bf51 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1724,6 +1724,56 @@ def test_union_with_pa_schema(primitive_fields: NestedField, table_v2: Table) -> assert new_schema == expected_schema +def test_rename_struct_containing_identifier_field(table_v2: Table) -> None: + schema = Schema( + NestedField( + field_id=1, + name="user", + field_type=StructType(NestedField(field_id=2, name="id", field_type=LongType(), required=True)), + required=True, + ), + identifier_field_ids=[2], + ) + + applied = UpdateSchema(transaction=Transaction(table_v2), schema=schema).rename_column("user", "account")._apply() + + assert applied.identifier_field_ids == [2] + assert applied.identifier_field_names() == {"account.id"} + + +def test_rename_identifier_field(table_v2: Table) -> None: + schema = Schema(NestedField(field_id=1, name="foo", field_type=StringType(), required=True), identifier_field_ids=[1]) + + applied = UpdateSchema(transaction=Transaction(table_v2), schema=schema).rename_column("foo", "bar")._apply() + + assert applied.identifier_field_ids == [1] + assert applied.identifier_field_names() == {"bar"} + + +def test_rename_column_after_set_identifier_fields(table_v2: Table) -> None: + user = NestedField( + field_id=1, + name="user", + field_type=StructType(NestedField(field_id=2, name="id", field_type=LongType(), required=True)), + required=True, + ) + update = UpdateSchema(transaction=Transaction(table_v2), schema=Schema(user)) + update.set_identifier_fields("user.id") + applied = update.rename_column("user", "account")._apply() + + assert applied.identifier_field_ids == [2] + assert applied.identifier_field_names() == {"account.id"} + + +def test_delete_identifier_field(table_v2: Table) -> None: + schema = Schema(NestedField(field_id=1, name="foo", field_type=StringType(), required=True), identifier_field_ids=[1]) + + with pytest.raises(ValueError) as exc_info: + UpdateSchema(transaction=Transaction(table_v2), schema=schema).delete_column("foo")._apply() + + assert "Cannot find identifier field foo. In case of deletion, update the identifier fields first." in str(exc_info) + + def test_arrow_schema() -> None: base_schema = Schema( NestedField(field_id=1, name="foo", field_type=StringType(), required=True),