From 6017094e14fe7d41542405956b1bb73585f22be0 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 01:02:46 +0530 Subject: [PATCH] Track identifier fields by ID in UpdateSchema Identifier fields are stored by field ID in the spec, but UpdateSchema kept them as dotted names while staging and only rewrote an exact name match on rename. Renaming a struct that holds an identifier field left the staged name pointing at the old path, so the commit failed with "Cannot find identifier field". Keep the identifier field IDs of the starting schema and resolve names passed to set_identifier_fields to IDs where they exist. Names that only exist after the update, such as newly added columns, are still resolved against the resulting schema at commit. --- pyiceberg/table/update/schema.py | 26 ++++++++++------- tests/test_schema.py | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index 828f1e877a..a0312745d2 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -77,6 +77,7 @@ class _Move: class UpdateSchema(UpdateTableMetadata["UpdateSchema"]): _schema: Schema _last_column_id: itertools.count[int] + _identifier_field_ids: set[int] _identifier_field_names: set[str] _adds: dict[int, list[NestedField]] = {} @@ -108,7 +109,8 @@ def __init__( self._last_column_id = itertools.count(1 + self._transaction.table_metadata.last_column_id) self._name_mapping = name_mapping - self._identifier_field_names = self._schema.identifier_field_names() + self._identifier_field_ids = set(self._schema.identifier_field_ids) + self._identifier_field_names = set() self._adds = {} self._updates = {} @@ -331,13 +333,6 @@ def rename_column(self, path_from: str | tuple[str, ...], new_name: str) -> Upda write_default=field_from.write_default, ) - # Lookup the field because of casing - from_field_correct_casing = self._schema.find_column_name(field_from.field_id) - if from_field_correct_casing in self._identifier_field_names: - self._identifier_field_names.remove(from_field_correct_casing) - new_identifier_path = f"{from_field_correct_casing[: -len(field_from.name)]}{new_name}" - self._identifier_field_names.add(new_identifier_path) - return self def make_column_optional(self, path: str | tuple[str, ...]) -> UpdateSchema: @@ -353,7 +348,13 @@ def make_column_optional(self, path: str | tuple[str, ...]) -> UpdateSchema: return self def set_identifier_fields(self, *fields: str) -> None: - self._identifier_field_names = set(fields) + self._identifier_field_ids = set() + self._identifier_field_names = set() + for field_name in fields: + try: + self._identifier_field_ids.add(self._schema.find_field(field_name, self._case_sensitive).field_id) + except ValueError: + self._identifier_field_names.add(field_name) def _set_column_requirement(self, path: str | tuple[str, ...], required: bool) -> None: path = (path,) if isinstance(path, str) else path @@ -670,7 +671,12 @@ def _apply(self) -> Schema: validate_partition_name( partition_field.name, partition_field.transform, partition_field.source_id, new_schema, set() ) - field_ids = set() + field_ids = set(self._identifier_field_ids) + for field_id in self._identifier_field_ids: + if new_schema.find_column_name(field_id) is None: + name = self._schema.find_column_name(field_id) + raise ValueError(f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first.") + for name in self._identifier_field_names: try: field = new_schema.find_field(name, case_sensitive=self._case_sensitive) diff --git a/tests/test_schema.py b/tests/test_schema.py index 94931277b4..15bac3bf51 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1724,6 +1724,56 @@ def test_union_with_pa_schema(primitive_fields: NestedField, table_v2: Table) -> assert new_schema == expected_schema +def test_rename_struct_containing_identifier_field(table_v2: Table) -> None: + schema = Schema( + NestedField( + field_id=1, + name="user", + field_type=StructType(NestedField(field_id=2, name="id", field_type=LongType(), required=True)), + required=True, + ), + identifier_field_ids=[2], + ) + + applied = UpdateSchema(transaction=Transaction(table_v2), schema=schema).rename_column("user", "account")._apply() + + assert applied.identifier_field_ids == [2] + assert applied.identifier_field_names() == {"account.id"} + + +def test_rename_identifier_field(table_v2: Table) -> None: + schema = Schema(NestedField(field_id=1, name="foo", field_type=StringType(), required=True), identifier_field_ids=[1]) + + applied = UpdateSchema(transaction=Transaction(table_v2), schema=schema).rename_column("foo", "bar")._apply() + + assert applied.identifier_field_ids == [1] + assert applied.identifier_field_names() == {"bar"} + + +def test_rename_column_after_set_identifier_fields(table_v2: Table) -> None: + user = NestedField( + field_id=1, + name="user", + field_type=StructType(NestedField(field_id=2, name="id", field_type=LongType(), required=True)), + required=True, + ) + update = UpdateSchema(transaction=Transaction(table_v2), schema=Schema(user)) + update.set_identifier_fields("user.id") + applied = update.rename_column("user", "account")._apply() + + assert applied.identifier_field_ids == [2] + assert applied.identifier_field_names() == {"account.id"} + + +def test_delete_identifier_field(table_v2: Table) -> None: + schema = Schema(NestedField(field_id=1, name="foo", field_type=StringType(), required=True), identifier_field_ids=[1]) + + with pytest.raises(ValueError) as exc_info: + UpdateSchema(transaction=Transaction(table_v2), schema=schema).delete_column("foo")._apply() + + assert "Cannot find identifier field foo. In case of deletion, update the identifier fields first." in str(exc_info) + + def test_arrow_schema() -> None: base_schema = Schema( NestedField(field_id=1, name="foo", field_type=StringType(), required=True),