Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 16 additions & 10 deletions pyiceberg/table/update/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ class _Move:
class UpdateSchema(UpdateTableMetadata["UpdateSchema"]):
_schema: Schema
_last_column_id: itertools.count[int]
_identifier_field_ids: set[int]
_identifier_field_names: set[str]

_adds: dict[int, list[NestedField]] = {}
Expand Down Expand Up @@ -108,7 +109,8 @@ def __init__(
self._last_column_id = itertools.count(1 + self._transaction.table_metadata.last_column_id)

self._name_mapping = name_mapping
self._identifier_field_names = self._schema.identifier_field_names()
self._identifier_field_ids = set(self._schema.identifier_field_ids)
self._identifier_field_names = set()

self._adds = {}
self._updates = {}
Expand Down Expand Up @@ -331,13 +333,6 @@ def rename_column(self, path_from: str | tuple[str, ...], new_name: str) -> Upda
write_default=field_from.write_default,
)

# Lookup the field because of casing
from_field_correct_casing = self._schema.find_column_name(field_from.field_id)
if from_field_correct_casing in self._identifier_field_names:
self._identifier_field_names.remove(from_field_correct_casing)
new_identifier_path = f"{from_field_correct_casing[: -len(field_from.name)]}{new_name}"
self._identifier_field_names.add(new_identifier_path)

return self

def make_column_optional(self, path: str | tuple[str, ...]) -> UpdateSchema:
Expand All @@ -353,7 +348,13 @@ def make_column_optional(self, path: str | tuple[str, ...]) -> UpdateSchema:
return self

def set_identifier_fields(self, *fields: str) -> None:
self._identifier_field_names = set(fields)
self._identifier_field_ids = set()
self._identifier_field_names = set()
for field_name in fields:
try:
self._identifier_field_ids.add(self._schema.find_field(field_name, self._case_sensitive).field_id)
except ValueError:
self._identifier_field_names.add(field_name)

def _set_column_requirement(self, path: str | tuple[str, ...], required: bool) -> None:
path = (path,) if isinstance(path, str) else path
Expand Down Expand Up @@ -670,7 +671,12 @@ def _apply(self) -> Schema:
validate_partition_name(
partition_field.name, partition_field.transform, partition_field.source_id, new_schema, set()
)
field_ids = set()
field_ids = set(self._identifier_field_ids)
for field_id in self._identifier_field_ids:
if new_schema.find_column_name(field_id) is None:
name = self._schema.find_column_name(field_id)
raise ValueError(f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first.")

for name in self._identifier_field_names:
try:
field = new_schema.find_field(name, case_sensitive=self._case_sensitive)
Expand Down
50 changes: 50 additions & 0 deletions tests/test_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -1724,6 +1724,56 @@ def test_union_with_pa_schema(primitive_fields: NestedField, table_v2: Table) ->
assert new_schema == expected_schema


def test_rename_struct_containing_identifier_field(table_v2: Table) -> None:
schema = Schema(
NestedField(
field_id=1,
name="user",
field_type=StructType(NestedField(field_id=2, name="id", field_type=LongType(), required=True)),
required=True,
),
identifier_field_ids=[2],
)

applied = UpdateSchema(transaction=Transaction(table_v2), schema=schema).rename_column("user", "account")._apply()

assert applied.identifier_field_ids == [2]
assert applied.identifier_field_names() == {"account.id"}


def test_rename_identifier_field(table_v2: Table) -> None:
schema = Schema(NestedField(field_id=1, name="foo", field_type=StringType(), required=True), identifier_field_ids=[1])

applied = UpdateSchema(transaction=Transaction(table_v2), schema=schema).rename_column("foo", "bar")._apply()

assert applied.identifier_field_ids == [1]
assert applied.identifier_field_names() == {"bar"}


def test_rename_column_after_set_identifier_fields(table_v2: Table) -> None:
user = NestedField(
field_id=1,
name="user",
field_type=StructType(NestedField(field_id=2, name="id", field_type=LongType(), required=True)),
required=True,
)
update = UpdateSchema(transaction=Transaction(table_v2), schema=Schema(user))
update.set_identifier_fields("user.id")
applied = update.rename_column("user", "account")._apply()

assert applied.identifier_field_ids == [2]
assert applied.identifier_field_names() == {"account.id"}


def test_delete_identifier_field(table_v2: Table) -> None:
schema = Schema(NestedField(field_id=1, name="foo", field_type=StringType(), required=True), identifier_field_ids=[1])

with pytest.raises(ValueError) as exc_info:
UpdateSchema(transaction=Transaction(table_v2), schema=schema).delete_column("foo")._apply()

assert "Cannot find identifier field foo. In case of deletion, update the identifier fields first." in str(exc_info)


def test_arrow_schema() -> None:
base_schema = Schema(
NestedField(field_id=1, name="foo", field_type=StringType(), required=True),
Expand Down
Loading