diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 99f983074b..2779ea673e 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -159,9 +159,22 @@ def _lazy_id_to_parent(self) -> dict[int, int]: def _lazy_name_to_id_lower(self) -> dict[str, int]: """Return an index of lower-case field names to field IDs. + Raises: + ValueError: When two fields have names that differ only by case. + This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. """ - return {name.lower(): field_id for name, field_id in self._name_to_id.items()} + index: dict[str, int] = {} + for name, field_id in self._name_to_id.items(): + lower_name = name.lower() + # Names that map to the same field ID (for example, short names of list element structs) are not collisions + if lower_name in index and index[lower_name] != field_id: + raise ValueError( + f"Cannot build lower case index: {self._lazy_id_to_name[index[lower_name]]} and " + f"{self._lazy_id_to_name[field_id]} collide" + ) + index[lower_name] = field_id + return index @cached_property def _lazy_id_to_name(self) -> dict[int, str]: @@ -207,13 +220,25 @@ def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> Nest raise ValueError(f"Could not find field with id: {name_or_id}") return self._lazy_id_to_field[name_or_id] + field = self._find_field_or_none(name_or_id, case_sensitive) + if field is None: + raise ValueError(f"Could not find field with name {name_or_id}, case_sensitive={case_sensitive}") + + return field + + def _find_field_or_none(self, name: str, case_sensitive: bool) -> NestedField | None: + """Return the field with the given name, or None when no field has that name. + + Raises: + ValueError: When case_sensitive is False and two fields have names that differ only by case. + """ if case_sensitive: - field_id = self._name_to_id.get(name_or_id) + field_id = self._name_to_id.get(name) else: - field_id = self._lazy_name_to_id_lower.get(name_or_id.lower()) + field_id = self._lazy_name_to_id_lower.get(name.lower()) if field_id is None: - raise ValueError(f"Could not find field with name {name_or_id}, case_sensitive={case_sensitive}") + return None return self._lazy_id_to_field[field_id] diff --git a/pyiceberg/table/metadata_columns.py b/pyiceberg/table/metadata_columns.py index c5d8bcf5da..5e5972532a 100644 --- a/pyiceberg/table/metadata_columns.py +++ b/pyiceberg/table/metadata_columns.py @@ -51,11 +51,7 @@ def _find_metadata_column(name: str, case_sensitive: bool) -> NestedField | None def _has_field(schema: Schema, name: str, case_sensitive: bool) -> bool: - try: - schema.find_field(name, case_sensitive=case_sensitive) - return True - except ValueError: - return False + return schema._find_field_or_none(name, case_sensitive) is not None def split_selected_fields( diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index 828f1e877a..b2b43bd90c 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -211,11 +211,7 @@ def add_column( parent_id = parent_field.field_id - existing_field = None - try: - existing_field = self._schema.find_field(full_name, self._case_sensitive) - except ValueError: - pass + existing_field = self._schema._find_field_or_none(full_name, self._case_sensitive) if existing_field is not None and existing_field.field_id not in self._deletes: raise ValueError(f"Cannot add column, name already exists: {full_name}") @@ -503,10 +499,9 @@ def update_column( return self def _find_for_move(self, name: str) -> int | None: - try: - return self._schema.find_field(name, self._case_sensitive).field_id - except ValueError: - pass + field = self._schema._find_field_or_none(name, self._case_sensitive) + if field is not None: + return field.field_id return self._added_name_to_id.get(name) @@ -672,12 +667,9 @@ def _apply(self) -> Schema: ) field_ids = set() for name in self._identifier_field_names: - try: - field = new_schema.find_field(name, case_sensitive=self._case_sensitive) - except ValueError as e: - raise ValueError( - f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first." - ) from e + field = new_schema._find_field_or_none(name, case_sensitive=self._case_sensitive) + if field is None: + raise ValueError(f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first.") field_ids.add(field.field_id) diff --git a/tests/table/test_init.py b/tests/table/test_init.py index af9a11a452..6811e55bd5 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -382,6 +382,59 @@ def test_table_scan_projection_single_column_case_sensitive(table_v2: Table) -> assert projection_schema.schema_id == 1 +def test_case_insensitive_operations_raise_on_case_variant_columns(catalog: Catalog) -> None: + """A case-insensitive lookup of ID on a table with ID and id raises, instead of acting on the id column.""" + import pyarrow as pa + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "ID", LongType(), required=False), NestedField(2, "id", LongType(), required=False)) + table = catalog.create_table("default.case_variants", schema=schema) + table.append(pa.table({"ID": pa.array([5, 6], pa.int64()), "id": pa.array([7, 5], pa.int64())})) + collision = "Cannot build lower case index: ID and id collide" + + with pytest.raises(ValueError, match=collision): + table.delete("ID = 5", case_sensitive=False) + with pytest.raises(ValueError, match=collision): + table.scan(row_filter=EqualTo("ID", 5), case_sensitive=False).to_arrow() + with pytest.raises(ValueError, match=collision): + table.update_sort_order(case_sensitive=False).asc("ID", IdentityTransform()).commit() + with pytest.raises(ValueError, match=collision): + with table.update_schema(case_sensitive=False) as update: + update.delete_column("ID") + with pytest.raises(ValueError, match=collision): + with table.update_schema(case_sensitive=False) as update: + update.rename_column("ID", "key") + with pytest.raises(ValueError, match=collision): + with table.update_schema(case_sensitive=False) as update: + update.add_column("Id", LongType()) + with pytest.raises(ValueError, match=collision): + with table.update_schema(case_sensitive=False) as update: + update.move_first("ID") + + reloaded = catalog.load_table("default.case_variants") + assert reloaded.schema() == schema + assert sorted((row["ID"], row["id"]) for row in reloaded.scan().to_arrow().to_pylist()) == [(5, 7), (6, 5)] + + +def test_case_sensitive_and_unambiguous_operations_are_unchanged(catalog: Catalog) -> None: + """Exact-case lookups still resolve each column on a table with ID and id, and unambiguous names work case-insensitively.""" + import pyarrow as pa + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "ID", LongType(), required=False), NestedField(2, "id", LongType(), required=False)) + table = catalog.create_table("default.case_variants_exact", schema=schema) + table.append(pa.table({"ID": pa.array([5, 6], pa.int64()), "id": pa.array([7, 5], pa.int64())})) + + assert table.scan(row_filter=EqualTo("ID", 5)).to_arrow()["id"].to_pylist() == [7] + table.delete("ID = 5") + assert sorted((row["ID"], row["id"]) for row in table.scan().to_arrow().to_pylist()) == [(6, 5)] + + unambiguous = catalog.create_table("default.unambiguous", schema=Schema(NestedField(1, "ID", LongType(), required=False))) + unambiguous.append(pa.table({"ID": pa.array([5, 6], pa.int64())})) + unambiguous.delete("id = 5", case_sensitive=False) + assert unambiguous.scan().to_arrow()["ID"].to_pylist() == [6] + + def test_table_scan_projection_unknown_column(table_v2: Table) -> None: scan = table_v2.scan() diff --git a/tests/test_schema.py b/tests/test_schema.py index 94931277b4..d0178b381e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -34,6 +34,7 @@ sanitize_column_names, ) from pyiceberg.table import Table, Transaction +from pyiceberg.table.metadata_columns import project_schema from pyiceberg.table.update.schema import UpdateSchema from pyiceberg.typedef import EMPTY_DICT, StructProtocol from pyiceberg.types import ( @@ -880,6 +881,83 @@ def test_schema_select_cant_be_found(table_schema_nested: Schema) -> None: assert "Could not find column: 'BAZ'" in str(exc_info.value) +def test_schema_case_insensitive_lookup_of_case_variant_names_raises() -> None: + schema = Schema( + NestedField(field_id=1, name="ID", field_type=LongType(), required=False), + NestedField(field_id=2, name="id", field_type=LongType(), required=False), + ) + assert schema.find_field("ID").field_id == 1 + assert schema.find_field("id").field_id == 2 + for name in ("ID", "Id", "id"): + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + schema.find_field(name, case_sensitive=False) + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + schema.select(name, case_sensitive=False) + + +def test_schema_case_insensitive_lookup_of_nested_case_variant_names_raises() -> None: + schema = Schema( + NestedField( + field_id=1, + name="profile", + field_type=StructType( + NestedField(field_id=2, name="CODE", field_type=LongType(), required=False), + NestedField(field_id=3, name="code", field_type=LongType(), required=False), + ), + required=False, + ), + ) + assert schema.find_field("profile.code").field_id == 3 + with pytest.raises(ValueError, match="Cannot build lower case index: profile.CODE and profile.code collide"): + schema.find_field("profile.code", case_sensitive=False) + + +def test_schema_case_insensitive_lookup_of_list_element_struct_field() -> None: + """The short name of a list element struct field maps to the same field ID, which is not a collision.""" + schema = Schema( + NestedField( + field_id=1, + name="items", + field_type=ListType( + element_id=2, + element_type=StructType(NestedField(field_id=3, name="value", field_type=LongType(), required=False)), + element_required=False, + ), + required=False, + ), + ) + assert schema.find_field("ITEMS.VALUE", case_sensitive=False).field_id == 3 + assert schema.find_field("items.element.value", case_sensitive=False).field_id == 3 + + +def test_schema_case_insensitive_lookup_rejects_collision_for_any_name() -> None: + schema = Schema( + NestedField(1, "ID", LongType()), + NestedField(2, "id", LongType()), + NestedField(3, "safe", LongType()), + ) + # As in Java, the lower-case index cannot be built, so unambiguous and missing names fail too + for name in ("safe", "missing"): + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + schema.find_field(name, case_sensitive=False) + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + schema.select(name, case_sensitive=False) + + +def test_metadata_column_projection_propagates_case_collision() -> None: + schema = Schema(NestedField(1, "ID", LongType()), NestedField(2, "id", LongType())) + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + project_schema(schema, ("_row_id",), case_sensitive=False, format_version=3) + + +def test_identifier_field_lookup_propagates_case_collision(table_v2: Table) -> None: + schema = Schema(NestedField(1, "ID", LongType(), required=True), NestedField(2, "id", LongType(), required=True)) + update = UpdateSchema(transaction=Transaction(table_v2), schema=schema, case_sensitive=False) + update.set_identifier_fields("ID") + with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"): + update._apply() + + def can_promote_decimal(file_type: DecimalType, read_type: DecimalType) -> bool: return file_type.precision <= read_type.precision and file_type.scale == read_type.scale