Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions pyiceberg/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,22 @@ def _lazy_id_to_parent(self) -> dict[int, int]:
def _lazy_name_to_id_lower(self) -> dict[str, int]:
"""Return an index of lower-case field names to field IDs.

Raises:
ValueError: When two fields have names that differ only by case.

This is calculated once when called for the first time. Subsequent calls to this method will use a cached index.
"""
return {name.lower(): field_id for name, field_id in self._name_to_id.items()}
index: dict[str, int] = {}
for name, field_id in self._name_to_id.items():
lower_name = name.lower()
# Names that map to the same field ID (for example, short names of list element structs) are not collisions
if lower_name in index and index[lower_name] != field_id:
raise ValueError(
f"Cannot build lower case index: {self._lazy_id_to_name[index[lower_name]]} and "
f"{self._lazy_id_to_name[field_id]} collide"
)
index[lower_name] = field_id
return index

@cached_property
def _lazy_id_to_name(self) -> dict[int, str]:
Expand Down Expand Up @@ -207,13 +220,25 @@ def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> Nest
raise ValueError(f"Could not find field with id: {name_or_id}")
return self._lazy_id_to_field[name_or_id]

field = self._find_field_or_none(name_or_id, case_sensitive)
if field is None:
raise ValueError(f"Could not find field with name {name_or_id}, case_sensitive={case_sensitive}")

return field

def _find_field_or_none(self, name: str, case_sensitive: bool) -> NestedField | None:
"""Return the field with the given name, or None when no field has that name.

Raises:
ValueError: When case_sensitive is False and two fields have names that differ only by case.
"""
if case_sensitive:
field_id = self._name_to_id.get(name_or_id)
field_id = self._name_to_id.get(name)
else:
field_id = self._lazy_name_to_id_lower.get(name_or_id.lower())
field_id = self._lazy_name_to_id_lower.get(name.lower())

if field_id is None:
raise ValueError(f"Could not find field with name {name_or_id}, case_sensitive={case_sensitive}")
return None

return self._lazy_id_to_field[field_id]

Expand Down
6 changes: 1 addition & 5 deletions pyiceberg/table/metadata_columns.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,7 @@ def _find_metadata_column(name: str, case_sensitive: bool) -> NestedField | None


def _has_field(schema: Schema, name: str, case_sensitive: bool) -> bool:
try:
schema.find_field(name, case_sensitive=case_sensitive)
return True
except ValueError:
return False
return schema._find_field_or_none(name, case_sensitive) is not None


def split_selected_fields(
Expand Down
22 changes: 7 additions & 15 deletions pyiceberg/table/update/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,11 +211,7 @@ def add_column(

parent_id = parent_field.field_id

existing_field = None
try:
existing_field = self._schema.find_field(full_name, self._case_sensitive)
except ValueError:
pass
existing_field = self._schema._find_field_or_none(full_name, self._case_sensitive)

if existing_field is not None and existing_field.field_id not in self._deletes:
raise ValueError(f"Cannot add column, name already exists: {full_name}")
Expand Down Expand Up @@ -503,10 +499,9 @@ def update_column(
return self

def _find_for_move(self, name: str) -> int | None:
try:
return self._schema.find_field(name, self._case_sensitive).field_id
except ValueError:
pass
field = self._schema._find_field_or_none(name, self._case_sensitive)
if field is not None:
return field.field_id

return self._added_name_to_id.get(name)

Expand Down Expand Up @@ -672,12 +667,9 @@ def _apply(self) -> Schema:
)
field_ids = set()
for name in self._identifier_field_names:
try:
field = new_schema.find_field(name, case_sensitive=self._case_sensitive)
except ValueError as e:
raise ValueError(
f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first."
) from e
field = new_schema._find_field_or_none(name, case_sensitive=self._case_sensitive)
if field is None:
raise ValueError(f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first.")

field_ids.add(field.field_id)

Expand Down
53 changes: 53 additions & 0 deletions tests/table/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,59 @@ def test_table_scan_projection_single_column_case_sensitive(table_v2: Table) ->
assert projection_schema.schema_id == 1


def test_case_insensitive_operations_raise_on_case_variant_columns(catalog: Catalog) -> None:
"""A case-insensitive lookup of ID on a table with ID and id raises, instead of acting on the id column."""
import pyarrow as pa

catalog.create_namespace("default")
schema = Schema(NestedField(1, "ID", LongType(), required=False), NestedField(2, "id", LongType(), required=False))
table = catalog.create_table("default.case_variants", schema=schema)
table.append(pa.table({"ID": pa.array([5, 6], pa.int64()), "id": pa.array([7, 5], pa.int64())}))
collision = "Cannot build lower case index: ID and id collide"

with pytest.raises(ValueError, match=collision):
table.delete("ID = 5", case_sensitive=False)
with pytest.raises(ValueError, match=collision):
table.scan(row_filter=EqualTo("ID", 5), case_sensitive=False).to_arrow()
with pytest.raises(ValueError, match=collision):
table.update_sort_order(case_sensitive=False).asc("ID", IdentityTransform()).commit()
with pytest.raises(ValueError, match=collision):
with table.update_schema(case_sensitive=False) as update:
update.delete_column("ID")
with pytest.raises(ValueError, match=collision):
with table.update_schema(case_sensitive=False) as update:
update.rename_column("ID", "key")
with pytest.raises(ValueError, match=collision):
with table.update_schema(case_sensitive=False) as update:
update.add_column("Id", LongType())
with pytest.raises(ValueError, match=collision):
with table.update_schema(case_sensitive=False) as update:
update.move_first("ID")

reloaded = catalog.load_table("default.case_variants")
assert reloaded.schema() == schema
assert sorted((row["ID"], row["id"]) for row in reloaded.scan().to_arrow().to_pylist()) == [(5, 7), (6, 5)]


def test_case_sensitive_and_unambiguous_operations_are_unchanged(catalog: Catalog) -> None:
"""Exact-case lookups still resolve each column on a table with ID and id, and unambiguous names work case-insensitively."""
import pyarrow as pa

catalog.create_namespace("default")
schema = Schema(NestedField(1, "ID", LongType(), required=False), NestedField(2, "id", LongType(), required=False))
table = catalog.create_table("default.case_variants_exact", schema=schema)
table.append(pa.table({"ID": pa.array([5, 6], pa.int64()), "id": pa.array([7, 5], pa.int64())}))

assert table.scan(row_filter=EqualTo("ID", 5)).to_arrow()["id"].to_pylist() == [7]
table.delete("ID = 5")
assert sorted((row["ID"], row["id"]) for row in table.scan().to_arrow().to_pylist()) == [(6, 5)]

unambiguous = catalog.create_table("default.unambiguous", schema=Schema(NestedField(1, "ID", LongType(), required=False)))
unambiguous.append(pa.table({"ID": pa.array([5, 6], pa.int64())}))
unambiguous.delete("id = 5", case_sensitive=False)
assert unambiguous.scan().to_arrow()["ID"].to_pylist() == [6]


def test_table_scan_projection_unknown_column(table_v2: Table) -> None:
scan = table_v2.scan()

Expand Down
78 changes: 78 additions & 0 deletions tests/test_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
sanitize_column_names,
)
from pyiceberg.table import Table, Transaction
from pyiceberg.table.metadata_columns import project_schema
from pyiceberg.table.update.schema import UpdateSchema
from pyiceberg.typedef import EMPTY_DICT, StructProtocol
from pyiceberg.types import (
Expand Down Expand Up @@ -880,6 +881,83 @@ def test_schema_select_cant_be_found(table_schema_nested: Schema) -> None:
assert "Could not find column: 'BAZ'" in str(exc_info.value)


def test_schema_case_insensitive_lookup_of_case_variant_names_raises() -> None:
schema = Schema(
NestedField(field_id=1, name="ID", field_type=LongType(), required=False),
NestedField(field_id=2, name="id", field_type=LongType(), required=False),
)
assert schema.find_field("ID").field_id == 1
assert schema.find_field("id").field_id == 2
for name in ("ID", "Id", "id"):
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
schema.find_field(name, case_sensitive=False)
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
schema.select(name, case_sensitive=False)


def test_schema_case_insensitive_lookup_of_nested_case_variant_names_raises() -> None:
schema = Schema(
NestedField(
field_id=1,
name="profile",
field_type=StructType(
NestedField(field_id=2, name="CODE", field_type=LongType(), required=False),
NestedField(field_id=3, name="code", field_type=LongType(), required=False),
),
required=False,
),
)
assert schema.find_field("profile.code").field_id == 3
with pytest.raises(ValueError, match="Cannot build lower case index: profile.CODE and profile.code collide"):
schema.find_field("profile.code", case_sensitive=False)


def test_schema_case_insensitive_lookup_of_list_element_struct_field() -> None:
"""The short name of a list element struct field maps to the same field ID, which is not a collision."""
schema = Schema(
NestedField(
field_id=1,
name="items",
field_type=ListType(
element_id=2,
element_type=StructType(NestedField(field_id=3, name="value", field_type=LongType(), required=False)),
element_required=False,
),
required=False,
),
)
assert schema.find_field("ITEMS.VALUE", case_sensitive=False).field_id == 3
assert schema.find_field("items.element.value", case_sensitive=False).field_id == 3


def test_schema_case_insensitive_lookup_rejects_collision_for_any_name() -> None:
schema = Schema(
NestedField(1, "ID", LongType()),
NestedField(2, "id", LongType()),
NestedField(3, "safe", LongType()),
)
# As in Java, the lower-case index cannot be built, so unambiguous and missing names fail too
for name in ("safe", "missing"):
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
schema.find_field(name, case_sensitive=False)
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
schema.select(name, case_sensitive=False)


def test_metadata_column_projection_propagates_case_collision() -> None:
schema = Schema(NestedField(1, "ID", LongType()), NestedField(2, "id", LongType()))
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
project_schema(schema, ("_row_id",), case_sensitive=False, format_version=3)


def test_identifier_field_lookup_propagates_case_collision(table_v2: Table) -> None:
schema = Schema(NestedField(1, "ID", LongType(), required=True), NestedField(2, "id", LongType(), required=True))
update = UpdateSchema(transaction=Transaction(table_v2), schema=schema, case_sensitive=False)
update.set_identifier_fields("ID")
with pytest.raises(ValueError, match="Cannot build lower case index: ID and id collide"):
update._apply()


def can_promote_decimal(file_type: DecimalType, read_type: DecimalType) -> bool:
return file_type.precision <= read_type.precision and file_type.scale == read_type.scale

Expand Down
Loading