diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..5961da883c 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1952,11 +1952,17 @@ def __init__( @property def _projected_field_ids(self) -> set[int]: """Set of field IDs that should be projected from the data files.""" + schema = self._table_metadata.schema() + filter_field_ids = extract_field_ids(self._bound_row_filter) + # A list or map cannot be projected by its own ID, so a filter on one reads its nested fields instead + nested_field_ids = prune_columns( + schema, {id for id in filter_field_ids if isinstance(schema.find_type(id), (MapType, ListType))} + ).field_ids return { id for id in self._projected_schema.field_ids if not isinstance(self._projected_schema.find_type(id), (MapType, ListType)) - }.union(extract_field_ids(self._bound_row_filter)) + }.union(id for id in filter_field_ids | nested_field_ids if not isinstance(schema.find_type(id), (MapType, ListType))) def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table: """Scan the Iceberg table and return a pa.Table. diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..f00646ed6c 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -72,6 +72,7 @@ GreaterThan, IsNull, Not, + NotNull, Or, ) from pyiceberg.expressions.literals import literal @@ -1953,6 +1954,46 @@ def test_projection_filter_on_unprojected_field(schema_int_str: Schema, file_int assert repr(result_table.schema) == "id: int32 not null" +def test_projection_null_filter_on_list_and_map(tmpdir: str) -> None: + schema = Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "tags", ListType(21, StringType(), element_required=False), required=False), + NestedField( + 3, + "props", + MapType(key_id=31, key_type=StringType(), value_id=32, value_type=IntegerType(), value_required=False), + required=False, + ), + NestedField( + 4, + "points", + ListType(41, StructType(NestedField(42, "x", IntegerType(), required=False)), element_required=False), + required=False, + ), + ) + pyarrow_schema = schema_to_pyarrow(schema, metadata={ICEBERG_SCHEMA: bytes(schema.model_dump_json(), UTF8)}) + file = _write_table_to_file( + f"{tmpdir}/null_list_map.parquet", + pyarrow_schema, + pa.Table.from_pylist( + [ + {"id": 1, "tags": ["a"], "props": [("k", 1)], "points": [{"x": 1}]}, + {"id": 2, "tags": [], "props": [], "points": []}, + {"id": 3, "tags": None, "props": None, "points": None}, + ], + schema=pyarrow_schema, + ), + ) + schema_id = Schema(NestedField(1, "id", IntegerType(), required=False)) + + for column in ("tags", "props", "points"): + for expr, expected in ((IsNull(column), [3]), (NotNull(column), [1, 2])): + assert project(schema, [file], expr)["id"].to_pylist() == expected + assert project(schema_id, [file], expr, schema).to_pydict() == {"id": expected} + + assert project(schema, [file], IsNull("tags")).to_pylist() == [{"id": 3, "tags": None, "props": None, "points": None}] + + def test_projection_filter_on_unknown_field(schema_int_str: Schema, file_int_str: str) -> None: schema = Schema(NestedField(1, "id", IntegerType()))