Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 48 additions & 4 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,18 @@
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
from pyiceberg.encryption.manager import _new_data_file_key_metadata
from pyiceberg.exceptions import ResolveError
from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or
from pyiceberg.expressions import (
AlwaysTrue,
And,
BooleanExpression,
BoundIsNaN,
BoundIsNull,
BoundNotEqualTo,
BoundNotIn,
BoundTerm,
Not,
Or,
)
from pyiceberg.expressions.literals import Literal
from pyiceberg.expressions.visitors import (
BoundBooleanExpressionVisitor,
Expand Down Expand Up @@ -1745,6 +1756,27 @@ def _get_column_projection_values(
return projected_missing_fields


def _has_nan_sensitive_predicate(expr: BooleanExpression) -> bool:
"""Return whether a predicate in the expression can match NaN values that pyarrow may prune away.

Parquet statistics exclude NaN, so a row group whose non-NaN values are all equal reports min == max.
pyarrow then prunes it for predicates that the NaN values satisfy: IsNaN, NotEqualTo and NotIn on a
float or double column.
"""
if isinstance(expr, (And, Or)):
return _has_nan_sensitive_predicate(expr.left) or _has_nan_sensitive_predicate(expr.right)
return isinstance(expr, (BoundIsNaN, BoundNotEqualTo, BoundNotIn)) and isinstance(
expr.term.ref().field.field_type, (FloatType, DoubleType)
)


def _nan_safe_conjuncts(expr: BooleanExpression) -> BooleanExpression:
"""Return the top-level conjuncts that pyarrow can prune on without dropping NaN rows, joined with AND."""
if isinstance(expr, And):
return And(_nan_safe_conjuncts(expr.left), _nan_safe_conjuncts(expr.right))
return AlwaysTrue() if _has_nan_sensitive_predicate(expr) else expr


def _task_to_record_batches(
io: FileIO,
task: FileScanTask,
Expand Down Expand Up @@ -1785,6 +1817,8 @@ def _task_to_record_batches(
)

pyarrow_filter = None
# The part of the filter that the scanner may push down to Arrow
scanner_filter = None
if bound_row_filter is not AlwaysTrue():
translated_row_filter = translate_column_names(
bound_row_filter, file_schema, case_sensitive=case_sensitive, projected_field_values=projected_missing_fields
Expand All @@ -1793,21 +1827,31 @@ def _task_to_record_batches(
# matches the same predicate that decided whether its file was read
bound_file_filter = rewrite_not(bind(file_schema, translated_row_filter, case_sensitive=case_sensitive))
pyarrow_filter = expression_to_pyarrow(bound_file_filter, file_schema)
if not _has_nan_sensitive_predicate(bound_file_filter):
scanner_filter = pyarrow_filter
elif (nan_safe_filter := _nan_safe_conjuncts(bound_file_filter)) is not AlwaysTrue():
# Parquet statistics exclude NaN, so pyarrow can prune a row group that holds NaN rows matching
# a NaN-sensitive conjunct (https://github.com/apache/arrow/issues/51491). Only the other
# conjuncts are pushed down.
scanner_filter = expression_to_pyarrow(nan_safe_filter, file_schema)

file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)

# Row IDs are derived from the position of each row in the file, so rows must not be filtered before they are counted
row_id_index = next(
(index for index, field in enumerate(projected_schema.fields) if field.field_id == ROW_ID.field_id), None
)
filter_after_read = bool(positional_deletes) or row_id_index is not None
if positional_deletes or row_id_index is not None:
# Positional deletes and row IDs need every row, so nothing is pushed down
scanner_filter = None
# Set when the scanner does not apply the whole filter, so each batch is filtered after the read
filter_after_read = scanner_filter is not pyarrow_filter

fragment_scanner = ds.Scanner.from_fragment(
fragment=fragment,
schema=physical_schema,
# This will push down the query to Arrow.
# But in case there are positional deletes or row IDs, rows must be counted before filtering
filter=pyarrow_filter if not filter_after_read else None,
filter=scanner_filter,
columns=[col.name for col in file_project_schema.columns],
)

Expand Down
22 changes: 22 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1415,6 +1415,28 @@ def test_replace_files_in_nan_identity_partition(catalog: Catalog) -> None:
assert sorted(tbl.scan(row_filter="value is nan").to_arrow()["id"].to_pylist()) == expected_nan_ids, name


def test_scan_nan_row_with_equal_finite_statistics(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.scan_nan_equal_statistics_{catalog.name}"
iceberg_schema = Schema(
NestedField(1, "id", IntegerType(), required=False),
NestedField(2, "value", DoubleType(), required=False),
)
tbl = catalog.create_table(identifier=identifier, schema=iceberg_schema)
# Parquet statistics ignore NaN, so the single row group reports min == max == 5.0 for this file.
tbl.append(
pa.Table.from_pydict(
{"id": [1, 2, 3], "value": [float("nan"), 5.0, 5.0]},
schema=schema_to_pyarrow(iceberg_schema),
)
)

assert tbl.scan(row_filter="value is nan").to_arrow()["id"].to_pylist() == [1]
assert tbl.scan(row_filter="value != 5.0").to_arrow()["id"].to_pylist() == [1]
assert tbl.scan(row_filter="value not in (5.0, 6.0)").to_arrow()["id"].to_pylist() == [1]
assert tbl.scan(row_filter="value is not nan").to_arrow()["id"].to_pylist() == [2, 3]


def test_scan_integer_column_with_decimal_literal(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.scan_integer_decimal_literal_{catalog.name}"
Expand Down
63 changes: 62 additions & 1 deletion tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,13 @@
BoundStartsWith,
EqualTo,
GreaterThan,
IsNaN,
IsNull,
LessThan,
Not,
NotEqualTo,
NotIn,
NotNaN,
Or,
)
from pyiceberg.expressions.literals import literal
Expand Down Expand Up @@ -4443,6 +4448,62 @@ def test_task_to_record_batches_filter_after_positional_deletes_empty_result(tmp
assert result_batches == []


def test_task_to_record_batches_keeps_nan_rows_for_nan_sensitive_filters(tmpdir: str) -> None:
"""Regression test: pyarrow must not prune a row group that holds NaN rows matching the filter.

Parquet statistics exclude NaN, so the row group [NaN, 5.0, 5.0] reports min == max == 5.0.
pyarrow prunes such a row group for IsNaN, NotEqualTo and NotIn, which drops the NaN row.
NotNaN is a control.
"""
from pyiceberg.expressions.visitors import bind

for arrow_type, iceberg_type in [(pa.float32(), FloatType()), (pa.float64(), DoubleType())]:
arrow_schema = pa.schema(
(
pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}),
pa.field("x", arrow_type, nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "2"}),
)
)
arrow_table = pa.table(
{"id": pa.array([0, 1, 2], type=pa.int32()), "x": pa.array([float("nan"), 5.0, 5.0], type=arrow_type)},
schema=arrow_schema,
)
data_file = _write_table_to_data_file(
f"{tmpdir}/nan_row_group_equal_bounds_{iceberg_type}.parquet", arrow_schema, arrow_table
)

table_schema = Schema(
NestedField(1, "id", IntegerType(), required=False),
NestedField(2, "x", iceberg_type, required=False),
)
projected_schema = Schema(NestedField(1, "id", IntegerType(), required=False))

cases: list[tuple[BooleanExpression, list[int]]] = [
(IsNaN("x"), [0]),
(NotEqualTo("x", 5.0), [0]),
(NotIn("x", {5.0, 6.0}), [0]),
(And(IsNaN("x"), EqualTo("id", 0)), [0]),
(Or(IsNaN("x"), EqualTo("id", 2)), [0, 2]),
(NotNaN("x"), [1, 2]),
]
for row_filter, expected_ids in cases:
# The filter column is not selected, so the filter must still see it after the read.
result_batches = list(
_task_to_record_batches(
PyArrowFileIO(),
FileScanTask(data_file),
bound_row_filter=bind(table_schema, row_filter, case_sensitive=True),
projected_schema=projected_schema,
table_schema=table_schema,
projected_field_ids={1, 2},
positional_deletes=None,
case_sensitive=True,
)
)
actual_ids = [value for batch in result_batches for value in batch.column(0).to_pylist()]
assert actual_ids == expected_ids, f"{iceberg_type}: {row_filter}"


@pytest.mark.parametrize(
"stored_row_ids, first_row_id, deleted_positions, row_filter, expected",
[
Expand Down Expand Up @@ -4757,7 +4818,7 @@ def test_orc_row_filtering_predicate_pushdown(tmp_path: Path) -> None:
import pyarrow as pa
import pyarrow.orc as orc

from pyiceberg.expressions import And, EqualTo, GreaterThan, In, LessThan, Or
from pyiceberg.expressions import And, EqualTo, GreaterThan, In, Or
from pyiceberg.io.pyarrow import ArrowScan, PyArrowFileIO
from pyiceberg.manifest import DataFileContent, FileFormat
from pyiceberg.partitioning import PartitionSpec
Expand Down
Loading