From 9b17b2ca50ec58a9cbebe033832c0a2b7534d033 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Wed, 11 Nov 2026 17:20:16 +0000 Subject: [PATCH 1/3] Fix scans dropping NaN rows for IsNaN, NotEqualTo, NotIn and NOT Parquet min/max statistics exclude NaN, so pyarrow could prune a row group whose non-NaN values are all equal and drop the NaN rows that these predicates match on float and double columns. Push only the conjuncts that cannot hide NaN rows to the scanner, and filter each batch with the full expression when a NaN-sensitive predicate is present. Co-authored-by: Isaac --- pyiceberg/io/pyarrow.py | 76 ++++++++++++++++++++----- tests/catalog/test_catalog_behaviors.py | 22 +++++++ tests/io/test_pyarrow.py | 60 +++++++++++++++++++ 3 files changed, 145 insertions(+), 13 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index f62e53a911..b0379bcef4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -70,7 +70,19 @@ from pyiceberg.conversions import to_bytes from pyiceberg.encryption.key_metadata import StandardKeyMetadata from pyiceberg.exceptions import ResolveError -from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or +from pyiceberg.expressions import ( + AlwaysTrue, + And, + BooleanExpression, + BoundIsNaN, + BoundIsNull, + BoundNotEqualTo, + BoundNotIn, + BoundPredicate, + BoundTerm, + Not, + Or, +) from pyiceberg.expressions.literals import Literal from pyiceberg.expressions.visitors import ( BoundBooleanExpressionVisitor, @@ -1721,6 +1733,29 @@ def _get_column_projection_values( return projected_missing_fields +def _has_nan_sensitive_predicate(expr: BooleanExpression, negated: bool = False) -> bool: + """Return whether a predicate in the expression can match NaN values that pyarrow may prune away. + + Parquet statistics exclude NaN, so a row group whose non-NaN values are all equal reports min == max. + pyarrow then prunes it for predicates that the NaN values satisfy: IsNaN, NotEqualTo and NotIn on a + float or double column, and any predicate on such a column under a NOT. + """ + if isinstance(expr, (And, Or)): + return _has_nan_sensitive_predicate(expr.left, negated) or _has_nan_sensitive_predicate(expr.right, negated) + if isinstance(expr, Not): + return _has_nan_sensitive_predicate(expr.child, not negated) + if isinstance(expr, BoundPredicate) and isinstance(expr.term.ref().field.field_type, (FloatType, DoubleType)): + return negated or isinstance(expr, (BoundIsNaN, BoundNotEqualTo, BoundNotIn)) + return False + + +def _nan_safe_conjuncts(expr: BooleanExpression) -> BooleanExpression: + """Return the top-level conjuncts that pyarrow can prune on without dropping NaN rows, joined with AND.""" + if isinstance(expr, And): + return And(_nan_safe_conjuncts(expr.left), _nan_safe_conjuncts(expr.right)) + return AlwaysTrue() if _has_nan_sensitive_predicate(expr) else expr + + def _task_to_record_batches( io: FileIO, task: FileScanTask, @@ -1761,12 +1796,27 @@ def _task_to_record_batches( ) pyarrow_filter = None + scanner_filter = None + # Set when the scanner does not apply the whole filter, so each batch is filtered after the read + filter_batches = False if bound_row_filter is not AlwaysTrue(): translated_row_filter = translate_column_names( bound_row_filter, file_schema, case_sensitive=case_sensitive, projected_field_values=projected_missing_fields ) bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive) pyarrow_filter = expression_to_pyarrow(bound_file_filter, file_schema) + if positional_deletes: + # Positional deletes must be applied before any row is filtered + filter_batches = True + elif _has_nan_sensitive_predicate(bound_file_filter): + # Parquet statistics exclude NaN, so pyarrow can prune a row group that holds NaN rows matching + # a NaN-sensitive conjunct. Only the other conjuncts are pushed down. + filter_batches = True + nan_safe_filter = _nan_safe_conjuncts(bound_file_filter) + if nan_safe_filter is not AlwaysTrue(): + scanner_filter = expression_to_pyarrow(nan_safe_filter, file_schema) + else: + scanner_filter = pyarrow_filter file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) @@ -1774,8 +1824,7 @@ def _task_to_record_batches( fragment=fragment, schema=physical_schema, # This will push down the query to Arrow. - # But in case there are positional deletes, we have to apply them first - filter=pyarrow_filter if not positional_deletes else None, + filter=scanner_filter, columns=[col.name for col in file_project_schema.columns], ) @@ -1790,16 +1839,17 @@ def _task_to_record_batches( # Create the mask of indices that we're interested in indices = _combine_positional_deletes(positional_deletes, current_index, current_index + len(batch)) current_batch = current_batch.take(indices) - if pyarrow_filter is not None: - # Temporary fix until PyArrow 21 is the minimum supported version - # (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises - # IndexError on PyArrow <21 when the result is empty; Table.filter does not. - table = pa.Table.from_batches([current_batch]) - table = table.filter(pyarrow_filter) - if table.num_rows == 0: - current_batch = current_batch.slice(0, 0) - else: - current_batch = table.combine_chunks().to_batches()[0] + + if filter_batches and pyarrow_filter is not None: + # Temporary fix until PyArrow 21 is the minimum supported version + # (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises + # IndexError on PyArrow <21 when the result is empty; Table.filter does not. + table = pa.Table.from_batches([current_batch]) + table = table.filter(pyarrow_filter) + if table.num_rows == 0: + current_batch = current_batch.slice(0, 0) + else: + current_batch = table.combine_chunks().to_batches()[0] # skip empty batches if current_batch.num_rows == 0: diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index 1ed9ea4096..328fd7689b 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1376,6 +1376,28 @@ def test_append_nan_to_identity_partitioned_table(catalog: Catalog) -> None: assert sorted(tbl.scan(row_filter="value is nan").to_arrow()["id"].to_pylist()) == [2, 4] +def test_scan_nan_row_with_equal_finite_statistics(catalog: Catalog) -> None: + catalog.create_namespace("default") + identifier = f"default.scan_nan_equal_statistics_{catalog.name}" + iceberg_schema = Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "value", DoubleType(), required=False), + ) + tbl = catalog.create_table(identifier=identifier, schema=iceberg_schema) + # Parquet statistics ignore NaN, so the single row group reports min == max == 5.0 for this file. + tbl.append( + pa.Table.from_pydict( + {"id": [1, 2, 3], "value": [float("nan"), 5.0, 5.0]}, + schema=schema_to_pyarrow(iceberg_schema), + ) + ) + + assert tbl.scan(row_filter="value is nan").to_arrow()["id"].to_pylist() == [1] + assert tbl.scan(row_filter="value != 5.0").to_arrow()["id"].to_pylist() == [1] + assert tbl.scan(row_filter="value not in (5.0, 6.0)").to_arrow()["id"].to_pylist() == [1] + assert tbl.scan(row_filter="value is not nan").to_arrow()["id"].to_pylist() == [2, 3] + + def test_scan_integer_column_with_decimal_literal(catalog: Catalog) -> None: catalog.create_namespace("default") identifier = f"default.scan_integer_decimal_literal_{catalog.name}" diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 419d130965..e292139cc4 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -68,8 +68,13 @@ BoundStartsWith, EqualTo, GreaterThan, + IsNaN, IsNull, + LessThan, Not, + NotEqualTo, + NotIn, + NotNaN, Or, ) from pyiceberg.expressions.literals import literal @@ -4198,6 +4203,61 @@ def test_task_to_record_batches_filter_after_positional_deletes_empty_result(tmp assert result_batches == [] +def test_task_to_record_batches_keeps_nan_rows_for_nan_sensitive_filters(tmpdir: str) -> None: + """Regression test: pyarrow must not prune a row group that holds NaN rows matching the filter. + + Parquet statistics exclude NaN, so the row group [NaN, 5.0, 5.0] reports min == max == 5.0. + pyarrow prunes such a row group for IsNaN, NotEqualTo, NotIn and a NOT over a float comparison, + which drops the NaN row. NotNaN and a NOT over an integer column are controls. + """ + from pyiceberg.expressions.visitors import bind + + arrow_schema = pa.schema( + ( + pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}), + pa.field("x", pa.float64(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "2"}), + ) + ) + arrow_table = pa.table( + {"id": pa.array([0, 1, 2], type=pa.int32()), "x": pa.array([float("nan"), 5.0, 5.0], type=pa.float64())}, + schema=arrow_schema, + ) + data_file = _write_table_to_data_file(f"{tmpdir}/nan_row_group_equal_bounds.parquet", arrow_schema, arrow_table) + + table_schema = Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "x", DoubleType(), required=False), + ) + projected_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) + + cases: list[tuple[BooleanExpression, list[int]]] = [ + (IsNaN("x"), [0]), + (NotEqualTo("x", 5.0), [0]), + (NotIn("x", {5.0, 6.0}), [0]), + (And(IsNaN("x"), EqualTo("id", 0)), [0]), + (Or(IsNaN("x"), EqualTo("id", 2)), [0, 2]), + (Not(LessThan("x", 6.0)), [0]), + (Not(EqualTo("id", 1)), [0, 2]), + (NotNaN("x"), [1, 2]), + ] + for row_filter, expected_ids in cases: + # The filter column is not selected, so the filter must still see it after the read. + result_batches = list( + _task_to_record_batches( + PyArrowFileIO(), + FileScanTask(data_file), + bound_row_filter=bind(table_schema, row_filter, case_sensitive=True), + projected_schema=projected_schema, + table_schema=table_schema, + projected_field_ids={1, 2}, + positional_deletes=None, + case_sensitive=True, + ) + ) + actual_ids = [value for batch in result_batches for value in batch.column(0).to_pylist()] + assert actual_ids == expected_ids, str(row_filter) + + def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None: """The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" data_file = DataFile.from_args( From 191387d0c070c63241a4dd31ac4010d42367a54f Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Wed, 11 Nov 2026 20:30:45 +0000 Subject: [PATCH 2/3] Drop NOT handling and cover float32 in NaN-sensitive scans NOT is left to a follow-up that applies rewrite_not before the Arrow conversion, as the metrics evaluators do. Run the regression cases on float and double columns, and link the upstream Arrow issue. Co-authored-by: Isaac --- pyiceberg/io/pyarrow.py | 16 ++++---- tests/io/test_pyarrow.py | 89 ++++++++++++++++++++-------------------- 2 files changed, 52 insertions(+), 53 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index b0379bcef4..f07cdd3108 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -78,7 +78,6 @@ BoundIsNull, BoundNotEqualTo, BoundNotIn, - BoundPredicate, BoundTerm, Not, Or, @@ -1733,19 +1732,17 @@ def _get_column_projection_values( return projected_missing_fields -def _has_nan_sensitive_predicate(expr: BooleanExpression, negated: bool = False) -> bool: +def _has_nan_sensitive_predicate(expr: BooleanExpression) -> bool: """Return whether a predicate in the expression can match NaN values that pyarrow may prune away. Parquet statistics exclude NaN, so a row group whose non-NaN values are all equal reports min == max. pyarrow then prunes it for predicates that the NaN values satisfy: IsNaN, NotEqualTo and NotIn on a - float or double column, and any predicate on such a column under a NOT. + float or double column. """ if isinstance(expr, (And, Or)): - return _has_nan_sensitive_predicate(expr.left, negated) or _has_nan_sensitive_predicate(expr.right, negated) - if isinstance(expr, Not): - return _has_nan_sensitive_predicate(expr.child, not negated) - if isinstance(expr, BoundPredicate) and isinstance(expr.term.ref().field.field_type, (FloatType, DoubleType)): - return negated or isinstance(expr, (BoundIsNaN, BoundNotEqualTo, BoundNotIn)) + return _has_nan_sensitive_predicate(expr.left) or _has_nan_sensitive_predicate(expr.right) + if isinstance(expr, (BoundIsNaN, BoundNotEqualTo, BoundNotIn)): + return isinstance(expr.term.ref().field.field_type, (FloatType, DoubleType)) return False @@ -1810,7 +1807,8 @@ def _task_to_record_batches( filter_batches = True elif _has_nan_sensitive_predicate(bound_file_filter): # Parquet statistics exclude NaN, so pyarrow can prune a row group that holds NaN rows matching - # a NaN-sensitive conjunct. Only the other conjuncts are pushed down. + # a NaN-sensitive conjunct (https://github.com/apache/arrow/issues/51491). Only the other + # conjuncts are pushed down. filter_batches = True nan_safe_filter = _nan_safe_conjuncts(bound_file_filter) if nan_safe_filter is not AlwaysTrue(): diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index e292139cc4..b06bfca13e 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -4207,55 +4207,56 @@ def test_task_to_record_batches_keeps_nan_rows_for_nan_sensitive_filters(tmpdir: """Regression test: pyarrow must not prune a row group that holds NaN rows matching the filter. Parquet statistics exclude NaN, so the row group [NaN, 5.0, 5.0] reports min == max == 5.0. - pyarrow prunes such a row group for IsNaN, NotEqualTo, NotIn and a NOT over a float comparison, - which drops the NaN row. NotNaN and a NOT over an integer column are controls. + pyarrow prunes such a row group for IsNaN, NotEqualTo and NotIn, which drops the NaN row. + NotNaN is a control. """ from pyiceberg.expressions.visitors import bind - arrow_schema = pa.schema( - ( - pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}), - pa.field("x", pa.float64(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "2"}), + for arrow_type, iceberg_type in [(pa.float32(), FloatType()), (pa.float64(), DoubleType())]: + arrow_schema = pa.schema( + ( + pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}), + pa.field("x", arrow_type, nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "2"}), + ) + ) + arrow_table = pa.table( + {"id": pa.array([0, 1, 2], type=pa.int32()), "x": pa.array([float("nan"), 5.0, 5.0], type=arrow_type)}, + schema=arrow_schema, + ) + data_file = _write_table_to_data_file( + f"{tmpdir}/nan_row_group_equal_bounds_{iceberg_type}.parquet", arrow_schema, arrow_table ) - ) - arrow_table = pa.table( - {"id": pa.array([0, 1, 2], type=pa.int32()), "x": pa.array([float("nan"), 5.0, 5.0], type=pa.float64())}, - schema=arrow_schema, - ) - data_file = _write_table_to_data_file(f"{tmpdir}/nan_row_group_equal_bounds.parquet", arrow_schema, arrow_table) - table_schema = Schema( - NestedField(1, "id", IntegerType(), required=False), - NestedField(2, "x", DoubleType(), required=False), - ) - projected_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) - - cases: list[tuple[BooleanExpression, list[int]]] = [ - (IsNaN("x"), [0]), - (NotEqualTo("x", 5.0), [0]), - (NotIn("x", {5.0, 6.0}), [0]), - (And(IsNaN("x"), EqualTo("id", 0)), [0]), - (Or(IsNaN("x"), EqualTo("id", 2)), [0, 2]), - (Not(LessThan("x", 6.0)), [0]), - (Not(EqualTo("id", 1)), [0, 2]), - (NotNaN("x"), [1, 2]), - ] - for row_filter, expected_ids in cases: - # The filter column is not selected, so the filter must still see it after the read. - result_batches = list( - _task_to_record_batches( - PyArrowFileIO(), - FileScanTask(data_file), - bound_row_filter=bind(table_schema, row_filter, case_sensitive=True), - projected_schema=projected_schema, - table_schema=table_schema, - projected_field_ids={1, 2}, - positional_deletes=None, - case_sensitive=True, - ) + table_schema = Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "x", iceberg_type, required=False), ) - actual_ids = [value for batch in result_batches for value in batch.column(0).to_pylist()] - assert actual_ids == expected_ids, str(row_filter) + projected_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) + + cases: list[tuple[BooleanExpression, list[int]]] = [ + (IsNaN("x"), [0]), + (NotEqualTo("x", 5.0), [0]), + (NotIn("x", {5.0, 6.0}), [0]), + (And(IsNaN("x"), EqualTo("id", 0)), [0]), + (Or(IsNaN("x"), EqualTo("id", 2)), [0, 2]), + (NotNaN("x"), [1, 2]), + ] + for row_filter, expected_ids in cases: + # The filter column is not selected, so the filter must still see it after the read. + result_batches = list( + _task_to_record_batches( + PyArrowFileIO(), + FileScanTask(data_file), + bound_row_filter=bind(table_schema, row_filter, case_sensitive=True), + projected_schema=projected_schema, + table_schema=table_schema, + projected_field_ids={1, 2}, + positional_deletes=None, + case_sensitive=True, + ) + ) + actual_ids = [value for batch in result_batches for value in batch.column(0).to_pylist()] + assert actual_ids == expected_ids, f"{iceberg_type}: {row_filter}" def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None: @@ -4514,7 +4515,7 @@ def test_orc_row_filtering_predicate_pushdown(tmp_path: Path) -> None: import pyarrow as pa import pyarrow.orc as orc - from pyiceberg.expressions import And, EqualTo, GreaterThan, In, LessThan, Or + from pyiceberg.expressions import And, EqualTo, GreaterThan, In, Or from pyiceberg.io.pyarrow import ArrowScan, PyArrowFileIO from pyiceberg.manifest import DataFileContent, FileFormat from pyiceberg.partitioning import PartitionSpec From 241c84ed2f3c089994d722da84923befa139a1ff Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Thu, 12 Nov 2026 01:27:45 +0000 Subject: [PATCH 3/3] Fix unreachable-statement lint in _has_nan_sensitive_predicate mypy (warn_unreachable) flagged the final return in the pre-commit environment; fold the leaf check into one expression. Co-authored-by: Isaac --- pyiceberg/io/pyarrow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 90ae0526ab..8d3968a03a 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1754,9 +1754,9 @@ def _has_nan_sensitive_predicate(expr: BooleanExpression) -> bool: """ if isinstance(expr, (And, Or)): return _has_nan_sensitive_predicate(expr.left) or _has_nan_sensitive_predicate(expr.right) - if isinstance(expr, (BoundIsNaN, BoundNotEqualTo, BoundNotIn)): - return isinstance(expr.term.ref().field.field_type, (FloatType, DoubleType)) - return False + return isinstance(expr, (BoundIsNaN, BoundNotEqualTo, BoundNotIn)) and isinstance( + expr.term.ref().field.field_type, (FloatType, DoubleType) + ) def _nan_safe_conjuncts(expr: BooleanExpression) -> BooleanExpression: