From 1dfbb8e21db33cabae48240cc43daee73784aa3e Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 15 Nov 2026 18:37:29 +0100 Subject: [PATCH] fix(pyarrow): omit incomplete parquet null counts --- pyiceberg/io/pyarrow.py | 6 ++++ tests/io/test_pyarrow_stats.py | 65 +++++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 6192f13b14..53db9cd6b7 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2749,6 +2749,7 @@ def data_file_statistics_from_parquet_metadata( col_aggs = {} invalidate_col: set[int] = set() + invalidate_null_counts: set[int] = set() without_bounds: set[int] = set() for r in range(parquet_metadata.num_row_groups): # References: @@ -2785,6 +2786,8 @@ def data_file_statistics_from_parquet_metadata( if statistics.has_null_count: null_value_counts[field_id] = null_value_counts.get(field_id, 0) + statistics.null_count + else: + invalidate_null_counts.add(field_id) if stats_col.mode == MetricsMode(MetricModeTypes.COUNTS): continue @@ -2831,6 +2834,9 @@ def data_file_statistics_from_parquet_metadata( col_aggs.pop(field_id, None) null_value_counts.pop(field_id, None) + for field_id in invalidate_null_counts: + null_value_counts.pop(field_id, None) + for field_id in without_bounds: col_aggs.pop(field_id, None) diff --git a/tests/io/test_pyarrow_stats.py b/tests/io/test_pyarrow_stats.py index ea41e3365c..1c5e802a26 100644 --- a/tests/io/test_pyarrow_stats.py +++ b/tests/io/test_pyarrow_stats.py @@ -28,6 +28,7 @@ timezone, ) from decimal import Decimal +from types import SimpleNamespace from typing import ( Any, ) @@ -43,6 +44,8 @@ STRUCT_INT32, STRUCT_INT64, ) +from pyiceberg.expressions import IsNull +from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator from pyiceberg.io.pyarrow import ( MetricModeTypes, MetricsMode, @@ -53,7 +56,7 @@ parquet_path_to_id_mapping, schema_to_pyarrow, ) -from pyiceberg.manifest import DataFile +from pyiceberg.manifest import DataFile, FileFormat from pyiceberg.schema import Schema, pre_order_visit from pyiceberg.table.metadata import ( TableMetadata, @@ -61,10 +64,12 @@ TableMetadataV1, TableMetadataV2, ) +from pyiceberg.typedef import Record from pyiceberg.types import ( BooleanType, FloatType, IntegerType, + NestedField, StringType, ) from pyiceberg.utils.datetime import date_to_days, datetime_to_micros, time_to_micros @@ -262,6 +267,64 @@ def test_null_and_nan_counts() -> None: # assert datafile.nan_value_counts[3] == 0 +@pytest.mark.parametrize("unknown_row_group", [0, 1]) +def test_unknown_row_group_null_count_is_omitted(unknown_row_group: int) -> None: + schema = Schema(NestedField(1, "value", StringType())) + arrow_schema = schema_to_pyarrow(schema) + table = pa.table({"value": ["present", None]}, schema=arrow_schema) + metadata_collector: list[Any] = [] + + with pa.BufferOutputStream() as output: + with pq.ParquetWriter(output, table.schema, metadata_collector=metadata_collector) as writer: + writer.write_table(table, row_group_size=1) + + metadata = metadata_collector[0] + assert metadata.num_row_groups == 2 + assert metadata.row_group(0).column(0).statistics.null_count == 0 + assert metadata.row_group(1).column(0).statistics.null_count == 1 + assert table.column("value").null_count == 1 + + row_groups: list[Any] = [] + for index in range(metadata.num_row_groups): + column = metadata.row_group(index).column(0) + column_metadata = SimpleNamespace( + data_page_offset=column.data_page_offset, + dictionary_page_offset=column.dictionary_page_offset, + has_dictionary_page=column.has_dictionary_page, + path_in_schema=column.path_in_schema, + total_compressed_size=column.total_compressed_size, + num_values=column.num_values, + is_stats_set=column.is_stats_set, + statistics=SimpleNamespace( + has_null_count=index != unknown_row_group, + null_count=column.statistics.null_count, + ), + ) + row_groups.append(SimpleNamespace(column=lambda _index, column=column_metadata: column)) + metadata_view = SimpleNamespace( + num_row_groups=metadata.num_row_groups, + num_columns=metadata.num_columns, + num_rows=metadata.num_rows, + row_group=lambda index: row_groups[index], + ) + + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata_view, + stats_columns=compute_statistics_plan(schema, {"write.metadata.metrics.default": "counts"}), + parquet_column_mapping=parquet_path_to_id_mapping(schema), + ) + assert statistics.null_value_counts == {} + + data_file = DataFile.from_args( + file_path="file.parquet", + file_format=FileFormat.PARQUET, + partition=Record(), + file_size_in_bytes=metadata.serialized_size, + **statistics.to_serialized_dict(), + ) + assert _InclusiveMetricsEvaluator(schema, IsNull("value")).eval(data_file) + + def test_bounds() -> None: metadata, table_metadata = construct_test_table()