From 82d50d5e6149ad701925b401724f4e9c2d57a203 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 01:27:48 +0530 Subject: [PATCH] Fail a retried upsert instead of inserting a concurrent key A commit retry replays the append that was planned before the concurrent commit, and the fast append producer skips concurrency validation, so a key another writer inserted after the upsert read ends up inserted twice. Record the filter for the rows an upsert inserts on the append producer and validate the files added since the starting snapshot against it when the commit is rebuilt for a retry. The upsert now fails with a CommitFailedException instead of leaving a duplicate key. --- pyiceberg/table/__init__.py | 13 +++++++++++- pyiceberg/table/update/snapshot.py | 34 +++++++++++++++++++++++++++++- tests/table/test_upsert.py | 29 +++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..3ebbbc1c4f 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -512,6 +512,8 @@ def append( df: pa.Table | pa.RecordBatchReader, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, + _upsert_insert_filter: BooleanExpression | None = None, + _case_sensitive: bool = True, ) -> None: """ Shorthand API for appending PyArrow data to a table transaction. @@ -576,6 +578,9 @@ def append( ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: + if _upsert_insert_filter is not None: + append_files.set_upsert_insert_filter(_upsert_insert_filter, _case_sensitive) + # For pa.Table we can short-circuit empty inputs cheaply. For a # RecordBatchReader the stream is consumed lazily by # _dataframe_to_data_files and an empty reader simply yields zero @@ -1014,7 +1019,13 @@ def upsert( if when_not_matched_insert_all: insert_row_cnt = len(rows_to_insert) if rows_to_insert: - self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties) + self.append( + rows_to_insert, + branch=branch, + snapshot_properties=snapshot_properties, + _upsert_insert_filter=upsert_util.create_match_filter(rows_to_insert, join_cols), + _case_sensitive=case_sensitive, + ) return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 0c9831450e..69f6bc71fc 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -29,6 +29,7 @@ from pyiceberg.avro.codecs import AvroCompressionCodec from pyiceberg.exceptions import ( + CommitFailedException, NoSuchSnapshotRefError, NotAncestorError, SnapshotRefTypeError, @@ -690,8 +691,39 @@ def _refresh_for_retry(self) -> None: class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]): + _upsert_insert_filter: BooleanExpression | None = None + + def set_upsert_insert_filter(self, insert_filter: BooleanExpression, case_sensitive: bool) -> None: + self._upsert_insert_filter = insert_filter + self._case_sensitive = case_sensitive + def _validate_concurrency(self) -> None: - """Skip validation; appends do not conflict with other operations.""" + """Skip validation for plain appends. + + An upsert registers the filter for the rows it inserts, so a retry fails instead of + inserting a key that another writer added after the upsert was planned. + """ + if self._upsert_insert_filter is None or self._commit_window is None or self._commit_window.is_empty(): + return + + from pyiceberg.table.update.validate import _validate_added_data_files + + catalog_head = self._commit_window.head + if catalog_head is None: + return + + try: + _validate_added_data_files( + self._transaction._table, + catalog_head, + self._upsert_insert_filter, + self._commit_window.base, + self._case_sensitive, + ) + except ValidationException as e: + raise CommitFailedException( + "Cannot retry the upsert because a concurrent commit added rows matching the rows to insert" + ) from e def _existing_manifests(self) -> list[ManifestFile]: """To determine if there are any existing manifest files. diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..596dd41d6d 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -23,7 +23,7 @@ from pyarrow import Table as pa_table from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.exceptions import CommitFailedException, NoSuchTableError from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference from pyiceberg.expressions.literals import LongLiteral from pyiceberg.io.pyarrow import schema_to_pyarrow @@ -33,7 +33,7 @@ from pyiceberg.table.snapshots import Operation from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update from pyiceberg.transforms import DayTransform -from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType +from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType from tests.catalog.test_base import InMemoryCatalog @@ -427,6 +427,31 @@ def test_upsert_into_empty_table(catalog: Catalog) -> None: assert upd.rows_inserted == 4 +def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None: + identifier = "default.test_upsert_retry_with_a_concurrently_inserted_key" + _drop_table(catalog, identifier) + + schema = Schema( + NestedField(1, "k", LongType(), required=True), + NestedField(2, "v", StringType(), required=False), + identifier_field_ids=[1], + ) + arrow_schema = schema_to_pyarrow(schema) + catalog.create_table(identifier, schema=schema).append(pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema)) + + writer_a = catalog.load_table(identifier) + writer_b = catalog.load_table(identifier) + writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema)) + + with pytest.raises(CommitFailedException): + writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema)) + + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [ + {"k": 1, "v": "a"}, + {"k": 2, "v": "b"}, + ] + + def test_create_match_filter_single_condition() -> None: """ Test create_match_filter with a composite key where the source yields exactly one unique key.