diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..3ebbbc1c4f 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -512,6 +512,8 @@ def append( df: pa.Table | pa.RecordBatchReader, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, + _upsert_insert_filter: BooleanExpression | None = None, + _case_sensitive: bool = True, ) -> None: """ Shorthand API for appending PyArrow data to a table transaction. @@ -576,6 +578,9 @@ def append( ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: + if _upsert_insert_filter is not None: + append_files.set_upsert_insert_filter(_upsert_insert_filter, _case_sensitive) + # For pa.Table we can short-circuit empty inputs cheaply. For a # RecordBatchReader the stream is consumed lazily by # _dataframe_to_data_files and an empty reader simply yields zero @@ -1014,7 +1019,13 @@ def upsert( if when_not_matched_insert_all: insert_row_cnt = len(rows_to_insert) if rows_to_insert: - self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties) + self.append( + rows_to_insert, + branch=branch, + snapshot_properties=snapshot_properties, + _upsert_insert_filter=upsert_util.create_match_filter(rows_to_insert, join_cols), + _case_sensitive=case_sensitive, + ) return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 0c9831450e..69f6bc71fc 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -29,6 +29,7 @@ from pyiceberg.avro.codecs import AvroCompressionCodec from pyiceberg.exceptions import ( + CommitFailedException, NoSuchSnapshotRefError, NotAncestorError, SnapshotRefTypeError, @@ -690,8 +691,39 @@ def _refresh_for_retry(self) -> None: class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]): + _upsert_insert_filter: BooleanExpression | None = None + + def set_upsert_insert_filter(self, insert_filter: BooleanExpression, case_sensitive: bool) -> None: + self._upsert_insert_filter = insert_filter + self._case_sensitive = case_sensitive + def _validate_concurrency(self) -> None: - """Skip validation; appends do not conflict with other operations.""" + """Skip validation for plain appends. + + An upsert registers the filter for the rows it inserts, so a retry fails instead of + inserting a key that another writer added after the upsert was planned. + """ + if self._upsert_insert_filter is None or self._commit_window is None or self._commit_window.is_empty(): + return + + from pyiceberg.table.update.validate import _validate_added_data_files + + catalog_head = self._commit_window.head + if catalog_head is None: + return + + try: + _validate_added_data_files( + self._transaction._table, + catalog_head, + self._upsert_insert_filter, + self._commit_window.base, + self._case_sensitive, + ) + except ValidationException as e: + raise CommitFailedException( + "Cannot retry the upsert because a concurrent commit added rows matching the rows to insert" + ) from e def _existing_manifests(self) -> list[ManifestFile]: """To determine if there are any existing manifest files. diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..596dd41d6d 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -23,7 +23,7 @@ from pyarrow import Table as pa_table from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.exceptions import CommitFailedException, NoSuchTableError from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference from pyiceberg.expressions.literals import LongLiteral from pyiceberg.io.pyarrow import schema_to_pyarrow @@ -33,7 +33,7 @@ from pyiceberg.table.snapshots import Operation from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update from pyiceberg.transforms import DayTransform -from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType +from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType from tests.catalog.test_base import InMemoryCatalog @@ -427,6 +427,31 @@ def test_upsert_into_empty_table(catalog: Catalog) -> None: assert upd.rows_inserted == 4 +def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None: + identifier = "default.test_upsert_retry_with_a_concurrently_inserted_key" + _drop_table(catalog, identifier) + + schema = Schema( + NestedField(1, "k", LongType(), required=True), + NestedField(2, "v", StringType(), required=False), + identifier_field_ids=[1], + ) + arrow_schema = schema_to_pyarrow(schema) + catalog.create_table(identifier, schema=schema).append(pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema)) + + writer_a = catalog.load_table(identifier) + writer_b = catalog.load_table(identifier) + writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema)) + + with pytest.raises(CommitFailedException): + writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema)) + + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [ + {"k": 1, "v": "a"}, + {"k": 2, "v": "b"}, + ] + + def test_create_match_filter_single_condition() -> None: """ Test create_match_filter with a composite key where the source yields exactly one unique key.