Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,8 @@ def append(
df: pa.Table | pa.RecordBatchReader,
snapshot_properties: dict[str, str] = EMPTY_DICT,
branch: str | None = MAIN_BRANCH,
_upsert_insert_filter: BooleanExpression | None = None,
_case_sensitive: bool = True,
) -> None:
"""
Shorthand API for appending PyArrow data to a table transaction.
Expand Down Expand Up @@ -576,6 +578,9 @@ def append(
)

with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files:
if _upsert_insert_filter is not None:
append_files.set_upsert_insert_filter(_upsert_insert_filter, _case_sensitive)

# For pa.Table we can short-circuit empty inputs cheaply. For a
# RecordBatchReader the stream is consumed lazily by
# _dataframe_to_data_files and an empty reader simply yields zero
Expand Down Expand Up @@ -1014,7 +1019,13 @@ def upsert(
if when_not_matched_insert_all:
insert_row_cnt = len(rows_to_insert)
if rows_to_insert:
self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties)
self.append(
rows_to_insert,
branch=branch,
snapshot_properties=snapshot_properties,
_upsert_insert_filter=upsert_util.create_match_filter(rows_to_insert, join_cols),
_case_sensitive=case_sensitive,
)

return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt)

Expand Down
34 changes: 33 additions & 1 deletion pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

from pyiceberg.avro.codecs import AvroCompressionCodec
from pyiceberg.exceptions import (
CommitFailedException,
NoSuchSnapshotRefError,
NotAncestorError,
SnapshotRefTypeError,
Expand Down Expand Up @@ -690,8 +691,39 @@ def _refresh_for_retry(self) -> None:


class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]):
_upsert_insert_filter: BooleanExpression | None = None

def set_upsert_insert_filter(self, insert_filter: BooleanExpression, case_sensitive: bool) -> None:
self._upsert_insert_filter = insert_filter
self._case_sensitive = case_sensitive

def _validate_concurrency(self) -> None:
"""Skip validation; appends do not conflict with other operations."""
"""Skip validation for plain appends.

An upsert registers the filter for the rows it inserts, so a retry fails instead of
inserting a key that another writer added after the upsert was planned.
"""
if self._upsert_insert_filter is None or self._commit_window is None or self._commit_window.is_empty():
return

from pyiceberg.table.update.validate import _validate_added_data_files

catalog_head = self._commit_window.head
if catalog_head is None:
return

try:
_validate_added_data_files(
self._transaction._table,
catalog_head,
self._upsert_insert_filter,
self._commit_window.base,
self._case_sensitive,
)
except ValidationException as e:
raise CommitFailedException(
"Cannot retry the upsert because a concurrent commit added rows matching the rows to insert"
) from e

def _existing_manifests(self) -> list[ManifestFile]:
"""To determine if there are any existing manifest files.
Expand Down
29 changes: 27 additions & 2 deletions tests/table/test_upsert.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from pyarrow import Table as pa_table

from pyiceberg.catalog import Catalog
from pyiceberg.exceptions import NoSuchTableError
from pyiceberg.exceptions import CommitFailedException, NoSuchTableError
from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference
from pyiceberg.expressions.literals import LongLiteral
from pyiceberg.io.pyarrow import schema_to_pyarrow
Expand All @@ -33,7 +33,7 @@
from pyiceberg.table.snapshots import Operation
from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update
from pyiceberg.transforms import DayTransform
from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType
from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType
from tests.catalog.test_base import InMemoryCatalog


Expand Down Expand Up @@ -427,6 +427,31 @@ def test_upsert_into_empty_table(catalog: Catalog) -> None:
assert upd.rows_inserted == 4


def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None:
identifier = "default.test_upsert_retry_with_a_concurrently_inserted_key"
_drop_table(catalog, identifier)

schema = Schema(
NestedField(1, "k", LongType(), required=True),
NestedField(2, "v", StringType(), required=False),
identifier_field_ids=[1],
)
arrow_schema = schema_to_pyarrow(schema)
catalog.create_table(identifier, schema=schema).append(pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema))

writer_a = catalog.load_table(identifier)
writer_b = catalog.load_table(identifier)
writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema))

with pytest.raises(CommitFailedException):
writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema))

assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [
{"k": 1, "v": "a"},
{"k": 2, "v": "b"},
]


def test_create_match_filter_single_condition() -> None:
"""
Test create_match_filter with a composite key where the source yields exactly one unique key.
Expand Down
Loading