From 864776d9bb7a12256ad0251de02b881eb765daec Mon Sep 17 00:00:00 2001 From: zn-danielbariudin Date: Sun, 15 Nov 2026 16:38:05 +0200 Subject: [PATCH] Avro: Cache the Avro to Iceberg schema conversion per schema string --- pyiceberg/avro/file.py | 10 +++++++--- tests/avro/test_file.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 4b23c30ac0..f1172a83b5 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -25,6 +25,7 @@ from collections.abc import Callable from dataclasses import dataclass from enum import Enum +from functools import lru_cache from types import TracebackType from typing import ( Generic, @@ -71,6 +72,11 @@ _SCHEMA_KEY = "avro.schema" +@lru_cache +def _avro_schema_string_to_iceberg(avro_schema_string: str) -> Schema: + return AvroSchemaConversion().avro_to_iceberg(json.loads(avro_schema_string)) + + class AvroFileHeader(Record): @property def magic(self) -> bytes: @@ -99,9 +105,7 @@ def compression_codec(self) -> type[Codec] | None: def get_schema(self) -> Schema: if _SCHEMA_KEY in self.meta: - avro_schema_string = self.meta[_SCHEMA_KEY] - avro_schema = json.loads(avro_schema_string) - return AvroSchemaConversion().avro_to_iceberg(avro_schema) + return _avro_schema_string_to_iceberg(self.meta[_SCHEMA_KEY]) else: raise ValueError("No schema found in Avro file headers") diff --git a/tests/avro/test_file.py b/tests/avro/test_file.py index d0598c8b86..595fceb585 100644 --- a/tests/avro/test_file.py +++ b/tests/avro/test_file.py @@ -99,6 +99,28 @@ def test_missing_schema() -> None: assert "No schema found in Avro file headers" in str(exc_info.value) +def test_get_schema_reuses_conversion_for_identical_schema_strings() -> None: + schema_string = ( + '{"type": "record", "name": "manifest_entry", "fields": [' + '{"name": "status", "type": "int", "field-id": 0}, ' + '{"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1}]}' + ) + other_schema_string = ( + '{"type": "record", "name": "manifest_entry", "fields": [{"name": "status", "type": "int", "field-id": 0}]}' + ) + + header = AvroFileHeader(bytes(0), {"avro.schema": schema_string}, bytes(16)) + same_schema_header = AvroFileHeader(bytes(0), {"avro.schema": schema_string}, bytes(16)) + other_schema_header = AvroFileHeader(bytes(0), {"avro.schema": other_schema_string}, bytes(16)) + + assert header.get_schema() == Schema( + NestedField(field_id=0, name="status", field_type=IntegerType(), required=True), + NestedField(field_id=1, name="snapshot_id", field_type=LongType(), required=False), + ) + assert header.get_schema() is same_schema_header.get_schema() + assert other_schema_header.get_schema() is not header.get_schema() + + # helper function to serialize our objects to dicts to enable # direct comparison with the dicts returned by fastavro def todict(obj: Any) -> Any: