diff --git a/.github/workflows/pypi-build-artifacts.yml b/.github/workflows/pypi-build-artifacts.yml index 2615e18590..08933b2221 100644 --- a/.github/workflows/pypi-build-artifacts.yml +++ b/.github/workflows/pypi-build-artifacts.yml @@ -51,6 +51,7 @@ jobs: 3.12 3.13 3.14 + 3.15 - name: Install UV uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 @@ -75,13 +76,16 @@ jobs: env: # Ignore 32 bit architectures CIBW_ARCHS: "auto64" - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.11,<3.15" + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.11,<3.16" # Keep these in sync with Python CI job `cibw-dev-env-smoke-test` # in .github/workflows/python-ci.yml to catch import-time regressions early. CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project" CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py" # Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support CIBW_SKIP: "cp3*t-*" + # The locked dev group (pyarrow 25.0.1, numpy 2.4.6) has no CPython 3.15 wheels yet: + # build the cp315 wheels, and test them once it does + CIBW_TEST_SKIP: "cp315-*" - name: Add source distribution diff --git a/pyiceberg/avro/codecs/zstandard_codec.py b/pyiceberg/avro/codecs/zstandard_codec.py index 4cc815214f..67e2347d01 100644 --- a/pyiceberg/avro/codecs/zstandard_codec.py +++ b/pyiceberg/avro/codecs/zstandard_codec.py @@ -16,38 +16,53 @@ # under the License. from __future__ import annotations +import sys from io import BytesIO from pyiceberg.avro.codecs.codec import Codec -try: - from zstandard import ZstdCompressor, ZstdDecompressor +if sys.version_info >= (3, 14): + from compression import zstd class ZStandardCodec(Codec): @staticmethod def compress(data: bytes) -> tuple[bytes, int]: - compressed_data = ZstdCompressor().compress(data) + compressed_data = zstd.compress(data) return compressed_data, len(compressed_data) @staticmethod def decompress(data: bytes) -> bytes: - uncompressed = bytearray() - dctx = ZstdDecompressor() - with dctx.stream_reader(BytesIO(data)) as reader: - while True: - chunk = reader.read(16384) - if not chunk: - break - uncompressed.extend(chunk) - return bytes(uncompressed) - -except ImportError: - - class ZStandardCodec(Codec): # type: ignore - @staticmethod - def compress(data: bytes) -> tuple[bytes, int]: - raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") + return zstd.decompress(data) - @staticmethod - def decompress(data: bytes) -> bytes: - raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") +else: + try: + from zstandard import ZstdCompressor, ZstdDecompressor + + class ZStandardCodec(Codec): + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + compressed_data = ZstdCompressor().compress(data) + return compressed_data, len(compressed_data) + + @staticmethod + def decompress(data: bytes) -> bytes: + uncompressed = bytearray() + dctx = ZstdDecompressor() + with dctx.stream_reader(BytesIO(data)) as reader: + while True: + chunk = reader.read(16384) + if not chunk: + break + uncompressed.extend(chunk) + return bytes(uncompressed) + + except ImportError: + + class ZStandardCodec(Codec): # type: ignore + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") + + @staticmethod + def decompress(data: bytes) -> bytes: + raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") diff --git a/pyiceberg/table/puffin.py b/pyiceberg/table/puffin.py index 13315bf802..b0794ebbe7 100644 --- a/pyiceberg/table/puffin.py +++ b/pyiceberg/table/puffin.py @@ -14,9 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import sys from typing import TYPE_CHECKING -import zstandard from pydantic import Field from pyiceberg.typedef import IcebergBaseModel @@ -25,6 +25,15 @@ if TYPE_CHECKING: import pyarrow as pa +if sys.version_info >= (3, 14): + from compression.zstd import decompress as _zstd_decompress +else: + import zstandard + + def _zstd_decompress(data: bytes) -> bytes: + return zstandard.ZstdDecompressor().decompress(data) + + # Short for: Puffin Fratercula arctica, version 1 MAGIC_BYTES = b"PFA1" @@ -73,7 +82,7 @@ def get_blob_payload(self, blob: PuffinBlobMetadata) -> bytes: if blob.compression_codec is None: return raw elif blob.compression_codec == "zstd": - return zstandard.ZstdDecompressor().decompress(raw) + return _zstd_decompress(raw) else: raise ValueError(f"Unsupported puffin compression codec: {blob.compression_codec!r}") diff --git a/pyproject.toml b/pyproject.toml index a377f434f8..103807f2fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "tenacity>=8.2.3", "pyroaring>=1.1.0", "cachetools>=5.5", - "zstandard>=0.13.0" + "zstandard>=0.13.0; python_version < '3.14'" ] [project.urls] diff --git a/uv.lock b/uv.lock index 49d0f7f147..c720c23509 100644 --- a/uv.lock +++ b/uv.lock @@ -506,8 +506,8 @@ resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions" }, + { name = "soupsieve", marker = "python_full_version >= '3.11.5'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11.5'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } wheels = [ @@ -4511,7 +4511,7 @@ dependencies = [ { name = "rich" }, { name = "strictyaml" }, { name = "tenacity" }, - { name = "zstandard" }, + { name = "zstandard", marker = "python_full_version < '3.14'" }, ] [package.optional-dependencies] @@ -4692,7 +4692,7 @@ requires-dist = [ { name = "thrift", marker = "extra == 'hive'", specifier = ">=0.13.0" }, { name = "thrift", marker = "extra == 'hive-kerberos'", specifier = ">=0.13.0" }, { name = "thrift-sasl", marker = "extra == 'hive-kerberos'", specifier = ">=0.4.3" }, - { name = "zstandard", specifier = ">=0.13.0" }, + { name = "zstandard", marker = "python_full_version < '3.14'", specifier = ">=0.13.0" }, ] provides-extras = ["pyarrow", "pandas", "duckdb", "ray", "bodo", "daft", "polars", "snappy", "hive", "hive-kerberos", "s3fs", "glue", "adlfs", "dynamodb", "bigquery", "sql-postgres", "sql-sqlite", "gcsfs", "rest-sigv4", "hf", "pyiceberg-core", "datafusion", "gcp-auth", "entra-auth", "geoarrow", "encryption"]