Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/pypi-build-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ jobs:
3.12
3.13
3.14
3.15

- name: Install UV
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
Expand All @@ -75,13 +76,16 @@ jobs:
env:
# Ignore 32 bit architectures
CIBW_ARCHS: "auto64"
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.11,<3.15"
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.11,<3.16"
# Keep these in sync with Python CI job `cibw-dev-env-smoke-test`
# in .github/workflows/python-ci.yml to catch import-time regressions early.
CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project"
CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py"
# Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support
CIBW_SKIP: "cp3*t-*"
# The locked dev group (pyarrow 25.0.1, numpy 2.4.6) has no CPython 3.15 wheels yet:
# build the cp315 wheels, and test them once it does
CIBW_TEST_SKIP: "cp315-*"


- name: Add source distribution
Expand Down
59 changes: 37 additions & 22 deletions pyiceberg/avro/codecs/zstandard_codec.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,38 +16,53 @@
# under the License.
from __future__ import annotations

import sys
from io import BytesIO

from pyiceberg.avro.codecs.codec import Codec

try:
from zstandard import ZstdCompressor, ZstdDecompressor
if sys.version_info >= (3, 14):
from compression import zstd

class ZStandardCodec(Codec):
@staticmethod
def compress(data: bytes) -> tuple[bytes, int]:
compressed_data = ZstdCompressor().compress(data)
compressed_data = zstd.compress(data)
return compressed_data, len(compressed_data)

@staticmethod
def decompress(data: bytes) -> bytes:
uncompressed = bytearray()
dctx = ZstdDecompressor()
with dctx.stream_reader(BytesIO(data)) as reader:
while True:
chunk = reader.read(16384)
if not chunk:
break
uncompressed.extend(chunk)
return bytes(uncompressed)

except ImportError:

class ZStandardCodec(Codec): # type: ignore
@staticmethod
def compress(data: bytes) -> tuple[bytes, int]:
raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`")
return zstd.decompress(data)

@staticmethod
def decompress(data: bytes) -> bytes:
raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`")
else:
try:
from zstandard import ZstdCompressor, ZstdDecompressor

class ZStandardCodec(Codec):
@staticmethod
def compress(data: bytes) -> tuple[bytes, int]:
compressed_data = ZstdCompressor().compress(data)
return compressed_data, len(compressed_data)

@staticmethod
def decompress(data: bytes) -> bytes:
uncompressed = bytearray()
dctx = ZstdDecompressor()
with dctx.stream_reader(BytesIO(data)) as reader:
while True:
chunk = reader.read(16384)
if not chunk:
break
uncompressed.extend(chunk)
return bytes(uncompressed)

except ImportError:

class ZStandardCodec(Codec): # type: ignore
@staticmethod
def compress(data: bytes) -> tuple[bytes, int]:
raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`")

@staticmethod
def decompress(data: bytes) -> bytes:
raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`")
13 changes: 11 additions & 2 deletions pyiceberg/table/puffin.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import sys
from typing import TYPE_CHECKING

import zstandard
from pydantic import Field

from pyiceberg.typedef import IcebergBaseModel
Expand All @@ -25,6 +25,15 @@
if TYPE_CHECKING:
import pyarrow as pa

if sys.version_info >= (3, 14):
from compression.zstd import decompress as _zstd_decompress
else:
import zstandard

def _zstd_decompress(data: bytes) -> bytes:
return zstandard.ZstdDecompressor().decompress(data)


# Short for: Puffin Fratercula arctica, version 1
MAGIC_BYTES = b"PFA1"

Expand Down Expand Up @@ -73,7 +82,7 @@ def get_blob_payload(self, blob: PuffinBlobMetadata) -> bytes:
if blob.compression_codec is None:
return raw
elif blob.compression_codec == "zstd":
return zstandard.ZstdDecompressor().decompress(raw)
return _zstd_decompress(raw)
else:
raise ValueError(f"Unsupported puffin compression codec: {blob.compression_codec!r}")

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ dependencies = [
"tenacity>=8.2.3",
"pyroaring>=1.1.0",
"cachetools>=5.5",
"zstandard>=0.13.0"
"zstandard>=0.13.0; python_version < '3.14'"
]

[project.urls]
Expand Down
8 changes: 4 additions & 4 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading