Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions pyiceberg/expressions/literals.py
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,16 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]:
raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite")
return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP))

@to.register(IntegerType)
@to.register(LongType)
def _(self, type_var: IntegerType | LongType) -> Literal[int]:
if not isfinite(self.value):
raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite")
if not self.value.is_integer():
raise ValueError(f"Could not convert {self.value} into a {type_var}, value has a fractional part")
# The decimal conversion checks the range of the column type
return DecimalLiteral(Decimal(self.value)).to(type_var)


class DateLiteral(Literal[int]):
def __init__(self, value: int) -> None:
Expand Down
17 changes: 13 additions & 4 deletions pyiceberg/expressions/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
ParserElement,
ParseResults,
QuotedString,
Regex,
Suppress,
Word,
alphanums,
Expand Down Expand Up @@ -62,6 +63,7 @@
from pyiceberg.expressions.literals import (
BooleanLiteral,
DecimalLiteral,
DoubleLiteral,
Literal,
LongLiteral,
StringLiteral,
Expand Down Expand Up @@ -105,12 +107,14 @@ def _(result: ParseResults) -> Reference:

boolean = one_of(["true", "false"], caseless=True)
string = sgl_quoted_string.set_results_name("raw_quoted_string")
# A number with an exponent is an approximate numeric literal (a double), as in ANSI SQL
double = Regex(r"[+-]?(?:\d+\.?\d*|\.\d+)[eE][+-]?\d+").set_results_name("double")
decimal = common.real().set_results_name("decimal")
integer = common.signed_integer().set_results_name("integer")
literal = Group(string | decimal | integer | boolean).set_results_name("literal")
literal_set = Group(
DelimitedList(string) | DelimitedList(decimal) | DelimitedList(integer) | DelimitedList(boolean)
).set_results_name("literal_set")
literal = Group(string | double | decimal | integer | boolean).set_results_name("literal")
literal_set = Group(DelimitedList(string) | DelimitedList(double | decimal | integer) | DelimitedList(boolean)).set_results_name(
"literal_set"
)


@boolean.set_parse_action
Expand All @@ -126,6 +130,11 @@ def _(result: ParseResults) -> Literal[str]:
return StringLiteral(result.raw_quoted_string[1:-1].replace("''", "'"))


@double.set_parse_action
def _(result: ParseResults) -> Literal[float]:
return DoubleLiteral(float(result.double))


@decimal.set_parse_action
def _(result: ParseResults) -> Literal[Decimal]:
return DecimalLiteral(Decimal(result.decimal))
Expand Down
17 changes: 15 additions & 2 deletions tests/expressions/test_literals.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,21 @@ def test_double_to_float_outside_bound() -> None:
assert below_min_lit == FloatBelowMin()


def test_double_to_integer_and_long_conversion() -> None:
for type_var in (IntegerType(), LongType()):
assert DoubleLiteral(1e3).to(type_var) == LongLiteral(1000)
assert DoubleLiteral(-2.0).to(type_var) == LongLiteral(-2)
with pytest.raises(ValueError, match="value has a fractional part"):
DoubleLiteral(1.5).to(type_var)
with pytest.raises(ValueError, match="value is not finite"):
DoubleLiteral(float("inf")).to(type_var)
# A float literal still does not convert to an integer type
with pytest.raises(TypeError):
FloatLiteral(1e3).to(type_var)
assert DoubleLiteral(1e10).to(IntegerType()) == IntAboveMax()
assert DoubleLiteral(-1e19).to(LongType()) == LongBelowMin()


@pytest.mark.parametrize(
"decimal_type, decimal_value", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")]
)
Expand Down Expand Up @@ -703,8 +718,6 @@ def test_invalid_long_conversions() -> None:
"test_type",
[
BooleanType(),
IntegerType(),
LongType(),
DateType(),
TimeType(),
TimestampType(),
Expand Down
61 changes: 60 additions & 1 deletion tests/expressions/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@
Reference,
StartsWith,
)
from pyiceberg.expressions.literals import DecimalLiteral, LongLiteral, literal
from pyiceberg.expressions.literals import DecimalLiteral, DoubleLiteral, LongLiteral, literal
from pyiceberg.expressions.visitors import bind
from pyiceberg.schema import Schema
from pyiceberg.types import DecimalType, DoubleType, LongType, NestedField


def test_always_true() -> None:
Expand Down Expand Up @@ -146,6 +149,62 @@ def test_not_in_different_types() -> None:
parser.parse("foo not in (5, 'a')")


def test_in_mixed_numbers() -> None:
for values in ("5, 6.5", "6.5, 5"):
result = parser.parse(f"foo in ({values})")
assert result == In("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5"))}), values
assert isinstance(result, In)
assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral}, values

result = parser.parse("foo not in (5, 6.5, 1e3)")
assert result == NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5")), DoubleLiteral(1000.0)})
assert isinstance(result, NotIn)
assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral, DoubleLiteral}


def test_scientific_notation() -> None:
# A number with an exponent is a double, as in ANSI SQL
for expression, expected in [
("foo > 1e-3", GreaterThan("foo", 0.001)),
("1e-3 < foo", GreaterThan("foo", 0.001)),
("foo < -1.5E3", LessThan("foo", -1500.0)),
("foo = 2e2", EqualTo("foo", 200.0)),
("foo = .5e1", EqualTo("foo", 5.0)),
]:
result = parser.parse(expression)
assert result == expected, expression
assert isinstance(result.literal, DoubleLiteral), expression

assert And(GreaterThanOrEqual("foo", 10.0), LessThanOrEqual("foo", 25.0)) == parser.parse("foo between 1e1 and 2.5e1")


def test_bind_numeric_literals() -> None:
schema = Schema(
NestedField(1, "dec", DecimalType(9, 2), required=False),
NestedField(2, "lng", LongType(), required=False),
NestedField(3, "dbl", DoubleType(), required=False),
)
for expression, expected in [
("dec = 1e3", DecimalLiteral(Decimal("1000.00"))),
("dec > 2.5e-1", DecimalLiteral(Decimal("0.25"))),
("dec = 12.34", DecimalLiteral(Decimal("12.34"))),
("lng = 1e3", LongLiteral(1000)),
("lng = 7", LongLiteral(7)),
("dbl > 1e-3", DoubleLiteral(0.001)),
("dbl > 2.5", DoubleLiteral(2.5)),
]:
bound = bind(schema, parser.parse(expression), case_sensitive=True)
assert bound.literal == expected, expression
assert type(bound.literal) is type(expected), expression
if isinstance(expected, DecimalLiteral):
assert bound.literal.value.as_tuple() == expected.value.as_tuple(), expression

bound = bind(schema, parser.parse("lng in (1, 2e1, 3)"), case_sensitive=True)
assert bound.literals == {LongLiteral(1), LongLiteral(20), LongLiteral(3)}
with pytest.raises(ValueError, match="value has a fractional part"):
bind(schema, parser.parse("lng = 1.5e0"), case_sensitive=True)


def test_simple_and() -> None:
assert And(GreaterThanOrEqual("foo", 5), LessThan("foo", 10)) == parser.parse("5 <= foo and foo < 10")

Expand Down
Loading