diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index b502b43539..96bb63775b 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -445,6 +445,16 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite") return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + @to.register(IntegerType) + @to.register(LongType) + def _(self, type_var: IntegerType | LongType) -> Literal[int]: + if not isfinite(self.value): + raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite") + if not self.value.is_integer(): + raise ValueError(f"Could not convert {self.value} into a {type_var}, value has a fractional part") + # The decimal conversion checks the range of the column type + return DecimalLiteral(Decimal(self.value)).to(type_var) + class DateLiteral(Literal[int]): def __init__(self, value: int) -> None: diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py index d813bcaa07..c5c0d02d3b 100644 --- a/pyiceberg/expressions/parser.py +++ b/pyiceberg/expressions/parser.py @@ -26,6 +26,7 @@ ParserElement, ParseResults, QuotedString, + Regex, Suppress, Word, alphanums, @@ -62,6 +63,7 @@ from pyiceberg.expressions.literals import ( BooleanLiteral, DecimalLiteral, + DoubleLiteral, Literal, LongLiteral, StringLiteral, @@ -105,12 +107,14 @@ def _(result: ParseResults) -> Reference: boolean = one_of(["true", "false"], caseless=True) string = sgl_quoted_string.set_results_name("raw_quoted_string") +# A number with an exponent is an approximate numeric literal (a double), as in ANSI SQL +double = Regex(r"[+-]?(?:\d+\.?\d*|\.\d+)[eE][+-]?\d+").set_results_name("double") decimal = common.real().set_results_name("decimal") integer = common.signed_integer().set_results_name("integer") -literal = Group(string | decimal | integer | boolean).set_results_name("literal") -literal_set = Group( - DelimitedList(string) | DelimitedList(decimal) | DelimitedList(integer) | DelimitedList(boolean) -).set_results_name("literal_set") +literal = Group(string | double | decimal | integer | boolean).set_results_name("literal") +literal_set = Group(DelimitedList(string) | DelimitedList(double | decimal | integer) | DelimitedList(boolean)).set_results_name( + "literal_set" +) @boolean.set_parse_action @@ -126,6 +130,11 @@ def _(result: ParseResults) -> Literal[str]: return StringLiteral(result.raw_quoted_string[1:-1].replace("''", "'")) +@double.set_parse_action +def _(result: ParseResults) -> Literal[float]: + return DoubleLiteral(float(result.double)) + + @decimal.set_parse_action def _(result: ParseResults) -> Literal[Decimal]: return DecimalLiteral(Decimal(result.decimal)) diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 9c4252e57d..551ae46e0d 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -268,6 +268,21 @@ def test_double_to_float_outside_bound() -> None: assert below_min_lit == FloatBelowMin() +def test_double_to_integer_and_long_conversion() -> None: + for type_var in (IntegerType(), LongType()): + assert DoubleLiteral(1e3).to(type_var) == LongLiteral(1000) + assert DoubleLiteral(-2.0).to(type_var) == LongLiteral(-2) + with pytest.raises(ValueError, match="value has a fractional part"): + DoubleLiteral(1.5).to(type_var) + with pytest.raises(ValueError, match="value is not finite"): + DoubleLiteral(float("inf")).to(type_var) + # A float literal still does not convert to an integer type + with pytest.raises(TypeError): + FloatLiteral(1e3).to(type_var) + assert DoubleLiteral(1e10).to(IntegerType()) == IntAboveMax() + assert DoubleLiteral(-1e19).to(LongType()) == LongBelowMin() + + @pytest.mark.parametrize( "decimal_type, decimal_value", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] ) @@ -703,8 +718,6 @@ def test_invalid_long_conversions() -> None: "test_type", [ BooleanType(), - IntegerType(), - LongType(), DateType(), TimeType(), TimestampType(), diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py index 581ee90b72..9b574b097a 100644 --- a/tests/expressions/test_parser.py +++ b/tests/expressions/test_parser.py @@ -43,7 +43,10 @@ Reference, StartsWith, ) -from pyiceberg.expressions.literals import DecimalLiteral, LongLiteral, literal +from pyiceberg.expressions.literals import DecimalLiteral, DoubleLiteral, LongLiteral, literal +from pyiceberg.expressions.visitors import bind +from pyiceberg.schema import Schema +from pyiceberg.types import DecimalType, DoubleType, LongType, NestedField def test_always_true() -> None: @@ -146,6 +149,62 @@ def test_not_in_different_types() -> None: parser.parse("foo not in (5, 'a')") +def test_in_mixed_numbers() -> None: + for values in ("5, 6.5", "6.5, 5"): + result = parser.parse(f"foo in ({values})") + assert result == In("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5"))}), values + assert isinstance(result, In) + assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral}, values + + result = parser.parse("foo not in (5, 6.5, 1e3)") + assert result == NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5")), DoubleLiteral(1000.0)}) + assert isinstance(result, NotIn) + assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral, DoubleLiteral} + + +def test_scientific_notation() -> None: + # A number with an exponent is a double, as in ANSI SQL + for expression, expected in [ + ("foo > 1e-3", GreaterThan("foo", 0.001)), + ("1e-3 < foo", GreaterThan("foo", 0.001)), + ("foo < -1.5E3", LessThan("foo", -1500.0)), + ("foo = 2e2", EqualTo("foo", 200.0)), + ("foo = .5e1", EqualTo("foo", 5.0)), + ]: + result = parser.parse(expression) + assert result == expected, expression + assert isinstance(result.literal, DoubleLiteral), expression + + assert And(GreaterThanOrEqual("foo", 10.0), LessThanOrEqual("foo", 25.0)) == parser.parse("foo between 1e1 and 2.5e1") + + +def test_bind_numeric_literals() -> None: + schema = Schema( + NestedField(1, "dec", DecimalType(9, 2), required=False), + NestedField(2, "lng", LongType(), required=False), + NestedField(3, "dbl", DoubleType(), required=False), + ) + for expression, expected in [ + ("dec = 1e3", DecimalLiteral(Decimal("1000.00"))), + ("dec > 2.5e-1", DecimalLiteral(Decimal("0.25"))), + ("dec = 12.34", DecimalLiteral(Decimal("12.34"))), + ("lng = 1e3", LongLiteral(1000)), + ("lng = 7", LongLiteral(7)), + ("dbl > 1e-3", DoubleLiteral(0.001)), + ("dbl > 2.5", DoubleLiteral(2.5)), + ]: + bound = bind(schema, parser.parse(expression), case_sensitive=True) + assert bound.literal == expected, expression + assert type(bound.literal) is type(expected), expression + if isinstance(expected, DecimalLiteral): + assert bound.literal.value.as_tuple() == expected.value.as_tuple(), expression + + bound = bind(schema, parser.parse("lng in (1, 2e1, 3)"), case_sensitive=True) + assert bound.literals == {LongLiteral(1), LongLiteral(20), LongLiteral(3)} + with pytest.raises(ValueError, match="value has a fractional part"): + bind(schema, parser.parse("lng = 1.5e0"), case_sensitive=True) + + def test_simple_and() -> None: assert And(GreaterThanOrEqual("foo", 5), LessThan("foo", 10)) == parser.parse("5 <= foo and foo < 10")