From f81b82789fd97ee63457bcabf94fcee1bee664dc Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Fri, 13 Nov 2026 17:20:49 +0000 Subject: [PATCH 1/2] Parse scientific notation and mixed int/decimal IN lists in filters The row filter parser only accepted decimals with a fractional part and literal sets of a single numeric kind, so `x > 1e-3` and `x in (1, 2.5)` raised a ParseException. Co-authored-by: Isaac --- pyiceberg/expressions/parser.py | 8 ++++---- tests/expressions/test_parser.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py index d813bcaa07..ac327abe87 100644 --- a/pyiceberg/expressions/parser.py +++ b/pyiceberg/expressions/parser.py @@ -105,12 +105,12 @@ def _(result: ParseResults) -> Reference: boolean = one_of(["true", "false"], caseless=True) string = sgl_quoted_string.set_results_name("raw_quoted_string") -decimal = common.real().set_results_name("decimal") +decimal = common.sci_real().set_results_name("decimal") integer = common.signed_integer().set_results_name("integer") literal = Group(string | decimal | integer | boolean).set_results_name("literal") -literal_set = Group( - DelimitedList(string) | DelimitedList(decimal) | DelimitedList(integer) | DelimitedList(boolean) -).set_results_name("literal_set") +literal_set = Group(DelimitedList(string) | DelimitedList(decimal | integer) | DelimitedList(boolean)).set_results_name( + "literal_set" +) @boolean.set_parse_action diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py index 581ee90b72..f44248873d 100644 --- a/tests/expressions/test_parser.py +++ b/tests/expressions/test_parser.py @@ -146,6 +146,26 @@ def test_not_in_different_types() -> None: parser.parse("foo not in (5, 'a')") +def test_in_mixed_numbers() -> None: + for values in ("5, 6.5", "6.5, 5"): + result = parser.parse(f"foo in ({values})") + assert result == In("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5"))}), values + assert isinstance(result, In) + assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral}, values + assert NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("1e3"))}) == parser.parse("foo not in (5, 1e3)") + + +def test_scientific_notation() -> None: + for expression, expected in [ + ("foo > 1e-3", GreaterThan("foo", Decimal("0.001"))), + ("1e-3 < foo", GreaterThan("foo", Decimal("0.001"))), + ("foo < -1.5E3", LessThan("foo", Decimal("-1500"))), + ("foo = 2e2", EqualTo("foo", Decimal("200"))), + ("foo between 1e1 and 2.5e1", And(GreaterThanOrEqual("foo", Decimal("10")), LessThanOrEqual("foo", Decimal("25")))), + ]: + assert parser.parse(expression) == expected, expression + + def test_simple_and() -> None: assert And(GreaterThanOrEqual("foo", 5), LessThan("foo", 10)) == parser.parse("5 <= foo and foo < 10") From bccbed0242a26ae3209a49f12e73e6efda426da9 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Mon, 16 Nov 2026 13:21:59 +0000 Subject: [PATCH 2/2] Parse exponent literals as doubles and convert whole doubles to integer types A number with an exponent is an approximate numeric literal in ANSI SQL, and Spark, Trino and DuckDB parse 1e-3 as a double. As a DecimalLiteral, 1e3 has scale -3 and could not bind to decimal columns. Parse the exponent forms as DoubleLiteral instead, and convert a whole DoubleLiteral to int and long columns (a fractional value raises, as for decimals) so that long_col = 1e3 binds. Co-authored-by: Isaac --- pyiceberg/expressions/literals.py | 10 ++++++ pyiceberg/expressions/parser.py | 15 ++++++-- tests/expressions/test_literals.py | 17 +++++++-- tests/expressions/test_parser.py | 55 +++++++++++++++++++++++++----- 4 files changed, 84 insertions(+), 13 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index b502b43539..96bb63775b 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -445,6 +445,16 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite") return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + @to.register(IntegerType) + @to.register(LongType) + def _(self, type_var: IntegerType | LongType) -> Literal[int]: + if not isfinite(self.value): + raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite") + if not self.value.is_integer(): + raise ValueError(f"Could not convert {self.value} into a {type_var}, value has a fractional part") + # The decimal conversion checks the range of the column type + return DecimalLiteral(Decimal(self.value)).to(type_var) + class DateLiteral(Literal[int]): def __init__(self, value: int) -> None: diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py index ac327abe87..c5c0d02d3b 100644 --- a/pyiceberg/expressions/parser.py +++ b/pyiceberg/expressions/parser.py @@ -26,6 +26,7 @@ ParserElement, ParseResults, QuotedString, + Regex, Suppress, Word, alphanums, @@ -62,6 +63,7 @@ from pyiceberg.expressions.literals import ( BooleanLiteral, DecimalLiteral, + DoubleLiteral, Literal, LongLiteral, StringLiteral, @@ -105,10 +107,12 @@ def _(result: ParseResults) -> Reference: boolean = one_of(["true", "false"], caseless=True) string = sgl_quoted_string.set_results_name("raw_quoted_string") -decimal = common.sci_real().set_results_name("decimal") +# A number with an exponent is an approximate numeric literal (a double), as in ANSI SQL +double = Regex(r"[+-]?(?:\d+\.?\d*|\.\d+)[eE][+-]?\d+").set_results_name("double") +decimal = common.real().set_results_name("decimal") integer = common.signed_integer().set_results_name("integer") -literal = Group(string | decimal | integer | boolean).set_results_name("literal") -literal_set = Group(DelimitedList(string) | DelimitedList(decimal | integer) | DelimitedList(boolean)).set_results_name( +literal = Group(string | double | decimal | integer | boolean).set_results_name("literal") +literal_set = Group(DelimitedList(string) | DelimitedList(double | decimal | integer) | DelimitedList(boolean)).set_results_name( "literal_set" ) @@ -126,6 +130,11 @@ def _(result: ParseResults) -> Literal[str]: return StringLiteral(result.raw_quoted_string[1:-1].replace("''", "'")) +@double.set_parse_action +def _(result: ParseResults) -> Literal[float]: + return DoubleLiteral(float(result.double)) + + @decimal.set_parse_action def _(result: ParseResults) -> Literal[Decimal]: return DecimalLiteral(Decimal(result.decimal)) diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 9c4252e57d..551ae46e0d 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -268,6 +268,21 @@ def test_double_to_float_outside_bound() -> None: assert below_min_lit == FloatBelowMin() +def test_double_to_integer_and_long_conversion() -> None: + for type_var in (IntegerType(), LongType()): + assert DoubleLiteral(1e3).to(type_var) == LongLiteral(1000) + assert DoubleLiteral(-2.0).to(type_var) == LongLiteral(-2) + with pytest.raises(ValueError, match="value has a fractional part"): + DoubleLiteral(1.5).to(type_var) + with pytest.raises(ValueError, match="value is not finite"): + DoubleLiteral(float("inf")).to(type_var) + # A float literal still does not convert to an integer type + with pytest.raises(TypeError): + FloatLiteral(1e3).to(type_var) + assert DoubleLiteral(1e10).to(IntegerType()) == IntAboveMax() + assert DoubleLiteral(-1e19).to(LongType()) == LongBelowMin() + + @pytest.mark.parametrize( "decimal_type, decimal_value", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] ) @@ -703,8 +718,6 @@ def test_invalid_long_conversions() -> None: "test_type", [ BooleanType(), - IntegerType(), - LongType(), DateType(), TimeType(), TimestampType(), diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py index f44248873d..9b574b097a 100644 --- a/tests/expressions/test_parser.py +++ b/tests/expressions/test_parser.py @@ -43,7 +43,10 @@ Reference, StartsWith, ) -from pyiceberg.expressions.literals import DecimalLiteral, LongLiteral, literal +from pyiceberg.expressions.literals import DecimalLiteral, DoubleLiteral, LongLiteral, literal +from pyiceberg.expressions.visitors import bind +from pyiceberg.schema import Schema +from pyiceberg.types import DecimalType, DoubleType, LongType, NestedField def test_always_true() -> None: @@ -152,18 +155,54 @@ def test_in_mixed_numbers() -> None: assert result == In("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5"))}), values assert isinstance(result, In) assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral}, values - assert NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("1e3"))}) == parser.parse("foo not in (5, 1e3)") + + result = parser.parse("foo not in (5, 6.5, 1e3)") + assert result == NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5")), DoubleLiteral(1000.0)}) + assert isinstance(result, NotIn) + assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral, DoubleLiteral} def test_scientific_notation() -> None: + # A number with an exponent is a double, as in ANSI SQL + for expression, expected in [ + ("foo > 1e-3", GreaterThan("foo", 0.001)), + ("1e-3 < foo", GreaterThan("foo", 0.001)), + ("foo < -1.5E3", LessThan("foo", -1500.0)), + ("foo = 2e2", EqualTo("foo", 200.0)), + ("foo = .5e1", EqualTo("foo", 5.0)), + ]: + result = parser.parse(expression) + assert result == expected, expression + assert isinstance(result.literal, DoubleLiteral), expression + + assert And(GreaterThanOrEqual("foo", 10.0), LessThanOrEqual("foo", 25.0)) == parser.parse("foo between 1e1 and 2.5e1") + + +def test_bind_numeric_literals() -> None: + schema = Schema( + NestedField(1, "dec", DecimalType(9, 2), required=False), + NestedField(2, "lng", LongType(), required=False), + NestedField(3, "dbl", DoubleType(), required=False), + ) for expression, expected in [ - ("foo > 1e-3", GreaterThan("foo", Decimal("0.001"))), - ("1e-3 < foo", GreaterThan("foo", Decimal("0.001"))), - ("foo < -1.5E3", LessThan("foo", Decimal("-1500"))), - ("foo = 2e2", EqualTo("foo", Decimal("200"))), - ("foo between 1e1 and 2.5e1", And(GreaterThanOrEqual("foo", Decimal("10")), LessThanOrEqual("foo", Decimal("25")))), + ("dec = 1e3", DecimalLiteral(Decimal("1000.00"))), + ("dec > 2.5e-1", DecimalLiteral(Decimal("0.25"))), + ("dec = 12.34", DecimalLiteral(Decimal("12.34"))), + ("lng = 1e3", LongLiteral(1000)), + ("lng = 7", LongLiteral(7)), + ("dbl > 1e-3", DoubleLiteral(0.001)), + ("dbl > 2.5", DoubleLiteral(2.5)), ]: - assert parser.parse(expression) == expected, expression + bound = bind(schema, parser.parse(expression), case_sensitive=True) + assert bound.literal == expected, expression + assert type(bound.literal) is type(expected), expression + if isinstance(expected, DecimalLiteral): + assert bound.literal.value.as_tuple() == expected.value.as_tuple(), expression + + bound = bind(schema, parser.parse("lng in (1, 2e1, 3)"), case_sensitive=True) + assert bound.literals == {LongLiteral(1), LongLiteral(20), LongLiteral(3)} + with pytest.raises(ValueError, match="value has a fractional part"): + bind(schema, parser.parse("lng = 1.5e0"), case_sensitive=True) def test_simple_and() -> None: