From bdd1d2464ef531d15812023a6fe36312b26f962c Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Wed, 19 Aug 2026 12:56:37 -0700 Subject: [PATCH 1/5] [Pratt parser] Port Pratt parser from cel/cpp --- parser/BUILD.bazel | 6 + parser/helper.go | 10 + parser/lexer.go | 727 ++++++++++++++++++++ parser/lexer_test.go | 590 +++++++++++++++++ parser/pratt_parser.go | 1012 ++++++++++++++++++++++++++++ parser/pratt_parser_test.go | 1239 +++++++++++++++++++++++++++++++++++ 6 files changed, 3584 insertions(+) create mode 100644 parser/lexer.go create mode 100644 parser/lexer_test.go create mode 100644 parser/pratt_parser.go create mode 100644 parser/pratt_parser_test.go diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel index c66661639..e4ff679d1 100644 --- a/parser/BUILD.bazel +++ b/parser/BUILD.bazel @@ -10,9 +10,11 @@ go_library( "errors.go", "helper.go", "input.go", + "lexer.go", "macro.go", "options.go", "parser.go", + "pratt_parser.go", "unescape.go", "unparser.go", ], @@ -38,7 +40,9 @@ go_test( size = "small", srcs = [ "helper_test.go", + "lexer_test.go", "parser_test.go", + "pratt_parser_test.go", "unescape_test.go", "unparser_test.go", ], @@ -46,9 +50,11 @@ go_test( ":go_default_library", ], deps = [ + "//common:go_default_library", "//common/ast:go_default_library", "//common/debug:go_default_library", "//common/operators:go_default_library", + "//common/runes:go_default_library", "//common/types:go_default_library", "//parser/gen:go_default_library", "//test:go_default_library", diff --git a/parser/helper.go b/parser/helper.go index b043ef54b..8603750a6 100644 --- a/parser/helper.go +++ b/parser/helper.go @@ -162,6 +162,9 @@ func (p *parserHelper) id(ctx any) int64 { case antlr.Token: offset.Start = p.sourceInfo.ComputeOffset(int32(c.GetLine()), int32(c.GetColumn())) offset.Stop = offset.Start + int32(len(c.GetText())) + case token: + offset.Start = c.start + offset.Stop = c.end case common.Location: offset.Start = p.sourceInfo.ComputeOffsetAbsolute(int32(c.Line()), int32(c.Column())) offset.Stop = offset.Start @@ -177,6 +180,13 @@ func (p *parserHelper) id(ctx any) int64 { return id } +func (p *parserHelper) idFromOffsets(start, stop int32) int64 { + id := p.nextID + p.sourceInfo.SetOffsetRange(id, ast.OffsetRange{Start: start, Stop: stop}) + p.nextID++ + return id +} + func (p *parserHelper) deleteID(id int64) { p.sourceInfo.ClearOffsetRange(id) if id == p.nextID-1 { diff --git a/parser/lexer.go b/parser/lexer.go new file mode 100644 index 000000000..a045fb561 --- /dev/null +++ b/parser/lexer.go @@ -0,0 +1,727 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "fmt" + + "cel.dev/cel-go/common/runes" +) + +type tokenKind int + +const ( + tokError tokenKind = iota + tokEnd + tokWhitespace + tokComment + + // Keywords + tokNull + tokFalse + tokTrue + tokIn + tokReservedWord + + // Literals + tokInt + tokUint + tokFloat + tokString + tokBytes + + // Identifiers + tokIdent + + // Delimiters + tokLeftBracket // [ + tokRightBracket // ] + tokLeftBrace // { + tokRightBrace // } + tokLeftParen // ( + tokRightParen // ) + + // Operators + tokDot // . + tokComma // , + tokMinus // - + tokPlus // + + tokAsterisk // * + tokSlash // / + tokPercent // % + tokQuestion // ? + tokColon // : + tokExclamation // ! + tokEqual // = + tokEqualEqual // == + tokExclamationEqual // != + tokLess // < + tokLessEqual // <= + tokGreater // > + tokGreaterEqual // >= + tokLogicalAnd // && + tokLogicalOr // || +) + +func (t tokenKind) String() string { + switch t { + case tokError: + return "error" + case tokEnd: + return "end" + case tokWhitespace: + return "whitespace" + case tokComment: + return "comment" + case tokNull: + return "null" + case tokFalse: + return "false" + case tokTrue: + return "true" + case tokIn: + return "in" + case tokReservedWord: + return "reserved_word" + case tokInt: + return "int" + case tokUint: + return "uint" + case tokFloat: + return "float" + case tokString: + return "string" + case tokBytes: + return "bytes" + case tokIdent: + return "ident" + case tokLeftBracket: + return "[" + case tokRightBracket: + return "]" + case tokLeftBrace: + return "{" + case tokRightBrace: + return "}" + case tokLeftParen: + return "(" + case tokRightParen: + return ")" + case tokDot: + return "." + case tokComma: + return "," + case tokMinus: + return "-" + case tokPlus: + return "+" + case tokAsterisk: + return "*" + case tokSlash: + return "/" + case tokPercent: + return "%" + case tokQuestion: + return "?" + case tokColon: + return ":" + case tokExclamation: + return "!" + case tokEqual: + return "=" + case tokEqualEqual: + return "==" + case tokExclamationEqual: + return "!=" + case tokLess: + return "<" + case tokLessEqual: + return "<=" + case tokGreater: + return ">" + case tokGreaterEqual: + return ">=" + case tokLogicalAnd: + return "&&" + case tokLogicalOr: + return "||" + default: + return "" + } +} + +type token struct { + kind tokenKind + start int32 + end int32 +} + +type lexerError struct { + start int32 + end int32 + message string +} + +type lexerPosition struct { + pos int32 + atEnd bool + done bool + err lexerError +} + +var keywords = map[string]tokenKind{ + "false": tokFalse, + "true": tokTrue, + "null": tokNull, + "in": tokIn, + "as": tokReservedWord, + "break": tokReservedWord, + "const": tokReservedWord, + "continue": tokReservedWord, + "else": tokReservedWord, + "for": tokReservedWord, + "function": tokReservedWord, + "if": tokReservedWord, + "import": tokReservedWord, + "let": tokReservedWord, + "loop": tokReservedWord, + "package": tokReservedWord, + "namespace": tokReservedWord, + "return": tokReservedWord, + "var": tokReservedWord, + "void": tokReservedWord, + "while": tokReservedWord, +} + +func isIdentTrailing(r rune) bool { + return r <= 0x7f && ((r >= '0' && r <= '9') || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_') +} + +func isDigit(r rune) bool { + return r >= '0' && r <= '9' +} + +func isHexDigit(r rune) bool { + return (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') +} + +func isAlpha(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') +} + +func isPlusOrMinus(r rune) bool { + return r == '+' || r == '-' +} + +// lexer performs fast tokenization of CEL expression source code. +type lexer struct { + content runes.Buffer + length int32 + pos int32 + atEnd bool + done bool + err lexerError +} + +func newLexer(content runes.Buffer) *lexer { + return &lexer{ + content: content, + length: int32(content.Len()), + pos: 0, + } +} + +func (l *lexer) SavePosition() lexerPosition { + return lexerPosition{ + pos: l.pos, + atEnd: l.atEnd, + done: l.done, + err: l.err, + } +} + +func (l *lexer) RestorePosition(p lexerPosition) { + l.pos = p.pos + l.atEnd = p.atEnd + l.done = p.done + l.err = p.err +} + +func (l *lexer) GetError() lexerError { + return l.err +} + +func (l *lexer) GetPosition() int32 { + return l.pos +} + +func (l *lexer) makeToken(kind tokenKind, start, end int32) token { + if l.atEnd { + l.done = true + } + return token{kind: kind, start: start, end: end} +} + +func (l *lexer) setError(start, end int32, msg string) token { + l.err = lexerError{start: start, end: end, message: msg} + return token{kind: tokError, start: start, end: end} +} + +func (l *lexer) advance(n int32) { + l.pos += n +} + +func (l *lexer) match(c rune) bool { + return l.pos < l.length && l.content.Get(int(l.pos)) == c +} + +func (l *lexer) matchIgnoreCase(c rune) bool { + if l.pos >= l.length { + return false + } + cp := l.content.Get(int(l.pos)) + if cp <= 0x7f && c <= 0x7f { + cpLower := cp + if cp >= 'A' && cp <= 'Z' { + cpLower += 'a' - 'A' + } + cLower := c + if c >= 'A' && c <= 'Z' { + cLower += 'a' - 'A' + } + return cpLower == cLower + } + return cp == c +} + +func (l *lexer) consume(c rune) bool { + if l.match(c) { + l.advance(1) + return true + } + return false +} + +func (l *lexer) consumeIgnoreCase(c rune) bool { + if l.matchIgnoreCase(c) { + l.advance(1) + return true + } + return false +} + +func (l *lexer) consumeIf(predicate func(rune) bool) bool { + if l.pos < l.length && predicate(l.content.Get(int(l.pos))) { + l.advance(1) + return true + } + return false +} + +func (l *lexer) consumeLine() { + for l.pos < l.length { + if l.content.Get(int(l.pos)) == '\n' { + l.advance(1) + return + } + l.advance(1) + } +} + +func (l *lexer) consumeWhitespace() { + for l.pos < l.length { + c := l.content.Get(int(l.pos)) + switch c { + case '\f', '\n', ' ', '\r', '\v', '\t': + l.advance(1) + default: + return + } + } +} + +func (l *lexer) consumeDigits() bool { + advanced := false + for l.pos < l.length { + c := l.content.Get(int(l.pos)) + if !isDigit(c) { + break + } + l.advance(1) + advanced = true + } + return advanced +} + +func (l *lexer) consumeHexDigits() bool { + advanced := false + for l.pos < l.length { + c := l.content.Get(int(l.pos)) + if !isHexDigit(c) { + break + } + l.advance(1) + advanced = true + } + return advanced +} + +func (l *lexer) consumeIntegralSuffix() tokenKind { + if l.consumeIgnoreCase('u') { + return tokUint + } + return tokInt +} + +func (l *lexer) consumeUntilAfter(c rune) bool { + for pos := l.pos; pos < l.length; pos++ { + if l.content.Get(int(pos)) == c { + l.pos = pos + 1 + return true + } + } + l.pos = l.length + return false +} + +func (l *lexer) consumeUntilAfterTriple(quote rune) bool { + pos := l.pos + for pos+3 <= l.length { + if l.content.Get(int(pos)) == quote && + l.content.Get(int(pos+1)) == quote && + l.content.Get(int(pos+2)) == quote { + l.pos = pos + 3 + return true + } + pos++ + } + l.pos = l.length + return false +} + +func (l *lexer) consumeUntilAfterUnescaped(c rune) bool { + pos := l.pos + escaped := false + for pos < l.length { + cc := l.content.Get(int(pos)) + if cc == '\\' { + escaped = !escaped + } else { + if cc == c && !escaped { + l.pos = pos + 1 + return true + } + escaped = false + } + pos++ + } + l.pos = l.length + return false +} + +func (l *lexer) consumeUntilAfterUnescapedTriple(quote rune) bool { + pos := l.pos + escaped := false + for pos < l.length { + cc := l.content.Get(int(pos)) + if cc == '\\' { + escaped = !escaped + } else { + if !escaped && pos+3 <= l.length { + if l.content.Get(int(pos)) == quote && + l.content.Get(int(pos+1)) == quote && + l.content.Get(int(pos+2)) == quote { + l.pos = pos + 3 + return true + } + } + escaped = false + } + pos++ + } + l.pos = l.length + return false +} + +func (l *lexer) consumeQuotedIdent() token { + start := l.pos + l.advance(1) + if !l.consumeUntilAfter('`') { + return l.setError(start, l.pos, "unterminated quoted identifier") + } + return l.makeToken(tokIdent, start, l.pos) +} + +func (l *lexer) consumeStringLiteral(start int32, quote rune, isBytes, isRaw bool) token { + l.advance(1) + if l.pos+2 <= l.length && l.content.Get(int(l.pos)) == quote && l.content.Get(int(l.pos+1)) == quote { + l.advance(2) + var found bool + if isRaw { + found = l.consumeUntilAfterTriple(quote) + } else { + found = l.consumeUntilAfterUnescapedTriple(quote) + } + if !found { + msg := "unterminated string literal" + if isBytes { + msg = "unterminated bytes literal" + } + return l.setError(start, l.pos, msg) + } + kind := tokString + if isBytes { + kind = tokBytes + } + return l.makeToken(kind, start, l.pos) + } + var found bool + if isRaw { + found = l.consumeUntilAfter(quote) + } else { + found = l.consumeUntilAfterUnescaped(quote) + } + if !found { + msg := "unterminated string literal" + if isBytes { + msg = "unterminated bytes literal" + } + return l.setError(start, l.pos, msg) + } + kind := tokString + if isBytes { + kind = tokBytes + } + return l.makeToken(kind, start, l.pos) +} + +func (l *lexer) consumePrefixedStringLiteral() (token, bool) { + start := l.pos + if l.pos >= l.length { + return token{}, false + } + c := l.content.Get(int(l.pos)) + isBytes := (c == 'b' || c == 'B') + isRaw := (c == 'r' || c == 'R') + lookahead := int32(1) + if l.pos+1 < l.length { + c2 := l.content.Get(int(l.pos + 1)) + if (isBytes && (c2 == 'r' || c2 == 'R')) || (!isBytes && (c2 == 'b' || c2 == 'B')) { + isBytes = true + isRaw = true + lookahead = 2 + } + } + if l.pos+lookahead < l.length { + quote := l.content.Get(int(l.pos + lookahead)) + if quote == '"' || quote == '\'' { + l.advance(lookahead) + return l.consumeStringLiteral(start, quote, isBytes, isRaw), true + } + } + return token{}, false +} + +func (l *lexer) consumeNumericLiteral() token { + start := l.pos + c := l.content.Get(int(l.pos)) + floatingPoint := false + if c == '.' { + floatingPoint = true + l.advance(1) + if !l.consumeDigits() { + return l.setError(start, l.pos, "floating point literal missing digits after decimal separator") + } + } else { + l.advance(1) + if c == '0' { + if l.consumeIgnoreCase('x') { + if !l.consumeHexDigits() { + return l.setError(start, l.pos, "integral literal missing digits after hexadecimal separator") + } + tokType := l.consumeIntegralSuffix() + if l.consumeIf(isIdentTrailing) { + return l.setError(start, l.pos, fmt.Sprintf("%s literal has unexpected trailing characters", tokType)) + } + return l.makeToken(tokType, start, l.pos) + } + } + _ = l.consumeDigits() + if l.pos < l.length && l.content.Get(int(l.pos)) == '.' && + l.pos+1 < l.length && isDigit(l.content.Get(int(l.pos+1))) { + floatingPoint = true + l.advance(1) + _ = l.consumeDigits() + } + } + if l.consumeIgnoreCase('e') { + floatingPoint = true + _ = l.consumeIf(isPlusOrMinus) + if !l.consumeDigits() { + return l.setError(start, l.pos, "floating point literal missing digits after exponent separator") + } + } + var tokType tokenKind + if floatingPoint { + tokType = tokFloat + } else { + tokType = l.consumeIntegralSuffix() + } + if l.consumeIf(isIdentTrailing) { + return l.setError(start, l.pos, fmt.Sprintf("%s literal has unexpected trailing characters", tokType)) + } + return l.makeToken(tokType, start, l.pos) +} + +func (l *lexer) consumeIdent() token { + start := l.pos + for l.pos < l.length { + c := l.content.Get(int(l.pos)) + if !isIdentTrailing(c) { + break + } + l.advance(1) + } + end := l.pos + word := l.content.Slice(int(start), int(end)) + if kind, ok := keywords[word]; ok { + return l.makeToken(kind, start, end) + } + return l.makeToken(tokIdent, start, end) +} + +// Lex scans and returns the next token from the source. +func (l *lexer) Lex() token { + start := l.pos + if l.pos >= l.length { + l.atEnd = true + l.done = true + return l.makeToken(tokEnd, start, start) + } + c := l.content.Get(int(l.pos)) + switch c { + case '\f', '\v', '\t', '\r', '\n', ' ': + l.consumeWhitespace() + return l.makeToken(tokWhitespace, start, l.pos) + case '.': + if l.pos+1 < l.length && isDigit(l.content.Get(int(l.pos+1))) { + return l.consumeNumericLiteral() + } + l.advance(1) + return l.makeToken(tokDot, start, l.pos) + case ',': + l.advance(1) + return l.makeToken(tokComma, start, l.pos) + case '!': + l.advance(1) + if l.consume('=') { + return l.makeToken(tokExclamationEqual, start, l.pos) + } + return l.makeToken(tokExclamation, start, l.pos) + case '?': + l.advance(1) + return l.makeToken(tokQuestion, start, l.pos) + case '(': + l.advance(1) + return l.makeToken(tokLeftParen, start, l.pos) + case ')': + l.advance(1) + return l.makeToken(tokRightParen, start, l.pos) + case '{': + l.advance(1) + return l.makeToken(tokLeftBrace, start, l.pos) + case '}': + l.advance(1) + return l.makeToken(tokRightBrace, start, l.pos) + case '[': + l.advance(1) + return l.makeToken(tokLeftBracket, start, l.pos) + case ']': + l.advance(1) + return l.makeToken(tokRightBracket, start, l.pos) + case '=': + l.advance(1) + if l.consume('=') { + return l.makeToken(tokEqualEqual, start, l.pos) + } + return l.makeToken(tokEqual, start, l.pos) + case '<': + l.advance(1) + if l.consume('=') { + return l.makeToken(tokLessEqual, start, l.pos) + } + return l.makeToken(tokLess, start, l.pos) + case '>': + l.advance(1) + if l.consume('=') { + return l.makeToken(tokGreaterEqual, start, l.pos) + } + return l.makeToken(tokGreater, start, l.pos) + case ':': + l.advance(1) + return l.makeToken(tokColon, start, l.pos) + case '%': + l.advance(1) + return l.makeToken(tokPercent, start, l.pos) + case '+': + l.advance(1) + return l.makeToken(tokPlus, start, l.pos) + case '-': + l.advance(1) + return l.makeToken(tokMinus, start, l.pos) + case '*': + l.advance(1) + return l.makeToken(tokAsterisk, start, l.pos) + case '/': + l.advance(1) + if l.consume('/') { + l.consumeLine() + return l.makeToken(tokComment, start, l.pos) + } + return l.makeToken(tokSlash, start, l.pos) + case '&': + l.advance(1) + if l.consume('&') { + return l.makeToken(tokLogicalAnd, start, l.pos) + } + return l.setError(start, l.pos, "unexpected single '&', expected '&&'") + case '|': + l.advance(1) + if l.consume('|') { + return l.makeToken(tokLogicalOr, start, l.pos) + } + return l.setError(start, l.pos, "unexpected single '|', expected '||'") + case '_': + return l.consumeIdent() + case '`': + return l.consumeQuotedIdent() + case '\'': + return l.consumeStringLiteral(start, '\'', false, false) + case '"': + return l.consumeStringLiteral(start, '"', false, false) + case 'r', 'R', 'b', 'B': + if tok, ok := l.consumePrefixedStringLiteral(); ok { + return tok + } + } + if isDigit(c) { + return l.consumeNumericLiteral() + } + if isAlpha(c) { + return l.consumeIdent() + } + l.advance(1) + return l.setError(start, l.pos, "unexpected character") +} diff --git a/parser/lexer_test.go b/parser/lexer_test.go new file mode 100644 index 000000000..62b449ad5 --- /dev/null +++ b/parser/lexer_test.go @@ -0,0 +1,590 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "testing" + + "cel.dev/cel-go/common" + "cel.dev/cel-go/common/runes" +) + +type expectedToken struct { + kind tokenKind + text string +} + +func TestLexer(t *testing.T) { + tests := []struct { + name string + input string + expected []expectedToken + }{ + { + name: "Empty", + input: "", + expected: []expectedToken{}, + }, + { + name: "Whitespace", + input: " \n \t\r\f\v", + expected: []expectedToken{ + {kind: tokWhitespace, text: " \n \t\r\f\v"}, + }, + }, + { + name: "KeywordsAndIdents", + input: "null false true in as return foo_bar _foo_bar _ `quoted.ident`", + expected: []expectedToken{ + {kind: tokNull, text: "null"}, + {kind: tokWhitespace, text: " "}, + {kind: tokFalse, text: "false"}, + {kind: tokWhitespace, text: " "}, + {kind: tokTrue, text: "true"}, + {kind: tokWhitespace, text: " "}, + {kind: tokIn, text: "in"}, + {kind: tokWhitespace, text: " "}, + {kind: tokReservedWord, text: "as"}, + {kind: tokWhitespace, text: " "}, + {kind: tokReservedWord, text: "return"}, + {kind: tokWhitespace, text: " "}, + {kind: tokIdent, text: "foo_bar"}, + {kind: tokWhitespace, text: " "}, + {kind: tokIdent, text: "_foo_bar"}, + {kind: tokWhitespace, text: " "}, + {kind: tokIdent, text: "_"}, + {kind: tokWhitespace, text: " "}, + {kind: tokIdent, text: "`quoted.ident`"}, + }, + }, + { + name: "Numbers", + input: "123 45u 0x1A 3.14 .5 1e6 2.5e-3 45U 0x1Au 0x1AU", + expected: []expectedToken{ + {kind: tokInt, text: "123"}, + {kind: tokWhitespace, text: " "}, + {kind: tokUint, text: "45u"}, + {kind: tokWhitespace, text: " "}, + {kind: tokInt, text: "0x1A"}, + {kind: tokWhitespace, text: " "}, + {kind: tokFloat, text: "3.14"}, + {kind: tokWhitespace, text: " "}, + {kind: tokFloat, text: ".5"}, + {kind: tokWhitespace, text: " "}, + {kind: tokFloat, text: "1e6"}, + {kind: tokWhitespace, text: " "}, + {kind: tokFloat, text: "2.5e-3"}, + {kind: tokWhitespace, text: " "}, + {kind: tokUint, text: "45U"}, + {kind: tokWhitespace, text: " "}, + {kind: tokUint, text: "0x1Au"}, + {kind: tokWhitespace, text: " "}, + {kind: tokUint, text: "0x1AU"}, + }, + }, + { + name: "IntEOF", + input: "123456", + expected: []expectedToken{ + {kind: tokInt, text: "123456"}, + }, + }, + { + name: "HexIntEOF", + input: "0x1A2B", + expected: []expectedToken{ + {kind: tokInt, text: "0x1A2B"}, + }, + }, + { + name: "FloatPositiveExponentEOF", + input: "1e+6", + expected: []expectedToken{ + {kind: tokFloat, text: "1e+6"}, + }, + }, + { + name: "FloatEOF", + input: ".12345", + expected: []expectedToken{ + {kind: tokFloat, text: ".12345"}, + }, + }, + { + name: "IntDotIdent", + input: "1.foo", + expected: []expectedToken{ + {kind: tokInt, text: "1"}, + {kind: tokDot, text: "."}, + {kind: tokIdent, text: "foo"}, + }, + }, + { + name: "IntDotWhitespace", + input: "1. ", + expected: []expectedToken{ + {kind: tokInt, text: "1"}, + {kind: tokDot, text: "."}, + {kind: tokWhitespace, text: " "}, + }, + }, + { + name: "IntDotEOF", + input: "1.", + expected: []expectedToken{ + {kind: tokInt, text: "1"}, + {kind: tokDot, text: "."}, + }, + }, + { + name: "ZeroNumbers", + input: "0 0u 0x0", + expected: []expectedToken{ + {kind: tokInt, text: "0"}, + {kind: tokWhitespace, text: " "}, + {kind: tokUint, text: "0u"}, + {kind: tokWhitespace, text: " "}, + {kind: tokInt, text: "0x0"}, + }, + }, + { + name: "StringsAndBytes", + input: "\"hello\" 'world' \"\"\" \"allowed!\" \"\"also allowed\"\" \\\"\"\"also allowed\"\"\\\" \"\"\" r\"raw\" b\"bytes\" rb'\\x00' '''multi\nsingle''' R\"raw_upper\" B\"bytes_upper\" b'''multi\nbytes''' br\"raw_bytes\" `a.b-c/d e`\n\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\\` \\x1A \\u00A0 \\U0001F600 \\012\"", + expected: []expectedToken{ + {kind: tokString, text: "\"hello\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "'world'"}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "\"\"\" \"allowed!\" \"\"also allowed\"\" \\\"\"\"also allowed\"\"\\\" \"\"\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "r\"raw\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "b\"bytes\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "rb'\\x00'"}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "'''multi\nsingle'''"}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "R\"raw_upper\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "B\"bytes_upper\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "b'''multi\nbytes'''"}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "br\"raw_bytes\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokIdent, text: "`a.b-c/d e`"}, + {kind: tokWhitespace, text: "\n"}, + {kind: tokString, text: "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\\` \\x1A \\u00A0 \\U0001F600 \\012\""}, + }, + }, + { + name: "EmptyStrings", + input: "\"\" '' \"\"\"\"\"\" '''''' r\"\" r'' r\"\"\"\"\"\" r'''''' b\"\" b'' b\"\"\"\"\"\" b''''''", + expected: []expectedToken{ + {kind: tokString, text: "\"\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "''"}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "\"\"\"\"\"\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "''''''"}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "r\"\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "r''"}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "r\"\"\"\"\"\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokString, text: "r''''''"}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "b\"\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "b''"}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "b\"\"\"\"\"\""}, + {kind: tokWhitespace, text: " "}, + {kind: tokBytes, text: "b''''''"}, + }, + }, + { + name: "OperatorsAndDelimiters", + input: ". , + - * / % == != < <= > >= && || ! ? : [] { } ( )", + expected: []expectedToken{ + {kind: tokDot, text: "."}, + {kind: tokWhitespace, text: " "}, + {kind: tokComma, text: ","}, + {kind: tokWhitespace, text: " "}, + {kind: tokPlus, text: "+"}, + {kind: tokWhitespace, text: " "}, + {kind: tokMinus, text: "-"}, + {kind: tokWhitespace, text: " "}, + {kind: tokAsterisk, text: "*"}, + {kind: tokWhitespace, text: " "}, + {kind: tokSlash, text: "/"}, + {kind: tokWhitespace, text: " "}, + {kind: tokPercent, text: "%"}, + {kind: tokWhitespace, text: " "}, + {kind: tokEqualEqual, text: "=="}, + {kind: tokWhitespace, text: " "}, + {kind: tokExclamationEqual, text: "!="}, + {kind: tokWhitespace, text: " "}, + {kind: tokLess, text: "<"}, + {kind: tokWhitespace, text: " "}, + {kind: tokLessEqual, text: "<="}, + {kind: tokWhitespace, text: " "}, + {kind: tokGreater, text: ">"}, + {kind: tokWhitespace, text: " "}, + {kind: tokGreaterEqual, text: ">="}, + {kind: tokWhitespace, text: " "}, + {kind: tokLogicalAnd, text: "&&"}, + {kind: tokWhitespace, text: " "}, + {kind: tokLogicalOr, text: "||"}, + {kind: tokWhitespace, text: " "}, + {kind: tokExclamation, text: "!"}, + {kind: tokWhitespace, text: " "}, + {kind: tokQuestion, text: "?"}, + {kind: tokWhitespace, text: " "}, + {kind: tokColon, text: ":"}, + {kind: tokWhitespace, text: " "}, + {kind: tokLeftBracket, text: "["}, + {kind: tokRightBracket, text: "]"}, + {kind: tokWhitespace, text: " "}, + {kind: tokLeftBrace, text: "{"}, + {kind: tokWhitespace, text: " "}, + {kind: tokRightBrace, text: "}"}, + {kind: tokWhitespace, text: " "}, + {kind: tokLeftParen, text: "("}, + {kind: tokWhitespace, text: " "}, + {kind: tokRightParen, text: ")"}, + }, + }, + { + name: "Comments", + input: "a\n// comment\nb", + expected: []expectedToken{ + {kind: tokIdent, text: "a"}, + {kind: tokWhitespace, text: "\n"}, + {kind: tokComment, text: "// comment\n"}, + {kind: tokIdent, text: "b"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + buf := runes.NewBuffer(tc.input) + lexer := newLexer(buf) + var tokens []expectedToken + for { + tok := lexer.Lex() + if tok.kind == tokEnd { + break + } + if tok.kind == tokError { + t.Fatalf("unexpected error token: %v (%s)", tok, lexer.GetError().message) + } + text := buf.Slice(int(tok.start), int(tok.end)) + tokens = append(tokens, expectedToken{kind: tok.kind, text: text}) + } + + if len(tokens) != len(tc.expected) { + t.Fatalf("got %d tokens, expected %d\ngot: %+v\nwant: %+v", len(tokens), len(tc.expected), tokens, tc.expected) + } + for i, exp := range tc.expected { + got := tokens[i] + if got.kind != exp.kind || got.text != exp.text { + t.Errorf("token[%d] = {kind: %v, text: %q}, want {kind: %v, text: %q}", i, got.kind, got.text, exp.kind, exp.text) + } + } + }) + } +} + +func TestLexerErrors(t *testing.T) { + tests := []struct { + name string + input string + expectedError string + }{ + { + name: "UnterminatedString", + input: "\"unterminated", + expectedError: "ERROR: :1:1: unterminated string literal\n" + + " | \"unterminated\n" + + " | ^", + }, + { + name: "HexMissingDigits", + input: "0x", + expectedError: "ERROR: :1:1: integral literal missing digits after hexadecimal separator\n" + + " | 0x\n" + + " | ^", + }, + { + name: "UnexpectedChar", + input: "@", + expectedError: "ERROR: :1:1: unexpected character\n" + + " | @\n" + + " | ^", + }, + { + name: "HexInvalidTrailing", + input: "0x1A_invalid", + expectedError: "ERROR: :1:1: int literal has unexpected trailing characters\n" + + " | 0x1A_invalid\n" + + " | ^", + }, + { + name: "IntInvalidTrailing", + input: "123_invalid", + expectedError: "ERROR: :1:1: int literal has unexpected trailing characters\n" + + " | 123_invalid\n" + + " | ^", + }, + { + name: "Int1x0", + input: "1x0", + expectedError: "ERROR: :1:1: int literal has unexpected trailing characters\n" + + " | 1x0\n" + + " | ^", + }, + { + name: "Int2x", + input: "2x", + expectedError: "ERROR: :1:1: int literal has unexpected trailing characters\n" + + " | 2x\n" + + " | ^", + }, + { + name: "UnterminatedQuotedIdent", + input: "`unterminated quoted", + expectedError: "ERROR: :1:1: unterminated quoted identifier\n" + + " | `unterminated quoted\n" + + " | ^", + }, + { + name: "UnterminatedMultiString", + input: "'''unterminated multi", + expectedError: "ERROR: :1:1: unterminated string literal\n" + + " | '''unterminated multi\n" + + " | ^", + }, + { + name: "UnterminatedRawString", + input: "r'unterminated raw", + expectedError: "ERROR: :1:1: unterminated string literal\n" + + " | r'unterminated raw\n" + + " | ^", + }, + { + name: "UnterminatedBytes", + input: "b'unterminated bytes", + expectedError: "ERROR: :1:1: unterminated bytes literal\n" + + " | b'unterminated bytes\n" + + " | ^", + }, + { + name: "ExponentMissingDigits", + input: "1e", + expectedError: "ERROR: :1:1: floating point literal missing digits after exponent separator\n" + + " | 1e\n" + + " | ^", + }, + { + name: "SingleAmpersand", + input: "&", + expectedError: "ERROR: :1:1: unexpected single '&', expected '&&'\n" + + " | &\n" + + " | ^", + }, + { + name: "SinglePipe", + input: "|", + expectedError: "ERROR: :1:1: unexpected single '|', expected '||'\n" + + " | |\n" + + " | ^", + }, + { + name: "EmojiUnexpectedChar", + input: "\"😀😀😀😀😀\" ~error", + expectedError: "ERROR: :1:9: unexpected character\n" + + " | \"😀😀😀😀😀\" ~error\n" + + " | .．．．．．..^", + }, + { + name: "MultiLineEmojiUnexpectedChar", + input: "\"😀😀\"\n ~error", + expectedError: "ERROR: :2:3: unexpected character\n" + + " | ~error\n" + + " | ..^", + }, + { + name: "UnicodeCJKFollowingExponentError", + input: "\"𠮷野家\" 1e", + expectedError: "ERROR: :1:7: floating point literal missing digits after exponent separator\n" + + " | \"𠮷野家\" 1e\n" + + " | .．．．..^", + }, + { + name: "SupplementaryPlaneQuotedIdentFollowingError", + input: "`𠮷_ident_🚀` @", + expectedError: "ERROR: :1:13: unexpected character\n" + + " | `𠮷_ident_🚀` @\n" + + " | .．.......．..^", + }, + { + name: "EmojiInStringErrorRecovery", + input: "\"✨🌟⭐\" @ 42", + expectedError: "ERROR: :1:7: unexpected character\n" + + " | \"✨🌟⭐\" @ 42\n" + + " | .．．．..^", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + buf := runes.NewBuffer(tc.input) + lexer := newLexer(buf) + var tok token + for { + tok = lexer.Lex() + if tok.kind == tokError || tok.kind == tokEnd { + break + } + } + if tok.kind != tokError { + t.Fatalf("expected tokError for input %q, got %v", tc.input, tok.kind) + } + + // Format error message using standard common.Error and common.Source + textSource := common.NewTextSource(tc.input) + loc, ok := textSource.OffsetLocation(tok.start) + if !ok { + t.Fatalf("textSource.OffsetLocation(%d) not found", tok.start) + } + err := common.NewError(0, lexer.GetError().message, loc) + gotDisplay := err.ToDisplayString(textSource) + + if gotDisplay != tc.expectedError { + t.Errorf("got error display:\n%s\n\nwant error display:\n%s", gotDisplay, tc.expectedError) + } + }) + } +} + +func TestLexerPositionSaveRestore(t *testing.T) { + buf := runes.NewBuffer("foo + bar * 42") + lexer := newLexer(buf) + + tok1 := lexer.Lex() + if tok1.kind != tokIdent { + t.Fatalf("tok1 = %v, want tokIdent", tok1.kind) + } + + tok2 := lexer.Lex() + if tok2.kind != tokWhitespace { + t.Fatalf("tok2 = %v, want tokWhitespace", tok2.kind) + } + + // Save position before '+' + saved := lexer.SavePosition() + + tok3 := lexer.Lex() + if tok3.kind != tokPlus { + t.Fatalf("tok3 = %v, want tokPlus", tok3.kind) + } + + tok4 := lexer.Lex() + if tok4.kind != tokWhitespace { + t.Fatalf("tok4 = %v, want tokWhitespace", tok4.kind) + } + + tok5 := lexer.Lex() + if tok5.kind != tokIdent { + t.Fatalf("tok5 = %v, want tokIdent", tok5.kind) + } + + // Restore position to before '+' + lexer.RestorePosition(saved) + + tok3Restored := lexer.Lex() + if tok3Restored.kind != tokPlus || tok3Restored.start != tok3.start || tok3Restored.end != tok3.end { + t.Errorf("tok3Restored = %v (%d, %d), want tokPlus (%d, %d)", tok3Restored.kind, tok3Restored.start, tok3Restored.end, tok3.start, tok3.end) + } + + tok4Restored := lexer.Lex() + if tok4Restored.kind != tokWhitespace { + t.Errorf("tok4Restored = %v, want tokWhitespace", tok4Restored.kind) + } + + tok5Restored := lexer.Lex() + if tok5Restored.kind != tokIdent { + t.Errorf("tok5Restored = %v, want tokIdent", tok5Restored.kind) + } +} + +func TestLexerErrorRecovery(t *testing.T) { + buf := runes.NewBuffer("1e, {2 3}") + lexer := newLexer(buf) + + tok := lexer.Lex() + if tok.kind != tokError { + t.Fatalf("tok = %v, want tokError", tok.kind) + } + if lexer.GetError().message != "floating point literal missing digits after exponent separator" { + t.Errorf("got error message %q", lexer.GetError().message) + } + + tok = lexer.Lex() + if tok.kind != tokComma { + t.Errorf("tok = %v, want tokComma", tok.kind) + } + + tok = lexer.Lex() + if tok.kind != tokWhitespace { + t.Errorf("tok = %v, want tokWhitespace", tok.kind) + } + + tok = lexer.Lex() + if tok.kind != tokLeftBrace { + t.Errorf("tok = %v, want tokLeftBrace", tok.kind) + } + + tok = lexer.Lex() + if tok.kind != tokInt { + t.Errorf("tok = %v, want tokInt", tok.kind) + } + if tok.start != 5 || tok.end != 6 { + t.Errorf("tok position = (%d, %d), want (5, 6)", tok.start, tok.end) + } +} + +func BenchmarkLexer(b *testing.B) { + expr := `a > 5 && b < 10 || c == "xyz" + 42u - 3.14 * (foo.bar(1, 2, [3, ?4]))` + buf := runes.NewBuffer(expr) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + lexer := newLexer(buf) + for { + tok := lexer.Lex() + if tok.kind == tokEnd || tok.kind == tokError { + break + } + } + } +} + diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go new file mode 100644 index 000000000..4a22fb9a3 --- /dev/null +++ b/parser/pratt_parser.go @@ -0,0 +1,1012 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "fmt" + "math" + "strconv" + "strings" + + "cel.dev/cel-go/common" + "cel.dev/cel-go/common/ast" + "cel.dev/cel-go/common/operators" + "cel.dev/cel-go/common/runes" + "cel.dev/cel-go/common/types" +) + +type binaryOpInfo struct { + precedence int + name string + kind tokenKind +} + +var ( + opLogicalOr = binaryOpInfo{precedence: 1, name: operators.LogicalOr, kind: tokLogicalOr} + opLogicalAnd = binaryOpInfo{precedence: 2, name: operators.LogicalAnd, kind: tokLogicalAnd} + opLess = binaryOpInfo{precedence: 3, name: operators.Less, kind: tokLess} + opLessEqual = binaryOpInfo{precedence: 3, name: operators.LessEquals, kind: tokLessEqual} + opGreater = binaryOpInfo{precedence: 3, name: operators.Greater, kind: tokGreater} + opGreaterEqual = binaryOpInfo{precedence: 3, name: operators.GreaterEquals, kind: tokGreaterEqual} + opEqualEqual = binaryOpInfo{precedence: 3, name: operators.Equals, kind: tokEqualEqual} + opExclamationEqual = binaryOpInfo{precedence: 3, name: operators.NotEquals, kind: tokExclamationEqual} + opIn = binaryOpInfo{precedence: 3, name: operators.In, kind: tokIn} + opPlus = binaryOpInfo{precedence: 4, name: operators.Add, kind: tokPlus} + opMinus = binaryOpInfo{precedence: 4, name: operators.Subtract, kind: tokMinus} + opAsterisk = binaryOpInfo{precedence: 5, name: operators.Multiply, kind: tokAsterisk} + opSlash = binaryOpInfo{precedence: 5, name: operators.Divide, kind: tokSlash} + opPercent = binaryOpInfo{precedence: 5, name: operators.Modulo, kind: tokPercent} + opDefault = binaryOpInfo{precedence: 0, name: "", kind: tokError} +) + +func getBinaryOpInfo(kind tokenKind) binaryOpInfo { + switch kind { + case tokLogicalOr: + return opLogicalOr + case tokLogicalAnd: + return opLogicalAnd + case tokLess: + return opLess + case tokLessEqual: + return opLessEqual + case tokGreater: + return opGreater + case tokGreaterEqual: + return opGreaterEqual + case tokEqualEqual: + return opEqualEqual + case tokExclamationEqual: + return opExclamationEqual + case tokIn: + return opIn + case tokPlus: + return opPlus + case tokMinus: + return opMinus + case tokAsterisk: + return opAsterisk + case tokSlash: + return opSlash + case tokPercent: + return opPercent + default: + return opDefault + } +} + +type prattParser struct { + content runes.Buffer + length int32 + helper *parserHelper + errors *parseErrors + exprFactory ast.ExprFactory + lexer *lexer + currTok token + peekTok token + macros map[string]Macro + recursionDepth int + recursionLimitExceeded bool + errorCount int + maxRecursionDepth int + maxExpressionNodeCount int + errorReportingLimit int + errorRecoveryLimit int + populateMacroCalls bool + enableOptionalSyntax bool + enableVariadicOperatorASTs bool + enableIdentEscapeSyntax bool +} + +// PrattParser encapsulates the context necessary to perform Pratt parsing for different expressions. +type PrattParser struct { + options +} + +// NewPrattParser builds and returns a new PrattParser using the provided options. +func NewPrattParser(opts ...Option) (*PrattParser, error) { + p := &PrattParser{} + p.enableHiddenAccumulatorName = true + p.enableIdentEscapeSyntax = true + for _, opt := range opts { + if err := opt(&p.options); err != nil { + return nil, err + } + } + if p.errorReportingLimit == 0 { + p.errorReportingLimit = 100 + } + if p.maxRecursionDepth == 0 { + p.maxRecursionDepth = 250 + } + if p.maxRecursionDepth == -1 { + p.maxRecursionDepth = math.MaxInt + } + if p.errorRecoveryLimit == 0 { + p.errorRecoveryLimit = 30 + } + if p.errorRecoveryLimit == -1 { + p.errorRecoveryLimit = math.MaxInt + } + if p.expressionSizeCodePointLimit == 0 { + p.expressionSizeCodePointLimit = 100_000 + } + if p.expressionSizeCodePointLimit == -1 { + p.expressionSizeCodePointLimit = math.MaxInt + } + if p.maxExpressionNodeCount == 0 { + p.maxExpressionNodeCount = 100_000 + } + if p.maxExpressionNodeCount == -1 { + p.maxExpressionNodeCount = math.MaxInt + } + return p, nil +} + +// Parse parses the expression represented by source using the Pratt parser and returns the result. +func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { + errs := common.NewErrors(source) + pratt := p.newWorker(source, errs) + var out ast.Expr + if pratt.length > int32(p.expressionSizeCodePointLimit) { + out = pratt.reportError(token{kind: tokError, start: 0, end: 0}, + "expression code point size exceeds limit: size: %d, limit %d", + pratt.length, p.expressionSizeCodePointLimit) + } else { + out = pratt.parse() + } + if len(errs.GetErrors()) > 0 { + return nil, errs + } + return ast.NewAST(out, pratt.helper.getSourceInfo()), errs +} + +func (p *PrattParser) newWorker(source common.Source, errs *common.Errors) *prattParser { + buf, ok := source.(runes.Buffer) + if !ok { + buf = runes.NewBuffer(source.Content()) + } + accu := AccumulatorName + if p.enableHiddenAccumulatorName { + accu = HiddenAccumulatorName + } + fac := ast.NewExprFactoryWithAccumulator(accu) + pp := &prattParser{ + content: buf, + length: int32(buf.Len()), + helper: newParserHelper(source, fac), + errors: &parseErrors{errs}, + exprFactory: fac, + lexer: newLexer(buf), + macros: p.macros, + maxRecursionDepth: p.maxRecursionDepth, + maxExpressionNodeCount: p.maxExpressionNodeCount, + errorReportingLimit: p.errorReportingLimit, + errorRecoveryLimit: p.errorRecoveryLimit, + populateMacroCalls: p.populateMacroCalls, + enableOptionalSyntax: p.enableOptionalSyntax, + enableVariadicOperatorASTs: p.enableVariadicOperatorASTs, + enableIdentEscapeSyntax: p.enableIdentEscapeSyntax, + } + pp.initTokenStream() + return pp +} + +func (p *prattParser) initTokenStream() { + p.currTok = token{kind: tokError, start: 0, end: 0} + p.peekTok = p.nextSignificantToken(true) +} + +func (p *prattParser) isRecoveryLimitExceeded() bool { + return p.errorCount > p.errorRecoveryLimit +} + +func (p *prattParser) nextSignificantToken(reportError bool) token { + if p.isRecoveryLimitExceeded() { + return token{kind: tokEnd, start: p.length, end: p.length} + } + for { + tok := p.lexer.Lex() + if tok.kind == tokWhitespace || tok.kind == tokComment { + continue + } + if tok.kind == tokError && reportError { + p.reportError(tok, "%s", p.lexer.GetError().message) + if p.isRecoveryLimitExceeded() { + return token{kind: tokEnd, start: p.length, end: p.length} + } + } + return tok + } +} + +func (p *prattParser) nextToken() token { + p.currTok = p.peekTok + if p.isRecoveryLimitExceeded() { + p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} + return p.currTok + } + if p.peekTok.kind != tokEnd { + p.peekTok = p.nextSignificantToken(true) + } + return p.currTok +} + +func (p *prattParser) tokenText(tok token) string { + if tok.start >= 0 && tok.end >= tok.start && tok.end <= p.length { + return p.content.Slice(int(tok.start), int(tok.end)) + } + return "" +} + +func (p *prattParser) nextID(tok token) int64 { + return p.helper.idFromOffsets(tok.start, tok.end) +} + +func (p *prattParser) expect(kind tokenKind, msg string) bool { + if p.peekTok.kind == kind { + p.nextToken() + return true + } + if p.isRecoveryLimitExceeded() { + return false + } + if p.peekTok.kind != tokError { + if msg == "" { + tokText := p.tokenText(p.peekTok) + formattedTok := fmt.Sprintf("'%s'", tokText) + if p.peekTok.kind == tokEnd { + formattedTok = "" + } + msg = fmt.Sprintf("Syntax error: mismatched input %s expecting '%s'", formattedTok, kind.String()) + } + p.reportError(p.peekTok, "%s", msg) + } + p.synchronizeOnDelimiter() + return false +} + +func (p *prattParser) synchronizeOnDelimiter() { + if p.isRecoveryLimitExceeded() { + p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} + return + } + for p.peekTok.kind != tokEnd { + if p.peekTok.kind == tokComma || + p.peekTok.kind == tokRightParen || + p.peekTok.kind == tokRightBracket || + p.peekTok.kind == tokRightBrace { + break + } + p.nextToken() + } +} + +func (p *prattParser) reportError(ctx any, format string, args ...any) ast.Expr { + if p.errorCount > p.errorRecoveryLimit { + return p.helper.newExpr(common.NoLocation) + } + p.errorCount++ + var location common.Location + err := p.helper.newExpr(ctx) + switch c := ctx.(type) { + case common.Location: + location = c + case token: + location = p.helper.getLocation(err.ID()) + default: + location = p.helper.getLocation(err.ID()) + } + if p.errorCount == p.errorRecoveryLimit+1 { + p.errors.syntaxError(location, fmt.Sprintf("error recovery attempt limit exceeded: %d", p.errorRecoveryLimit)) + p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} + return err + } + if p.errorCount <= p.errorReportingLimit { + p.errors.reportErrorAtID(err.ID(), location, format, args...) + } + return err +} + +func (p *prattParser) newLogicManager(function string, term ast.Expr) *logicManager { + if p.enableVariadicOperatorASTs { + return newVariadicLogicManager(p.exprFactory, function, term) + } + return newBalancingLogicManager(p.exprFactory, function, term) +} + +func (p *prattParser) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { + if expr, found := p.expandMacro(exprID, function, nil, args...); found { + return expr + } + return p.helper.newGlobalCall(exprID, function, args...) +} + +func (p *prattParser) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { + if expr, found := p.expandMacro(exprID, function, target, args...); found { + return expr + } + return p.helper.newReceiverCall(exprID, function, target, args...) +} + +func (p *prattParser) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { + if len(p.macros) == 0 { + return nil, false + } + macro, found := p.macros[makeMacroKey(function, len(args), target != nil)] + if !found { + macro, found = p.macros[makeVarArgMacroKey(function, target != nil)] + if !found { + return nil, false + } + } + if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { + loc := p.helper.getLocation(exprID) + p.helper.deleteID(exprID) + return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true + } + eh := exprHelperPool.Get().(*exprHelper) + eh.parserHelper = p.helper + eh.id = exprID + expr, err := macro.Expander()(eh, target, args) + exprHelperPool.Put(eh) + if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { + loc := p.helper.getLocation(exprID) + p.helper.deleteID(exprID) + return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true + } + if err != nil { + loc := err.Location + if loc == nil { + loc = p.helper.getLocation(exprID) + } + p.helper.deleteID(exprID) + return p.reportError(loc, "%s", err.Message), true + } + if expr == nil { + return nil, false + } + if p.populateMacroCalls { + p.helper.addMacroCall(expr.ID(), function, target, args...) + } + p.helper.deleteID(exprID) + return expr, true +} + +func (p *prattParser) normalizeIdent(tok token, allowQuoted bool) string { + text := p.tokenText(tok) + if len(text) == 0 { + return "" + } + if text[0] == '`' { + if !allowQuoted { + p.reportError(tok, "unexpected quoted identifier") + return "" + } + if !p.enableIdentEscapeSyntax { + p.reportError(tok, "unsupported syntax '`'") + } + if len(text) < 2 || text[len(text)-1] != '`' { + p.reportError(tok, "unterminated quoted identifier") + return "" + } + // Validate the quoted identifier syntax: + // ESC_IDENTIFIER : '`' (LETTER | DIGIT | '_' | '.' | '-' | '/' | ' ')+ '`'; + inner := text[1 : len(text)-1] + if len(inner) == 0 { + p.reportError(tok, "unexpected quoted identifier") + return "" + } + for _, c := range inner { + if !isAlpha(c) && !isDigit(c) && c != '_' && c != '.' && c != '-' && c != '/' && c != ' ' { + p.reportError(tok, "unexpected quoted identifier") + return "" + } + } + return inner + } + return text +} + +func (p *prattParser) parse() ast.Expr { + expr := p.parseExpr() + if p.recursionLimitExceeded || p.isRecoveryLimitExceeded() { + return expr + } + if p.peekTok.kind != tokEnd && p.peekTok.kind != tokError { + p.reportError(p.peekTok, "Syntax error: mismatched input '%s' expecting ", p.tokenText(p.peekTok)) + } + return expr +} + +func (p *prattParser) parseExpr() ast.Expr { + if p.recursionLimitExceeded || p.isRecoveryLimitExceeded() { + return p.helper.newExpr(common.NoLocation) + } + if p.recursionDepth > p.maxRecursionDepth { + p.recursionLimitExceeded = true + p.errors.internalError(fmt.Sprintf("expression recursion limit exceeded: %d", p.maxRecursionDepth)) + return p.helper.newExpr(common.NoLocation) + } + p.recursionDepth++ + expr := p.parseBinaryAndTernary(0) + p.recursionDepth-- + return expr +} + +func (p *prattParser) parseBinaryAndTernary(minPrec int) ast.Expr { + lhs := p.parseSelectorChain() + for { + tok := p.peekTok.kind + if tok == tokQuestion && minPrec <= 0 { + lhs = p.parseTernary(lhs) + continue + } + + opInfo := getBinaryOpInfo(tok) + if opInfo.kind == tokError || opInfo.precedence < minPrec { + break + } + + if opInfo.name == operators.LogicalOr || opInfo.name == operators.LogicalAnd { + lhs = p.parseBalancedLogicalChain(lhs, opInfo) + continue + } + + opTok := p.nextToken() + opID := p.nextID(opTok) + rhs := p.parseBinaryAndTernary(opInfo.precedence + 1) + lhs = p.helper.newGlobalCall(opID, opInfo.name, lhs, rhs) + } + return lhs +} + +func (p *prattParser) parseTernary(lhs ast.Expr) ast.Expr { + qTok := p.nextToken() + opID := p.nextID(qTok) + trueExpr := p.parseBinaryAndTernary(1) + if !p.expect(tokColon, "expected ':' in conditional expression") { + return lhs + } + falseExpr := p.parseBinaryAndTernary(0) + return p.helper.newGlobalCall(opID, operators.Conditional, lhs, trueExpr, falseExpr) +} + +func (p *prattParser) parseBalancedLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.Expr { + l := p.newLogicManager(opInfo.name, lhs) + for p.peekTok.kind == opInfo.kind { + opTok := p.nextToken() + opID := p.nextID(opTok) + rhs := p.parseBinaryAndTernary(opInfo.precedence + 1) + l.addTerm(opID, rhs) + } + return l.toExpr() +} + +func (p *prattParser) parseSelectorChain() ast.Expr { + lhs := p.parseUnary() + return p.parseSelectorChainTail(lhs) +} + +func (p *prattParser) parseSelectorChainTail(lhs ast.Expr) ast.Expr { + for { + switch p.peekTok.kind { + case tokDot: + dotTok := p.nextToken() + optional := false + if p.peekTok.kind == tokQuestion { + p.nextToken() + optional = true + if !p.enableOptionalSyntax { + p.reportError(dotTok, "unsupported syntax '.?'") + } + } + fieldTok := p.nextToken() + if fieldTok.kind != tokIdent && fieldTok.kind != tokReservedWord { + if fieldTok.kind != tokError { + p.reportError(fieldTok, "expected identifier after '.'") + } + p.synchronizeOnDelimiter() + return lhs + } + isMemberCall := p.peekTok.kind == tokLeftParen + field := p.normalizeIdent(fieldTok, !isMemberCall) + if optional { + opID := p.nextID(dotTok) + fieldID := p.nextID(fieldTok) + lhs = p.helper.newGlobalCall(opID, operators.OptSelect, lhs, p.helper.newLiteralString(fieldID, field)) + } else if isMemberCall { + lparen := p.nextToken() + callID := p.nextID(lparen) + args := p.parseArguments(tokRightParen) + lhs = p.receiverCallOrMacro(callID, field, lhs, args...) + } else { + dotID := p.nextID(dotTok) + lhs = p.helper.newSelect(dotID, lhs, field) + } + case tokLeftBracket: + bracketTok := p.nextToken() + opID := p.nextID(bracketTok) + optional := false + if p.peekTok.kind == tokQuestion { + p.nextToken() + optional = true + if !p.enableOptionalSyntax { + p.reportError(bracketTok, "unsupported syntax '?'") + } + } + index := p.parseExpr() + p.expect(tokRightBracket, "expected ']'") + opName := operators.Index + if optional { + opName = operators.OptIndex + } + lhs = p.helper.newGlobalCall(opID, opName, lhs, index) + case tokLeftBrace: + if structName, ok := p.extractStructName(lhs); ok { + lhs = p.parseStruct(lhs.ID(), structName) + } else { + return lhs + } + default: + return lhs + } + } +} + +func (p *prattParser) extractStructName(expr ast.Expr) (string, bool) { + if expr == nil || expr.Kind() == ast.LiteralKind { + return "", false + } + if expr.Kind() == ast.IdentKind { + name := expr.AsIdent() + p.helper.deleteID(expr.ID()) + return name, true + } + if expr.Kind() == ast.SelectKind { + sel := expr.AsSelect() + if sel.IsTestOnly() { + return "", false + } + prefix, ok := p.extractStructName(sel.Operand()) + if !ok { + return "", false + } + p.helper.deleteID(expr.ID()) + return prefix + "." + sel.FieldName(), true + } + return "", false +} + +func (p *prattParser) parseStruct(objID int64, structName string) ast.Expr { + p.nextToken() // consumes { + var fields []ast.EntryExpr + for p.peekTok.kind != tokRightBrace && p.peekTok.kind != tokEnd { + optional := false + if p.peekTok.kind == tokQuestion { + q := p.nextToken() + optional = true + if !p.enableOptionalSyntax { + p.reportError(q, "unsupported syntax '?'") + } + } + fieldTok := p.nextToken() + if fieldTok.kind != tokIdent && fieldTok.kind != tokReservedWord { + p.reportError(fieldTok, "expected struct field name") + p.synchronizeOnDelimiter() + break + } + fieldName := p.normalizeIdent(fieldTok, true) + colonTok := p.peekTok + if !p.expect(tokColon, "expected ':' in struct field") { + break + } + fieldID := p.nextID(colonTok) + val := p.parseExpr() + fields = append(fields, p.helper.newObjectField(fieldID, fieldName, val, optional)) + if p.peekTok.kind == tokComma { + p.nextToken() + } else { + break + } + } + p.expect(tokRightBrace, "expected '}'") + return p.helper.newObject(objID, structName, fields...) +} + +func (p *prattParser) parseUnary() ast.Expr { + tok := p.peekTok.kind + if tok == tokExclamation || tok == tokMinus { + return p.parseUnaryOps() + } + return p.parsePrimary() +} + +func (p *prattParser) parseUnaryOps() ast.Expr { + op := p.nextToken() + if p.peekTok.kind == tokExclamation || p.peekTok.kind == tokMinus { + return p.parseUnaryOpsChain(op) + } + + if op.kind == tokMinus { + if p.peekTok.kind == tokInt { + return p.parseNegativeIntLiteral(p.nextID(op)) + } + if p.peekTok.kind == tokFloat { + return p.parseNegativeDoubleLiteral(p.nextID(op)) + } + } + + opID := p.nextID(op) + operand := p.parseSelectorChain() + opName := operators.LogicalNot + if op.kind == tokMinus { + opName = operators.Negate + } + return p.globalCallOrMacro(opID, opName, operand) +} + +func (p *prattParser) parseUnaryOpsChain(firstOp token) ast.Expr { + type unaryOpInfo struct { + kind tokenKind + id int64 + } + ops := []unaryOpInfo{{kind: firstOp.kind, id: p.nextID(firstOp)}} + + for p.peekTok.kind == tokExclamation || p.peekTok.kind == tokMinus { + op := p.nextToken() + ops = append(ops, unaryOpInfo{kind: op.kind, id: p.nextID(op)}) + } + + var operand ast.Expr + if len(ops) > 0 && ops[len(ops)-1].kind == tokMinus { + switch p.peekTok.kind { + case tokInt: + lastOp := ops[len(ops)-1] + ops = ops[:len(ops)-1] + operand = p.parseNegativeIntLiteral(lastOp.id) + case tokFloat: + lastOp := ops[len(ops)-1] + ops = ops[:len(ops)-1] + operand = p.parseNegativeDoubleLiteral(lastOp.id) + default: + operand = p.parseSelectorChain() + } + } else { + operand = p.parseSelectorChain() + } + + for i := len(ops) - 1; i >= 0; i-- { + opName := operators.LogicalNot + if ops[i].kind == tokMinus { + opName = operators.Negate + } + operand = p.helper.newGlobalCall(ops[i].id, opName, operand) + } + return operand +} + +func (p *prattParser) countGroupingParentheses() int { + if p.peekTok.kind != tokLeftParen { + return 0 + } + saved := p.lexer.SavePosition() + + leadingOpenParens := 1 + tok := p.nextSignificantToken(false) + for tok.kind == tokLeftParen { + leadingOpenParens++ + tok = p.nextSignificantToken(false) + } + if leadingOpenParens == 1 { + p.lexer.RestorePosition(saved) + return 1 + } + openParens := leadingOpenParens + consecutiveLeadingClosed := 0 + for openParens > 0 { + if tok.kind == tokEnd || tok.kind == tokError { + p.lexer.RestorePosition(saved) + return 1 + } + switch tok.kind { + case tokLeftParen: + openParens++ + consecutiveLeadingClosed = 0 + case tokRightParen: + if leadingOpenParens == openParens { + leadingOpenParens-- + consecutiveLeadingClosed++ + } else { + consecutiveLeadingClosed = 0 + } + openParens-- + default: + consecutiveLeadingClosed = 0 + } + if openParens > 0 { + tok = p.nextSignificantToken(false) + } + } + p.lexer.RestorePosition(saved) + if consecutiveLeadingClosed > 1 { + return consecutiveLeadingClosed + } + return 1 +} + +func (p *prattParser) parsePrimary() ast.Expr { + switch p.peekTok.kind { + case tokLeftParen: + groupingCount := p.countGroupingParentheses() + for i := 0; i < groupingCount; i++ { + p.nextToken() + } + expr := p.parseExpr() + for i := 0; i < groupingCount; i++ { + p.expect(tokRightParen, "expected ')'") + } + return expr + case tokNull: + return p.helper.exprFactory.NewLiteral(p.nextID(p.nextToken()), types.NullValue) + case tokTrue: + tok := p.nextToken() + return p.helper.newLiteralBool(p.nextID(tok), true) + case tokFalse: + tok := p.nextToken() + return p.helper.newLiteralBool(p.nextID(tok), false) + case tokInt: + return p.parseIntLiteral() + case tokUint: + return p.parseUintLiteral() + case tokFloat: + return p.parseDoubleLiteral() + case tokString: + return p.parseStringLiteral() + case tokBytes: + return p.parseBytesLiteral() + case tokLeftBracket: + return p.parseList() + case tokLeftBrace: + return p.parseMap() + case tokDot, tokIdent, tokReservedWord: + return p.parseIdentOrCall() + default: + badTok := p.nextToken() + if badTok.kind != tokError { + if badTok.kind == tokEnd { + p.reportError(badTok, "Syntax error: mismatched input '' expecting expression") + } else { + p.reportError(badTok, "unexpected token") + } + } + return p.helper.newExpr(badTok) + } +} + +func (p *prattParser) parseList() ast.Expr { + openTok := p.nextToken() + listID := p.nextID(openTok) + var elems []ast.Expr + var optionals []int32 + for p.peekTok.kind != tokRightBracket && p.peekTok.kind != tokEnd { + optional := false + if p.peekTok.kind == tokQuestion { + q := p.nextToken() + optional = true + if !p.enableOptionalSyntax { + p.reportError(q, "unsupported syntax '?'") + } + } + if optional { + optionals = append(optionals, int32(len(elems))) + } + elem := p.parseExpr() + elems = append(elems, elem) + if p.peekTok.kind == tokComma { + p.nextToken() + if p.peekTok.kind == tokRightBracket { + break + } + continue + } + break + } + p.expect(tokRightBracket, "expected ']'") + return p.helper.newList(listID, elems, optionals...) +} + +func (p *prattParser) parseMap() ast.Expr { + openTok := p.nextToken() + mapID := p.nextID(openTok) + var entries []ast.EntryExpr + for p.peekTok.kind != tokRightBrace && p.peekTok.kind != tokEnd { + optional := false + if p.peekTok.kind == tokQuestion { + q := p.nextToken() + optional = true + if !p.enableOptionalSyntax { + p.reportError(q, "unsupported syntax '?'") + } + } + key := p.parseExpr() + colonTok := p.peekTok + if !p.expect(tokColon, "expected ':' in map entry") { + break + } + entryID := p.nextID(colonTok) + val := p.parseExpr() + entries = append(entries, p.helper.newMapEntry(entryID, key, val, optional)) + if p.peekTok.kind == tokComma { + p.nextToken() + if p.peekTok.kind == tokRightBrace { + break + } + continue + } + break + } + p.expect(tokRightBrace, "expected '}'") + return p.helper.newMap(mapID, entries...) +} + +func (p *prattParser) parseIdentOrCall() ast.Expr { + leadingDot := false + firstTok := p.peekTok + if p.peekTok.kind == tokDot { + p.nextToken() + leadingDot = true + } + idTok := p.nextToken() + if idTok.kind != tokIdent && idTok.kind != tokReservedWord { + if idTok.kind != tokError { + p.reportError(idTok, "expected identifier") + } + return p.helper.newExpr(idTok) + } + idText := p.normalizeIdent(idTok, false) + if idTok.kind == tokReservedWord { + if _, ok := reservedIds[idText]; ok { + p.reportError(idTok, "reserved identifier: %s", idText) + } + } + name := idText + if leadingDot { + name = "." + idText + } + id := p.nextID(firstTok) + if p.peekTok.kind == tokLeftParen { + p.nextToken() + args := p.parseArguments(tokRightParen) + return p.globalCallOrMacro(id, name, args...) + } + return p.helper.newIdent(id, name) +} + +func (p *prattParser) parseArguments(closeTok tokenKind) []ast.Expr { + var args []ast.Expr + if p.peekTok.kind != closeTok && p.peekTok.kind != tokEnd { + for { + args = append(args, p.parseExpr()) + if p.peekTok.kind == tokComma { + p.nextToken() + if p.peekTok.kind == closeTok { + break + } + continue + } + break + } + } + p.expect(closeTok, "") + return args +} + +func (p *prattParser) parseIntLiteral() ast.Expr { + tok := p.nextToken() + id := p.nextID(tok) + text := p.tokenText(tok) + base := 10 + if strings.HasPrefix(text, "0x") || strings.HasPrefix(text, "0X") { + base = 16 + text = text[2:] + } + val, err := strconv.ParseInt(text, base, 64) + if err != nil { + return p.reportError(tok, "invalid int literal") + } + return p.helper.newLiteralInt(id, val) +} + +func (p *prattParser) parseNegativeIntLiteral(opID int64) ast.Expr { + tok := p.nextToken() + text := p.tokenText(tok) + base := 10 + if strings.HasPrefix(text, "0x") || strings.HasPrefix(text, "0X") { + base = 16 + text = text[2:] + } + val, err := strconv.ParseInt("-"+text, base, 64) + if err != nil { + return p.reportError(tok, "invalid int literal") + } + return p.helper.newLiteralInt(opID, val) +} + +func (p *prattParser) parseUintLiteral() ast.Expr { + tok := p.nextToken() + id := p.nextID(tok) + text := p.tokenText(tok) + text = text[:len(text)-1] + base := 10 + if strings.HasPrefix(text, "0x") || strings.HasPrefix(text, "0X") { + base = 16 + text = text[2:] + } + val, err := strconv.ParseUint(text, base, 64) + if err != nil { + return p.reportError(tok, "invalid uint literal") + } + return p.helper.newLiteralUint(id, val) +} + +func (p *prattParser) parseDoubleLiteral() ast.Expr { + tok := p.nextToken() + id := p.nextID(tok) + text := p.tokenText(tok) + val, err := strconv.ParseFloat(text, 64) + if err != nil { + return p.reportError(tok, "invalid double literal") + } + return p.helper.newLiteralDouble(id, val) +} + +func (p *prattParser) parseNegativeDoubleLiteral(opID int64) ast.Expr { + tok := p.nextToken() + text := p.tokenText(tok) + val, err := strconv.ParseFloat(text, 64) + if err != nil { + return p.reportError(tok, "invalid double literal") + } + return p.helper.newLiteralDouble(opID, -val) +} + +func (p *prattParser) parseStringLiteral() ast.Expr { + tok := p.nextToken() + id := p.nextID(tok) + text := p.tokenText(tok) + unescaped, err := unescape(text, false) + if err != nil { + return p.reportError(tok, "%s", err.Error()) + } + return p.helper.newLiteralString(id, unescaped) +} + +func (p *prattParser) parseBytesLiteral() ast.Expr { + tok := p.nextToken() + id := p.nextID(tok) + text := p.tokenText(tok) + if strings.HasPrefix(text, "b") || strings.HasPrefix(text, "B") { + text = text[1:] + } else if strings.HasPrefix(text, "rb") || strings.HasPrefix(text, "RB") || strings.HasPrefix(text, "rB") || strings.HasPrefix(text, "Rb") { + text = "r" + text[2:] + } else if strings.HasPrefix(text, "br") || strings.HasPrefix(text, "BR") || strings.HasPrefix(text, "bR") || strings.HasPrefix(text, "Br") { + text = text[1:] + } + unescaped, err := unescape(text, true) + if err != nil { + return p.reportError(tok, "%s", err.Error()) + } + return p.helper.newLiteralBytes(id, []byte(unescaped)) +} diff --git a/parser/pratt_parser_test.go b/parser/pratt_parser_test.go new file mode 100644 index 000000000..ef2f3303d --- /dev/null +++ b/parser/pratt_parser_test.go @@ -0,0 +1,1239 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "fmt" + "strings" + "testing" + + "cel.dev/cel-go/common" + "cel.dev/cel-go/common/ast" + "cel.dev/cel-go/common/debug" + "cel.dev/cel-go/test" +) + +var prattTestCases = []testInfo{ + // Constants + { + I: `42`, + P: `42^#1:*expr.Constant_Int64Value#`, + }, + { + I: `0x2A`, + P: `42^#1:*expr.Constant_Int64Value#`, + }, + { + I: `42u`, + P: `42u^#1:*expr.Constant_Uint64Value#`, + }, + { + I: `0x2Au`, + P: `42u^#1:*expr.Constant_Uint64Value#`, + }, + { + I: `3.14`, + P: `3.14^#1:*expr.Constant_DoubleValue#`, + }, + { + I: `-42`, + P: `-42^#1:*expr.Constant_Int64Value#`, + }, + { + I: `-3.14`, + P: `-3.14^#1:*expr.Constant_DoubleValue#`, + }, + { + I: `"hello world"`, + P: `"hello world"^#1:*expr.Constant_StringValue#`, + }, + { + I: `b"bytes"`, + P: `b"bytes"^#1:*expr.Constant_BytesValue#`, + }, + { + I: `true`, + P: `true^#1:*expr.Constant_BoolValue#`, + }, + { + I: `false`, + P: `false^#1:*expr.Constant_BoolValue#`, + }, + { + I: `null`, + P: `null^#1:*expr.Constant_NullValue#`, + }, + { + I: `9223372036854775807`, + P: `9223372036854775807^#1:*expr.Constant_Int64Value#`, + }, + { + I: `-9223372036854775808`, + P: `-9223372036854775808^#1:*expr.Constant_Int64Value#`, + }, + { + I: `-0x1A`, + P: `-26^#1:*expr.Constant_Int64Value#`, + }, + { + I: `-0X1a`, + P: `-26^#1:*expr.Constant_Int64Value#`, + }, + { + I: `-0x8000000000000000`, + P: `-9223372036854775808^#1:*expr.Constant_Int64Value#`, + }, + { + I: `0u`, + P: `0u^#1:*expr.Constant_Uint64Value#`, + }, + { + I: `-5.5e-3`, + P: `-0.0055^#1:*expr.Constant_DoubleValue#`, + }, + { + I: "\"\u2764\"", + P: "\"❤\"^#1:*expr.Constant_StringValue#", + }, + { + I: "\"\\a\\b\\f\\n\\r\\t\\v'\\\"\\\\\\? Legal escapes\"", + P: `"\a\b\f\n\r\t\v'\"\\? Legal escapes"^#1:*expr.Constant_StringValue#`, + }, + + // Identifiers and Parentheses + { + I: `a`, + P: `a^#1:*expr.Expr_IdentExpr#`, + }, + { + I: `(a)`, + P: `a^#1:*expr.Expr_IdentExpr#`, + }, + { + I: `((a))`, + P: `a^#1:*expr.Expr_IdentExpr#`, + }, + + // Unary operators + { + I: `!a`, + P: `!_( + a^#2:*expr.Expr_IdentExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `!false`, + P: `!_( + false^#2:*expr.Constant_BoolValue# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `-x`, + P: `-_( + x^#2:*expr.Expr_IdentExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `---a`, + P: `-_( + -_( + -_( + a^#4:*expr.Expr_IdentExpr# + )^#3:*expr.Expr_CallExpr# + )^#2:*expr.Expr_CallExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `- -1`, + P: `-_( + -1^#2:*expr.Constant_Int64Value# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `4--4`, + P: `_-_( + 4^#1:*expr.Constant_Int64Value#, + -4^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `!!a`, + P: `!_( + !_( + a^#3:*expr.Expr_IdentExpr# + )^#2:*expr.Expr_CallExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `-!true`, + P: `-_( + !_( + true^#3:*expr.Constant_BoolValue# + )^#2:*expr.Expr_CallExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `-!-x`, + P: `-_( + !_( + -_( + x^#4:*expr.Expr_IdentExpr# + )^#3:*expr.Expr_CallExpr# + )^#2:*expr.Expr_CallExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + + // Binary & Ternary operators + { + I: `1 + 2 * 3 - 4 / 2 % 3`, + P: `_-_( + _+_( + 1^#1:*expr.Constant_Int64Value#, + _*_( + 2^#3:*expr.Constant_Int64Value#, + 3^#5:*expr.Constant_Int64Value# + )^#4:*expr.Expr_CallExpr# + )^#2:*expr.Expr_CallExpr#, + _%_( + _/_( + 4^#7:*expr.Constant_Int64Value#, + 2^#9:*expr.Constant_Int64Value# + )^#8:*expr.Expr_CallExpr#, + 3^#11:*expr.Constant_Int64Value# + )^#10:*expr.Expr_CallExpr# + )^#6:*expr.Expr_CallExpr#`, + }, + { + I: `a < 10 && b <= 20 || c > 30 && d >= 40 || e == 50 && f != 60`, + P: `_||_( + _||_( + _&&_( + _<_( + a^#1:*expr.Expr_IdentExpr#, + 10^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#, + _<=_( + b^#5:*expr.Expr_IdentExpr#, + 20^#7:*expr.Constant_Int64Value# + )^#6:*expr.Expr_CallExpr# + )^#4:*expr.Expr_CallExpr#, + _&&_( + _>_( + c^#9:*expr.Expr_IdentExpr#, + 30^#11:*expr.Constant_Int64Value# + )^#10:*expr.Expr_CallExpr#, + _>=_( + d^#13:*expr.Expr_IdentExpr#, + 40^#15:*expr.Constant_Int64Value# + )^#14:*expr.Expr_CallExpr# + )^#12:*expr.Expr_CallExpr# + )^#8:*expr.Expr_CallExpr#, + _&&_( + _==_( + e^#17:*expr.Expr_IdentExpr#, + 50^#19:*expr.Constant_Int64Value# + )^#18:*expr.Expr_CallExpr#, + _!=_( + f^#21:*expr.Expr_IdentExpr#, + 60^#23:*expr.Constant_Int64Value# + )^#22:*expr.Expr_CallExpr# + )^#20:*expr.Expr_CallExpr# + )^#16:*expr.Expr_CallExpr#`, + }, + { + I: `a && b && c && d`, + P: `_&&_( + _&&_( + a^#1:*expr.Expr_IdentExpr#, + b^#3:*expr.Expr_IdentExpr# + )^#2:*expr.Expr_CallExpr#, + _&&_( + c^#5:*expr.Expr_IdentExpr#, + d^#7:*expr.Expr_IdentExpr# + )^#6:*expr.Expr_CallExpr# + )^#4:*expr.Expr_CallExpr#`, + }, + { + I: `a && b && c && d`, + P: `_&&_( + a^#1:*expr.Expr_IdentExpr#, + b^#3:*expr.Expr_IdentExpr#, + c^#5:*expr.Expr_IdentExpr#, + d^#7:*expr.Expr_IdentExpr# + )^#2:*expr.Expr_CallExpr#`, + Opts: []Option{EnableVariadicOperatorASTs(true)}, + }, + { + I: `a || b || c || d`, + P: `_||_( + _||_( + a^#1:*expr.Expr_IdentExpr#, + b^#3:*expr.Expr_IdentExpr# + )^#2:*expr.Expr_CallExpr#, + _||_( + c^#5:*expr.Expr_IdentExpr#, + d^#7:*expr.Expr_IdentExpr# + )^#6:*expr.Expr_CallExpr# + )^#4:*expr.Expr_CallExpr#`, + }, + { + I: `a || b || c || d`, + P: `_||_( + a^#1:*expr.Expr_IdentExpr#, + b^#3:*expr.Expr_IdentExpr#, + c^#5:*expr.Expr_IdentExpr#, + d^#7:*expr.Expr_IdentExpr# + )^#2:*expr.Expr_CallExpr#`, + Opts: []Option{EnableVariadicOperatorASTs(true)}, + }, + { + I: `10 - 3 - 2`, + P: `_-_( + _-_( + 10^#1:*expr.Constant_Int64Value#, + 3^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#, + 2^#5:*expr.Constant_Int64Value# + )^#4:*expr.Expr_CallExpr#`, + }, + { + I: `(((10 - 3) - 2))`, + P: `_-_( + _-_( + 10^#1:*expr.Constant_Int64Value#, + 3^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#, + 2^#5:*expr.Constant_Int64Value# + )^#4:*expr.Expr_CallExpr#`, + }, + { + I: `x in [1, 2, 3]`, + P: `@in( + x^#1:*expr.Expr_IdentExpr#, + [ + 1^#4:*expr.Constant_Int64Value#, + 2^#5:*expr.Constant_Int64Value#, + 3^#6:*expr.Constant_Int64Value# + ]^#3:*expr.Expr_ListExpr# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `a ? b : c`, + P: `_?_:_( + a^#1:*expr.Expr_IdentExpr#, + b^#3:*expr.Expr_IdentExpr#, + c^#4:*expr.Expr_IdentExpr# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `a ? b : c ? d : e`, + P: `_?_:_( + a^#1:*expr.Expr_IdentExpr#, + b^#3:*expr.Expr_IdentExpr#, + _?_:_( + c^#4:*expr.Expr_IdentExpr#, + d^#6:*expr.Expr_IdentExpr#, + e^#7:*expr.Expr_IdentExpr# + )^#5:*expr.Expr_CallExpr# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `(((1 + 2))) * (3 + 4)`, + P: `_*_( + _+_( + 1^#1:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#, + _+_( + 3^#5:*expr.Constant_Int64Value#, + 4^#7:*expr.Constant_Int64Value# + )^#6:*expr.Expr_CallExpr# + )^#4:*expr.Expr_CallExpr#`, + }, + + // Members, Selects, Indexing, and Calls + { + I: `a.b()`, + P: `a^#1:*expr.Expr_IdentExpr#.b()^#2:*expr.Expr_CallExpr#`, + }, + { + I: `a.b(1)`, + P: `a^#1:*expr.Expr_IdentExpr#.b( + 1^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `a.b(1, 2)`, + P: `a^#1:*expr.Expr_IdentExpr#.b( + 1^#3:*expr.Constant_Int64Value#, + 2^#4:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `"foo".size()`, + P: `"foo"^#1:*expr.Constant_StringValue#.size()^#2:*expr.Expr_CallExpr#`, + }, + { + I: `a.b.c`, + P: `a^#1:*expr.Expr_IdentExpr#.b^#2:*expr.Expr_SelectExpr#.c^#3:*expr.Expr_SelectExpr#`, + }, + { + I: `a[0]`, + P: `_[_]( + a^#1:*expr.Expr_IdentExpr#, + 0^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `[1, 3, 4][0]`, + P: `_[_]( + [ + 1^#2:*expr.Constant_Int64Value#, + 3^#3:*expr.Constant_Int64Value#, + 4^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#, + 0^#6:*expr.Constant_Int64Value# + )^#5:*expr.Expr_CallExpr#`, + }, + { + I: `a[b[c]]`, + P: `_[_]( + a^#1:*expr.Expr_IdentExpr#, + _[_]( + b^#3:*expr.Expr_IdentExpr#, + c^#5:*expr.Expr_IdentExpr# + )^#4:*expr.Expr_CallExpr# + )^#2:*expr.Expr_CallExpr#`, + }, + { + I: `a()`, + P: `a()^#1:*expr.Expr_CallExpr#`, + }, + { + I: `a(b)`, + P: `a( + b^#2:*expr.Expr_IdentExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `func(1, 2, 3)`, + P: `func( + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `.func(1, 2)`, + P: `.func( + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value# + )^#1:*expr.Expr_CallExpr#`, + }, + + // Collections & Structs + { + I: `[1, 2, 3]`, + P: `[ + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#`, + }, + { + I: `[1, 2, 3,]`, + P: `[ + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#`, + }, + { + I: `[]`, + P: `[]^#1:*expr.Expr_ListExpr#`, + }, + { + I: `{"a": 1, "b": 2}`, + P: `{ + "a"^#2:*expr.Constant_StringValue#:1^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, + "b"^#5:*expr.Constant_StringValue#:2^#7:*expr.Constant_Int64Value#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, + }, + { + I: `{foo: 5, bar: "xyz"}`, + P: `{ + foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, + bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, + }, + { + I: `{}`, + P: `{}^#1:*expr.Expr_StructExpr#`, + }, + { + I: `google.protobuf.Empty{}`, + P: `google.protobuf.Empty{}^#3:*expr.Expr_StructExpr#`, + }, + { + I: `foo{ a: b }`, + P: `foo{ + a:b^#2:*expr.Expr_IdentExpr#^#1:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, + }, + { + I: `pkg.Msg{field1: "val", field2: 42}`, + P: `pkg.Msg{ + field1:"val"^#3:*expr.Constant_StringValue#^#2:*expr.Expr_CreateStruct_Entry#, + field2:42^#5:*expr.Constant_Int64Value#^#4:*expr.Expr_CreateStruct_Entry# + }^#2:*expr.Expr_StructExpr#`, + }, + { + I: `.pkg.Msg{field1: "val"}`, + P: `.pkg.Msg{ + field1:"val"^#3:*expr.Constant_StringValue#^#2:*expr.Expr_CreateStruct_Entry# + }^#2:*expr.Expr_StructExpr#`, + }, + + // Macros + { + I: `has(a.b)`, + P: `a^#2:*expr.Expr_IdentExpr#.b~test-only~^#4:has#`, + }, + { + I: `[1, 2, 3].all(x, x > 0)`, + P: `__comprehension__( + // Variable + x, + // Target + [ + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#, + // Accumulator + @result, + // Init + true^#10:*expr.Constant_BoolValue#, + // LoopCondition + @not_strictly_false( + @result^#11:*expr.Expr_IdentExpr# + )^#12:*expr.Expr_CallExpr#, + // LoopStep + _&&_( + @result^#13:*expr.Expr_IdentExpr#, + _>_( + x^#7:*expr.Expr_IdentExpr#, + 0^#9:*expr.Constant_Int64Value# + )^#8:*expr.Expr_CallExpr# + )^#14:*expr.Expr_CallExpr#, + // Result + @result^#15:*expr.Expr_IdentExpr#)^#16:all#`, + }, + { + I: `[1, 2, 3].exists(x, x == 2)`, + P: `__comprehension__( + // Variable + x, + // Target + [ + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#, + // Accumulator + @result, + // Init + false^#10:*expr.Constant_BoolValue#, + // LoopCondition + @not_strictly_false( + !_( + @result^#11:*expr.Expr_IdentExpr# + )^#12:*expr.Expr_CallExpr# + )^#13:*expr.Expr_CallExpr#, + // LoopStep + _||_( + @result^#14:*expr.Expr_IdentExpr#, + _==_( + x^#7:*expr.Expr_IdentExpr#, + 2^#9:*expr.Constant_Int64Value# + )^#8:*expr.Expr_CallExpr# + )^#15:*expr.Expr_CallExpr#, + // Result + @result^#16:*expr.Expr_IdentExpr#)^#17:exists#`, + }, + { + I: `[1, 2, 3].exists_one(x, x == 2)`, + P: `__comprehension__( + // Variable + x, + // Target + [ + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#, + // Accumulator + @result, + // Init + 0^#10:*expr.Constant_Int64Value#, + // LoopCondition + true^#11:*expr.Constant_BoolValue#, + // LoopStep + _?_:_( + _==_( + x^#7:*expr.Expr_IdentExpr#, + 2^#9:*expr.Constant_Int64Value# + )^#8:*expr.Expr_CallExpr#, + _+_( + @result^#12:*expr.Expr_IdentExpr#, + 1^#13:*expr.Constant_Int64Value# + )^#14:*expr.Expr_CallExpr#, + @result^#15:*expr.Expr_IdentExpr# + )^#16:*expr.Expr_CallExpr#, + // Result + _==_( + @result^#17:*expr.Expr_IdentExpr#, + 1^#18:*expr.Constant_Int64Value# + )^#19:*expr.Expr_CallExpr#)^#20:exists_one#`, + }, + { + I: `[1, 2, 3].map(x, x * 2)`, + P: `__comprehension__( + // Variable + x, + // Target + [ + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#, + // Accumulator + @result, + // Init + []^#10:*expr.Expr_ListExpr#, + // LoopCondition + true^#11:*expr.Constant_BoolValue#, + // LoopStep + _+_( + @result^#12:*expr.Expr_IdentExpr#, + [ + _*_( + x^#7:*expr.Expr_IdentExpr#, + 2^#9:*expr.Constant_Int64Value# + )^#8:*expr.Expr_CallExpr# + ]^#13:*expr.Expr_ListExpr# + )^#14:*expr.Expr_CallExpr#, + // Result + @result^#15:*expr.Expr_IdentExpr#)^#16:map#`, + }, + { + I: `[1, 2, 3].filter(x, x > 1)`, + P: `__comprehension__( + // Variable + x, + // Target + [ + 1^#2:*expr.Constant_Int64Value#, + 2^#3:*expr.Constant_Int64Value#, + 3^#4:*expr.Constant_Int64Value# + ]^#1:*expr.Expr_ListExpr#, + // Accumulator + @result, + // Init + []^#10:*expr.Expr_ListExpr#, + // LoopCondition + true^#11:*expr.Constant_BoolValue#, + // LoopStep + _?_:_( + _>_( + x^#7:*expr.Expr_IdentExpr#, + 1^#9:*expr.Constant_Int64Value# + )^#8:*expr.Expr_CallExpr#, + _+_( + @result^#12:*expr.Expr_IdentExpr#, + [ + x^#6:*expr.Expr_IdentExpr# + ]^#13:*expr.Expr_ListExpr# + )^#14:*expr.Expr_CallExpr#, + @result^#15:*expr.Expr_IdentExpr# + )^#16:*expr.Expr_CallExpr#, + // Result + @result^#17:*expr.Expr_IdentExpr#)^#18:filter#`, + }, + + // Optional Syntax + { + I: `a.?b`, + P: `_?._( + a^#1:*expr.Expr_IdentExpr#, + "b"^#3:*expr.Constant_StringValue# + )^#2:*expr.Expr_CallExpr#`, + Opts: []Option{EnableOptionalSyntax(true)}, + }, + { + I: `a[?0]`, + P: `_[?_]( + a^#1:*expr.Expr_IdentExpr#, + 0^#3:*expr.Constant_Int64Value# + )^#2:*expr.Expr_CallExpr#`, + Opts: []Option{EnableOptionalSyntax(true)}, + }, + { + I: `[?a, b, ?c]`, + P: `[ + a^#2:*expr.Expr_IdentExpr#, + b^#3:*expr.Expr_IdentExpr#, + c^#4:*expr.Expr_IdentExpr# + ]^#1:*expr.Expr_ListExpr#`, + Opts: []Option{EnableOptionalSyntax(true)}, + }, + { + I: `{?a: 1, b: 2}`, + P: `{ + ?a^#2:*expr.Expr_IdentExpr#:1^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, + b^#5:*expr.Expr_IdentExpr#:2^#7:*expr.Constant_Int64Value#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, + Opts: []Option{EnableOptionalSyntax(true)}, + }, + { + I: `pkg.Msg{?field: 42}`, + P: `pkg.Msg{ + ?field:42^#3:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry# + }^#2:*expr.Expr_StructExpr#`, + Opts: []Option{EnableOptionalSyntax(true)}, + }, + + // Escaped Identifier Syntax + { + I: "msg.`field-name`", + P: `msg^#1:*expr.Expr_IdentExpr#.field-name^#2:*expr.Expr_SelectExpr#`, + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "msg.`a.b.c`", + P: `msg^#1:*expr.Expr_IdentExpr#.a.b.c^#2:*expr.Expr_SelectExpr#`, + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "msg.`field name`", + P: `msg^#1:*expr.Expr_IdentExpr#.field name^#2:*expr.Expr_SelectExpr#`, + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "Msg{`field-1`: 42}", + P: `Msg{ + field-1:42^#2:*expr.Constant_Int64Value#^#1:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + + // Error Cases + { + I: "(1 + 2", + E: "ERROR: :1:7: expected ')'\n" + + " | (1 + 2\n" + + " | ......^", + }, + { + I: "[1, 2", + E: "ERROR: :1:6: expected ']'\n" + + " | [1, 2\n" + + " | .....^", + }, + { + I: "{1: 2", + E: "ERROR: :1:6: expected '}'\n" + + " | {1: 2\n" + + " | .....^", + }, + { + I: "a ? b", + E: "ERROR: :1:6: expected ':' in conditional expression\n" + + " | a ? b\n" + + " | .....^", + }, + { + I: "1 + 2 3", + E: "ERROR: :1:7: Syntax error: mismatched input '3' expecting \n" + + " | 1 + 2 3\n" + + " | ......^", + }, + { + I: "0xFFFFFFFFFFFFFFFFF", + E: "ERROR: :1:1: invalid int literal\n" + + " | 0xFFFFFFFFFFFFFFFFF\n" + + " | ^", + }, + { + I: "0xFFFFFFFFFFFFFFFFFu", + E: "ERROR: :1:1: invalid uint literal\n" + + " | 0xFFFFFFFFFFFFFFFFFu\n" + + " | ^", + }, + { + I: "1.99e90000009", + E: "ERROR: :1:1: invalid double literal\n" + + " | 1.99e90000009\n" + + " | ^", + }, + { + I: "as", + E: "ERROR: :1:1: reserved identifier: as\n" + + " | as\n" + + " | ^", + }, + { + I: "msg.`ident`", + E: "ERROR: :1:5: unsupported syntax '`'\n" + + " | msg.`ident`\n" + + " | ....^", + Opts: []Option{EnableIdentEscapeSyntax(false)}, + }, + { + I: "a.?b", + E: "ERROR: :1:2: unsupported syntax '.?'\n" + + " | a.?b\n" + + " | .^", + }, + { + I: "has(m)", + E: "ERROR: :1:5: invalid argument to has() macro\n" + + " | has(m)\n" + + " | ....^", + Opts: []Option{Macros(AllMacros...)}, + }, + { + I: "[1, 2].all(1 + 2, true)", + E: "ERROR: :1:14: argument must be a simple name\n" + + " | [1, 2].all(1 + 2, true)\n" + + " | .............^", + Opts: []Option{Macros(AllMacros...)}, + }, + { + I: "[1, 2].all(__result__, true)", + E: "ERROR: :1:12: iteration variable overwrites accumulator variable\n" + + " | [1, 2].all(__result__, true)\n" + + " | ...........^", + Opts: []Option{Macros(AllMacros...)}, + }, + { + I: "1{}", + E: "ERROR: :1:2: Syntax error: mismatched input '{' expecting \n" + + " | 1{}\n" + + " | .^", + }, + { + I: "a.", + E: "ERROR: :1:3: expected identifier after '.'\n" + + " | a.\n" + + " | ..^", + }, + { + I: ". *", + E: "ERROR: :1:3: expected identifier\n" + + " | . *\n" + + " | ..^", + }, + { + I: ".as", + E: "ERROR: :1:2: reserved identifier: as\n" + + " | .as\n" + + " | .^", + }, + { + I: "* 2", + E: "ERROR: :1:1: unexpected token\n" + + " | * 2\n" + + " | ^\n" + + "ERROR: :1:3: Syntax error: mismatched input '2' expecting \n" + + " | * 2\n" + + " | ..^", + }, + { + I: "{'k' 'v'}", + E: "ERROR: :1:6: expected ':' in map entry\n" + + " | {'k' 'v'}\n" + + " | .....^", + }, + { + I: "Msg{1: 2}", + E: "ERROR: :1:5: expected struct field name\n" + + " | Msg{1: 2}\n" + + " | ....^", + }, + { + I: "Msg{f 10}", + E: "ERROR: :1:7: expected ':' in struct field\n" + + " | Msg{f 10}\n" + + " | ......^", + }, + { + I: "Msg{f: 10", + E: "ERROR: :1:10: expected '}'\n" + + " | Msg{f: 10\n" + + " | .........^", + }, + { + I: "f(1, 2", + E: "ERROR: :1:7: Syntax error: mismatched input expecting ')'\n" + + " | f(1, 2\n" + + " | ......^", + }, + { + I: "1e", + E: "ERROR: :1:1: floating point literal missing digits after exponent separator\n" + + " | 1e\n" + + " | ^", + }, + { + I: "\"unterminated", + E: "ERROR: :1:1: unterminated string literal\n" + + " | \"unterminated\n" + + " | ^", + }, + { + I: "b\"unterminated", + E: "ERROR: :1:1: unterminated bytes literal\n" + + " | b\"unterminated\n" + + " | ^", + }, + { + I: "a.`foo`()", + E: "ERROR: :1:3: unexpected quoted identifier\n" + + " | a.`foo`()\n" + + " | ..^", + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "`foo`", + E: "ERROR: :1:1: unexpected quoted identifier\n" + + " | `foo`\n" + + " | ^", + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "a.`b@c`", + E: "ERROR: :1:3: unexpected quoted identifier\n" + + " | a.`b@c`\n" + + " | ..^", + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "a.``", + E: "ERROR: :1:3: unexpected quoted identifier\n" + + " | a.``\n" + + " | ..^", + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "`foo", + E: "ERROR: :1:1: unterminated quoted identifier\n" + + " | `foo\n" + + " | ^", + Opts: []Option{EnableIdentEscapeSyntax(true)}, + }, + { + I: "-0x8000000000000001", + E: "ERROR: :1:2: invalid int literal\n" + + " | -0x8000000000000001\n" + + " | .^", + }, + { + I: "-9223372036854775809", + E: "ERROR: :1:2: invalid int literal\n" + + " | -9223372036854775809\n" + + " | .^", + }, + { + I: "a[?0]", + E: "ERROR: :1:2: unsupported syntax '?'\n" + + " | a[?0]\n" + + " | .^", + Opts: []Option{EnableOptionalSyntax(false)}, + }, + { + I: "[?1]", + E: "ERROR: :1:2: unsupported syntax '?'\n" + + " | [?1]\n" + + " | .^", + Opts: []Option{EnableOptionalSyntax(false)}, + }, + { + I: "{?'k': 'v'}", + E: "ERROR: :1:2: unsupported syntax '?'\n" + + " | {?'k': 'v'}\n" + + " | .^", + Opts: []Option{EnableOptionalSyntax(false)}, + }, + { + I: "Msg{?f: 1}", + E: "ERROR: :1:5: unsupported syntax '?'\n" + + " | Msg{?f: 1}\n" + + " | ....^", + Opts: []Option{EnableOptionalSyntax(false)}, + }, + { + I: "a.?`foo`", + E: "ERROR: :1:4: unsupported syntax '`'\n" + + " | a.?`foo`\n" + + " | ...^", + Opts: []Option{EnableOptionalSyntax(true), EnableIdentEscapeSyntax(false)}, + }, +} + +func parse(source common.Source, opts ...Option) (*ast.AST, *common.Errors) { + p, err := NewPrattParser(opts...) + if err != nil { + panic(err) + } + return p.Parse(source) +} + +func TestPrattParser(t *testing.T) { + for i, tst := range prattTestCases { + name := fmt.Sprintf("%d %s", i, tst.I) + // Local variable required as the closure will reference the value for the last + // 'tst' value rather than the local 'tc' instance declared within the loop. + tc := tst + t.Run(name, func(t *testing.T) { + t.Parallel() + opts := tc.Opts + if len(opts) == 0 { + opts = []Option{Macros(AllMacros...), PopulateMacroCalls(true)} + } + src := common.NewTextSource(tc.I) + parsed, errors := parse(src, opts...) + if len(errors.GetErrors()) > 0 { + actualErr := errors.ToDisplayString() + if tc.E == "" { + t.Fatalf("Unexpected errors: %v", actualErr) + } else if !test.Compare(actualErr, tc.E) { + t.Fatal(test.DiffMessage("Error mismatch", actualErr, tc.E)) + } + return + } else if tc.E != "" { + t.Fatalf("Expected error not thrown: '%s'", tc.E) + } + failureDisplayMethod := fmt.Sprintf("Parse(\"%s\")", tc.I) + actualWithKind := debug.ToAdornedDebugString(parsed.Expr(), &kindAndIDAdorner{parsed.SourceInfo()}) + if !test.Compare(actualWithKind, tc.P) { + t.Fatal(test.DiffMessage(fmt.Sprintf("Structure - %s", failureDisplayMethod), actualWithKind, tc.P)) + } + }) + } +} +func TestPrattParserSourceInfoPositions(t *testing.T) { + src := common.NewTextSource("a + b") + p, err := NewPrattParser() + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + parsed, errs := p.Parse(src) + if len(errs.GetErrors()) > 0 { + t.Fatalf("Parse() failed: %s", errs.ToDisplayString()) + } + sourceInfo := parsed.SourceInfo() + root := parsed.Expr() + if sourceInfo.GetStartLocation(root.ID()).Column() != 2 { + t.Errorf("expected root column 2, got %d", sourceInfo.GetStartLocation(root.ID()).Column()) + } + args := root.AsCall().Args() + if len(args) != 2 { + t.Fatalf("expected 2 args, got %d", len(args)) + } + if sourceInfo.GetStartLocation(args[0].ID()).Column() != 0 { + t.Errorf("expected arg[0] column 0, got %d", sourceInfo.GetStartLocation(args[0].ID()).Column()) + } + if sourceInfo.GetStartLocation(args[1].ID()).Column() != 4 { + t.Errorf("expected arg[1] column 4, got %d", sourceInfo.GetStartLocation(args[1].ID()).Column()) + } +} + +func TestPrattParserRecursionDepth(t *testing.T) { + t.Run("DeeplyNestedBracketsLimitExceeded", func(t *testing.T) { + p, err := NewPrattParser(MaxRecursionDepth(5)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("[[[[[[1]]]]]]")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected recursion limit error, got none") + } + }) + + t.Run("IgnoreExtraParens", func(t *testing.T) { + p, err := NewPrattParser(MaxRecursionDepth(1)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("((((1))))")) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error: %s", errs.ToDisplayString()) + } + }) + + t.Run("DeeplyNestedParens1000", func(t *testing.T) { + p, err := NewPrattParser(MaxRecursionDepth(1)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + expr1 := strings.Repeat("(", 1000) + "42" + strings.Repeat(")", 1000) + _, errs := p.Parse(common.NewTextSource(expr1)) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on 1000 parens literal: %s", errs.ToDisplayString()) + } + + expr2 := strings.Repeat("(", 1000) + "1 + 2" + strings.Repeat(")", 1000) + _, errs = p.Parse(common.NewTextSource(expr2)) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on 1000 parens binary: %s", errs.ToDisplayString()) + } + }) + + t.Run("SequentialScopesDoNotAccumulateDepth", func(t *testing.T) { + p, err := NewPrattParser(MaxRecursionDepth(2)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("[1] + [2] + [3]")) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on sequential scopes: %s", errs.ToDisplayString()) + } + }) +} + +func TestPrattParserMacroCalls(t *testing.T) { + t.Run("DisabledByDefault", func(t *testing.T) { + p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(false)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + if len(parsed.SourceInfo().MacroCalls()) != 0 { + t.Errorf("expected 0 macro calls, got %d", len(parsed.SourceInfo().MacroCalls())) + } + }) + + t.Run("GlobalMacroCallRecorded", func(t *testing.T) { + p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 1 { + t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) + } + }) + + t.Run("ReceiverMacroCallRecorded", func(t *testing.T) { + p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("[1, 2].exists(x, x > 0)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 1 { + t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) + } + }) + + t.Run("NestedMacroCallsRecorded", func(t *testing.T) { + p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("[1, 2].all(x, has(x.b))")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 2 { + t.Fatalf("expected 2 macro calls, got %d", len(macroCalls)) + } + }) +} + +func TestPrattParserErrorRecoveryLimits(t *testing.T) { + t.Run("LimitZero", func(t *testing.T) { + p, err := NewPrattParser(ErrorRecoveryLimit(0)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("......")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected error recovery limit error, got none") + } + }) + + t.Run("LimitOne", func(t *testing.T) { + p, err := NewPrattParser(ErrorRecoveryLimit(1)) + if err != nil { + t.Fatalf("NewPrattParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("......")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected error recovery limit error, got none") + } + }) +} +func BenchmarkParsers(b *testing.B) { + exprs := []string{ + `42`, + `a > 5 && b < 10 || c == "xyz"`, + `[1, 2, 3].all(x, x > 0) && [4, 5, 6].exists(y, y == 5)`, + `pkg.Msg{field1: "value", field2: 123, list_field: [1, 2, 3], map_field: {"a": true, "b": false}}`, + `a.b.c.d.e.f(1, 2, [3, ?4], {?5: 6}) ? (x + y * z - w / v) : (!p && !q || r.s)`, + } + + antlrParser, _ := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnableOptionalSyntax(true)) + prattParser, _ := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true), EnableOptionalSyntax(true)) + + for _, expr := range exprs { + src := common.NewTextSource(expr) + + b.Run("ANTLR/"+expr[:min(len(expr), 20)], func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = antlrParser.Parse(src) + } + }) + + b.Run("Pratt/"+expr[:min(len(expr), 20)], func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = prattParser.Parse(src) + } + }) + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} From 52a1a2d4c0c315b19df967ac6a638f870fd92229 Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Thu, 20 Aug 2026 11:03:32 -0700 Subject: [PATCH 2/5] [Pratt parser] Check expression size limit before allocating worker --- parser/pratt_parser.go | 23 +++++++++++------------ parser/pratt_parser_test.go | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go index 4a22fb9a3..1ccfef268 100644 --- a/parser/pratt_parser.go +++ b/parser/pratt_parser.go @@ -157,26 +157,25 @@ func NewPrattParser(opts ...Option) (*PrattParser, error) { // Parse parses the expression represented by source using the Pratt parser and returns the result. func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { errs := common.NewErrors(source) - pratt := p.newWorker(source, errs) - var out ast.Expr - if pratt.length > int32(p.expressionSizeCodePointLimit) { - out = pratt.reportError(token{kind: tokError, start: 0, end: 0}, + buf, ok := source.(runes.Buffer) + if !ok { + buf = runes.NewBuffer(source.Content()) + } + if buf.Len() > p.expressionSizeCodePointLimit { + errs.ReportError(common.NoLocation, "expression code point size exceeds limit: size: %d, limit %d", - pratt.length, p.expressionSizeCodePointLimit) - } else { - out = pratt.parse() + buf.Len(), p.expressionSizeCodePointLimit) + return nil, errs } + pratt := p.newWorker(source, buf, errs) + out := pratt.parse() if len(errs.GetErrors()) > 0 { return nil, errs } return ast.NewAST(out, pratt.helper.getSourceInfo()), errs } -func (p *PrattParser) newWorker(source common.Source, errs *common.Errors) *prattParser { - buf, ok := source.(runes.Buffer) - if !ok { - buf = runes.NewBuffer(source.Content()) - } +func (p *PrattParser) newWorker(source common.Source, buf runes.Buffer, errs *common.Errors) *prattParser { accu := AccumulatorName if p.enableHiddenAccumulatorName { accu = HiddenAccumulatorName diff --git a/parser/pratt_parser_test.go b/parser/pratt_parser_test.go index ef2f3303d..4ad012efc 100644 --- a/parser/pratt_parser_test.go +++ b/parser/pratt_parser_test.go @@ -1198,6 +1198,22 @@ func TestPrattParserErrorRecoveryLimits(t *testing.T) { } }) } + +func TestPrattParserExpressionSizeCodePointLimit(t *testing.T) { + p, err := NewPrattParser(Macros(AllMacros...), ExpressionSizeCodePointLimit(2)) + if err != nil { + t.Fatal(err) + } + src := common.NewTextSource("foo") + _, errs := p.Parse(src) + if got, want := len(errs.GetErrors()), 1; got != want { + t.Fatalf("got %d errors, want %d errors: %s", got, want, errs.ToDisplayString()) + } + if got, want := errs.GetErrors()[0].Message, "expression code point size exceeds limit: size: 3, limit 2"; got != want { + t.Fatalf("got %q, want %q: %s", got, want, errs.GetErrors()[0].ToDisplayString(src)) + } +} + func BenchmarkParsers(b *testing.B) { exprs := []string{ `42`, From b6b2bd723ed389cd556976ed18cff6cab957c3b8 Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Thu, 20 Aug 2026 11:06:49 -0700 Subject: [PATCH 3/5] [Pratt parser] Inline newWorker into Parse --- parser/pratt_parser.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go index 1ccfef268..8e81b8f86 100644 --- a/parser/pratt_parser.go +++ b/parser/pratt_parser.go @@ -167,21 +167,12 @@ func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { buf.Len(), p.expressionSizeCodePointLimit) return nil, errs } - pratt := p.newWorker(source, buf, errs) - out := pratt.parse() - if len(errs.GetErrors()) > 0 { - return nil, errs - } - return ast.NewAST(out, pratt.helper.getSourceInfo()), errs -} - -func (p *PrattParser) newWorker(source common.Source, buf runes.Buffer, errs *common.Errors) *prattParser { accu := AccumulatorName if p.enableHiddenAccumulatorName { accu = HiddenAccumulatorName } fac := ast.NewExprFactoryWithAccumulator(accu) - pp := &prattParser{ + pratt := &prattParser{ content: buf, length: int32(buf.Len()), helper: newParserHelper(source, fac), @@ -198,8 +189,12 @@ func (p *PrattParser) newWorker(source common.Source, buf runes.Buffer, errs *co enableVariadicOperatorASTs: p.enableVariadicOperatorASTs, enableIdentEscapeSyntax: p.enableIdentEscapeSyntax, } - pp.initTokenStream() - return pp + pratt.initTokenStream() + out := pratt.parse() + if len(errs.GetErrors()) > 0 { + return nil, errs + } + return ast.NewAST(out, pratt.helper.getSourceInfo()), errs } func (p *prattParser) initTokenStream() { From 8fa278501fad79ac6f8197f2c681d05f3fb6bb50 Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Thu, 20 Aug 2026 15:00:28 -0700 Subject: [PATCH 4/5] [Pratt parser] Test multiple errors on disabled ident escape syntax --- parser/pratt_parser_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/parser/pratt_parser_test.go b/parser/pratt_parser_test.go index 4ad012efc..17f450056 100644 --- a/parser/pratt_parser_test.go +++ b/parser/pratt_parser_test.go @@ -795,10 +795,13 @@ var prattTestCases = []testInfo{ " | ^", }, { - I: "msg.`ident`", + I: "msg.`ident` + msg.`other`", E: "ERROR: :1:5: unsupported syntax '`'\n" + - " | msg.`ident`\n" + - " | ....^", + " | msg.`ident` + msg.`other`\n" + + " | ....^\n" + + "ERROR: :1:19: unsupported syntax '`'\n" + + " | msg.`ident` + msg.`other`\n" + + " | ..................^", Opts: []Option{EnableIdentEscapeSyntax(false)}, }, { From a774979fb6c9a30653a89b031d1d1619d382eaf0 Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Thu, 20 Aug 2026 15:09:14 -0700 Subject: [PATCH 5/5] [Pratt parser] Rename parseBalancedLogicalChain to parseLogicalChain --- parser/pratt_parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go index 8e81b8f86..c2eefd927 100644 --- a/parser/pratt_parser.go +++ b/parser/pratt_parser.go @@ -454,7 +454,7 @@ func (p *prattParser) parseBinaryAndTernary(minPrec int) ast.Expr { } if opInfo.name == operators.LogicalOr || opInfo.name == operators.LogicalAnd { - lhs = p.parseBalancedLogicalChain(lhs, opInfo) + lhs = p.parseLogicalChain(lhs, opInfo) continue } @@ -477,7 +477,7 @@ func (p *prattParser) parseTernary(lhs ast.Expr) ast.Expr { return p.helper.newGlobalCall(opID, operators.Conditional, lhs, trueExpr, falseExpr) } -func (p *prattParser) parseBalancedLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.Expr { +func (p *prattParser) parseLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.Expr { l := p.newLogicManager(opInfo.name, lhs) for p.peekTok.kind == opInfo.kind { opTok := p.nextToken()