From 1250b8b90cf32effaf8a761d72b5e6965d8fa94f Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Fri, 21 Aug 2026 15:37:31 -0700 Subject: [PATCH 1/3] [Pratt parser] Extract ANTLR parser into antlr_parser.go --- parser/BUILD.bazel | 4 +- parser/{parser.go => antlr_parser.go} | 197 ++++++++++---------------- 2 files changed, 75 insertions(+), 126 deletions(-) rename parser/{parser.go => antlr_parser.go} (83%) diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel index e4ff679d1..f30188b3c 100644 --- a/parser/BUILD.bazel +++ b/parser/BUILD.bazel @@ -7,14 +7,14 @@ package( go_library( name = "go_default_library", srcs = [ + "antlr_parser.go", "errors.go", "helper.go", "input.go", "lexer.go", "macro.go", "options.go", - "parser.go", - "pratt_parser.go", + "pratt_parser.go", "unescape.go", "unparser.go", ], diff --git a/parser/parser.go b/parser/antlr_parser.go similarity index 83% rename from parser/parser.go rename to parser/antlr_parser.go index 2df20a704..bab466f27 100644 --- a/parser/parser.go +++ b/parser/antlr_parser.go @@ -1,4 +1,4 @@ -// Copyright 2018 Google LLC +// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package parser declares an expression parser with support for macro -// expansion. package parser import ( "errors" "fmt" + "math" "regexp" "strconv" "strings" @@ -33,14 +32,14 @@ import ( "cel.dev/cel-go/parser/gen" ) -// Parser encapsulates the context necessary to perform parsing for different expressions. -type Parser struct { +// AntlrParser encapsulates the context necessary to perform ANTLR parsing for different expressions. +type AntlrParser struct { options } -// NewParser builds and returns a new Parser using the provided options. -func NewParser(opts ...Option) (*Parser, error) { - p := &Parser{} +// NewAntlrParser builds and returns a new AntlrParser using the provided options. +func NewAntlrParser(opts ...Option) (*AntlrParser, error) { + p := &AntlrParser{} p.enableHiddenAccumulatorName = true p.enableIdentEscapeSyntax = true for _, opt := range opts { @@ -55,7 +54,7 @@ func NewParser(opts ...Option) (*Parser, error) { p.maxRecursionDepth = 250 } if p.maxRecursionDepth == -1 { - p.maxRecursionDepth = int((^uint(0)) >> 1) + p.maxRecursionDepth = math.MaxInt } if p.errorRecoveryTokenLookaheadLimit == 0 { p.errorRecoveryTokenLookaheadLimit = 256 @@ -64,45 +63,32 @@ func NewParser(opts ...Option) (*Parser, error) { p.errorRecoveryLimit = 30 } if p.errorRecoveryLimit == -1 { - p.errorRecoveryLimit = int((^uint(0)) >> 1) + p.errorRecoveryLimit = math.MaxInt } if p.expressionSizeCodePointLimit == 0 { p.expressionSizeCodePointLimit = 100_000 } if p.expressionSizeCodePointLimit == -1 { - p.expressionSizeCodePointLimit = int((^uint(0)) >> 1) + p.expressionSizeCodePointLimit = math.MaxInt } if p.maxExpressionNodeCount == 0 { p.maxExpressionNodeCount = 100_000 } if p.maxExpressionNodeCount == -1 { - p.maxExpressionNodeCount = int((^uint(0)) >> 1) + p.maxExpressionNodeCount = math.MaxInt } - // Bool is false by default, so populateMacroCalls will be false by default return p, nil } -// mustNewParser does the work of NewParser and panics if an error occurs. -// -// This function is only intended for internal use and is for backwards compatibility in Parse and -// ParseWithMacros, where we know the options will result in an error. -func mustNewParser(opts ...Option) *Parser { - p, err := NewParser(opts...) - if err != nil { - panic(err) - } - return p -} - // Parse parses the expression represented by source and returns the result. -func (p *Parser) Parse(source common.Source) (*ast.AST, *common.Errors) { +func (p *AntlrParser) Parse(source common.Source) (*ast.AST, *common.Errors) { errs := common.NewErrors(source) accu := AccumulatorName if p.enableHiddenAccumulatorName { accu = HiddenAccumulatorName } fac := ast.NewExprFactoryWithAccumulator(accu) - impl := parser{ + impl := antlrParser{ errors: &parseErrors{errs}, exprFactory: fac, helper: newParserHelper(source, fac), @@ -132,32 +118,6 @@ func (p *Parser) Parse(source common.Source) (*ast.AST, *common.Errors) { return ast.NewAST(out, impl.helper.getSourceInfo()), errs } -// reservedIds are not legal to use as variables. We exclude them post-parse, as they *are* valid -// field names for protos, and it would complicate the grammar to distinguish the cases. -var reservedIds = map[string]struct{}{ - "as": {}, - "break": {}, - "const": {}, - "continue": {}, - "else": {}, - "false": {}, - "for": {}, - "function": {}, - "if": {}, - "import": {}, - "in": {}, - "let": {}, - "loop": {}, - "package": {}, - "namespace": {}, - "null": {}, - "return": {}, - "true": {}, - "var": {}, - "void": {}, - "while": {}, -} - func unescapeIdent(in string) (string, error) { if len(in) <= 2 { return "", errors.New("invalid escaped identifier: underflow") @@ -165,28 +125,6 @@ func unescapeIdent(in string) (string, error) { return in[1 : len(in)-1], nil } -// normalizeIdent returns the interpreted identifier. -func (p *parser) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error) { - switch ident := ctx.(type) { - case *gen.SimpleIdentifierContext: - return ident.GetId().GetText(), nil - case *gen.EscapedIdentifierContext: - if !p.enableIdentEscapeSyntax { - return "", errors.New("unsupported syntax: '`'") - } - return unescapeIdent(ident.GetId().GetText()) - } - return "", errors.New("unsupported ident kind") -} - -// Parse converts a source input a parsed expression. -// This function calls ParseWithMacros with AllMacros. -// -// Deprecated: Use NewParser().Parse() instead. -func Parse(source common.Source) (*ast.AST, *common.Errors) { - return mustNewParser(Macros(AllMacros...)).Parse(source) -} - type recursionError struct { message string } @@ -317,7 +255,7 @@ func (rl *recoveryLimitErrorStrategy) checkAttempts(recognizer antlr.Parser) { var _ antlr.ErrorStrategy = &recoveryLimitErrorStrategy{} -type parser struct { +type antlrParser struct { gen.BaseCELVisitor errors *parseErrors exprFactory ast.ExprFactory @@ -336,9 +274,23 @@ type parser struct { enableIdentEscapeSyntax bool } -var _ gen.CELVisitor = (*parser)(nil) +var _ gen.CELVisitor = (*antlrParser)(nil) -func (p *parser) parse(expr runes.Buffer, desc string) ast.Expr { +// normalizeIdent returns the interpreted identifier. +func (p *antlrParser) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error) { + switch ident := ctx.(type) { + case *gen.SimpleIdentifierContext: + return ident.GetId().GetText(), nil + case *gen.EscapedIdentifierContext: + if !p.enableIdentEscapeSyntax { + return "", errors.New("unsupported syntax: '`'") + } + return unescapeIdent(ident.GetId().GetText()) + } + return "", errors.New("unsupported ident kind") +} + +func (p *antlrParser) parse(expr runes.Buffer, desc string) ast.Expr { lexer := gen.NewCELLexer(newCharStream(expr, desc)) lexer.RemoveErrorListeners() lexer.AddErrorListener(p) @@ -381,7 +333,7 @@ func (p *parser) parse(expr runes.Buffer, desc string) ast.Expr { } // Visitor implementations. -func (p *parser) Visit(tree antlr.ParseTree) any { +func (p *antlrParser) Visit(tree antlr.ParseTree) any { t := unnest(tree) switch tree := t.(type) { case *gen.StartContext: @@ -466,16 +418,15 @@ func (p *parser) Visit(tree antlr.ParseTree) any { return p.reportError(common.NoLocation, "unknown parse element encountered: %s", txt) } return p.helper.newExpr(common.NoLocation) - } // Visit a parse tree produced by CELParser#start. -func (p *parser) VisitStart(ctx *gen.StartContext) any { +func (p *antlrParser) VisitStart(ctx *gen.StartContext) any { return p.Visit(ctx.Expr()) } // Visit a parse tree produced by CELParser#expr. -func (p *parser) VisitExpr(ctx *gen.ExprContext) any { +func (p *antlrParser) VisitExpr(ctx *gen.ExprContext) any { result := p.Visit(ctx.GetE()).(ast.Expr) if ctx.GetOp() == nil { return result @@ -487,7 +438,7 @@ func (p *parser) VisitExpr(ctx *gen.ExprContext) any { } // Visit a parse tree produced by CELParser#conditionalOr. -func (p *parser) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { +func (p *antlrParser) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { result := p.Visit(ctx.GetE()).(ast.Expr) l := p.newLogicManager(operators.LogicalOr, result) rest := ctx.GetE1() @@ -503,7 +454,7 @@ func (p *parser) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { } // Visit a parse tree produced by CELParser#conditionalAnd. -func (p *parser) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { +func (p *antlrParser) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { result := p.Visit(ctx.GetE()).(ast.Expr) l := p.newLogicManager(operators.LogicalAnd, result) rest := ctx.GetE1() @@ -519,7 +470,7 @@ func (p *parser) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { } // Visit a parse tree produced by CELParser#relation. -func (p *parser) VisitRelation(ctx *gen.RelationContext) any { +func (p *antlrParser) VisitRelation(ctx *gen.RelationContext) any { opText := "" if ctx.GetOp() != nil { opText = ctx.GetOp().GetText() @@ -534,7 +485,7 @@ func (p *parser) VisitRelation(ctx *gen.RelationContext) any { } // Visit a parse tree produced by CELParser#calc. -func (p *parser) VisitCalc(ctx *gen.CalcContext) any { +func (p *antlrParser) VisitCalc(ctx *gen.CalcContext) any { opText := "" if ctx.GetOp() != nil { opText = ctx.GetOp().GetText() @@ -548,12 +499,12 @@ func (p *parser) VisitCalc(ctx *gen.CalcContext) any { return p.reportError(ctx, "operator not found") } -func (p *parser) VisitUnary(ctx *gen.UnaryContext) any { +func (p *antlrParser) VisitUnary(ctx *gen.UnaryContext) any { return p.helper.newLiteralString(ctx, "<>") } // Visit a parse tree produced by CELParser#LogicalNot. -func (p *parser) VisitLogicalNot(ctx *gen.LogicalNotContext) any { +func (p *antlrParser) VisitLogicalNot(ctx *gen.LogicalNotContext) any { if len(ctx.GetOps())%2 == 0 { return p.Visit(ctx.Member()) } @@ -562,7 +513,7 @@ func (p *parser) VisitLogicalNot(ctx *gen.LogicalNotContext) any { return p.globalCallOrMacro(opID, operators.LogicalNot, target) } -func (p *parser) VisitNegate(ctx *gen.NegateContext) any { +func (p *antlrParser) VisitNegate(ctx *gen.NegateContext) any { if len(ctx.GetOps())%2 == 0 { return p.Visit(ctx.Member()) } @@ -572,7 +523,7 @@ func (p *parser) VisitNegate(ctx *gen.NegateContext) any { } // VisitSelect visits a parse tree produced by CELParser#Select. -func (p *parser) VisitSelect(ctx *gen.SelectContext) any { +func (p *antlrParser) VisitSelect(ctx *gen.SelectContext) any { operand := p.Visit(ctx.Member()).(ast.Expr) // Handle the error case where no valid identifier is specified. if ctx.GetId() == nil || ctx.GetOp() == nil { @@ -596,7 +547,7 @@ func (p *parser) VisitSelect(ctx *gen.SelectContext) any { } // VisitMemberCall visits a parse tree produced by CELParser#MemberCall. -func (p *parser) VisitMemberCall(ctx *gen.MemberCallContext) any { +func (p *antlrParser) VisitMemberCall(ctx *gen.MemberCallContext) any { operand := p.Visit(ctx.Member()).(ast.Expr) // Handle the error case where no valid identifier is specified. if ctx.GetId() == nil { @@ -608,7 +559,7 @@ func (p *parser) VisitMemberCall(ctx *gen.MemberCallContext) any { } // Visit a parse tree produced by CELParser#Index. -func (p *parser) VisitIndex(ctx *gen.IndexContext) any { +func (p *antlrParser) VisitIndex(ctx *gen.IndexContext) any { target := p.Visit(ctx.Member()).(ast.Expr) // Handle the error case where no valid identifier is specified. if ctx.GetOp() == nil { @@ -627,7 +578,7 @@ func (p *parser) VisitIndex(ctx *gen.IndexContext) any { } // Visit a parse tree produced by CELParser#CreateMessage. -func (p *parser) VisitCreateMessage(ctx *gen.CreateMessageContext) any { +func (p *antlrParser) VisitCreateMessage(ctx *gen.CreateMessageContext) any { messageName := "" for _, id := range ctx.GetIds() { if len(messageName) != 0 { @@ -644,7 +595,7 @@ func (p *parser) VisitCreateMessage(ctx *gen.CreateMessageContext) any { } // Visit a parse tree of field initializers. -func (p *parser) VisitIFieldInitializerList(ctx gen.IFieldInitializerListContext) any { +func (p *antlrParser) VisitIFieldInitializerList(ctx gen.IFieldInitializerListContext) any { if ctx == nil || ctx.GetFields() == nil { // This is the result of a syntax error handled elswhere, return empty. return []ast.EntryExpr{} @@ -681,7 +632,7 @@ func (p *parser) VisitIFieldInitializerList(ctx gen.IFieldInitializerListContext } // Visit a parse tree produced by CELParser#Ident. -func (p *parser) VisitIdent(ctx *gen.IdentContext) any { +func (p *antlrParser) VisitIdent(ctx *gen.IdentContext) any { identName := "" if ctx.GetLeadingDot() != nil { identName = "." @@ -700,7 +651,7 @@ func (p *parser) VisitIdent(ctx *gen.IdentContext) any { } // Visit a parse tree produced by CELParser#GlobalCallContext. -func (p *parser) VisitGlobalCall(ctx *gen.GlobalCallContext) any { +func (p *antlrParser) VisitGlobalCall(ctx *gen.GlobalCallContext) any { identName := "" if ctx.GetLeadingDot() != nil { identName = "." @@ -717,18 +668,17 @@ func (p *parser) VisitGlobalCall(ctx *gen.GlobalCallContext) any { identName += id opID := p.helper.id(ctx.GetOp()) return p.globalCallOrMacro(opID, identName, p.visitExprList(ctx.GetArgs())...) - } // Visit a parse tree produced by CELParser#CreateList. -func (p *parser) VisitCreateList(ctx *gen.CreateListContext) any { +func (p *antlrParser) VisitCreateList(ctx *gen.CreateListContext) any { listID := p.helper.id(ctx.GetOp()) elems, optionals := p.visitListInit(ctx.GetElems()) return p.helper.newList(listID, elems, optionals...) } // Visit a parse tree produced by CELParser#CreateStruct. -func (p *parser) VisitCreateStruct(ctx *gen.CreateStructContext) any { +func (p *antlrParser) VisitCreateStruct(ctx *gen.CreateStructContext) any { structID := p.helper.id(ctx.GetOp()) entries := []ast.EntryExpr{} if ctx.GetEntries() != nil { @@ -738,7 +688,7 @@ func (p *parser) VisitCreateStruct(ctx *gen.CreateStructContext) any { } // Visit a parse tree produced by CELParser#mapInitializerList. -func (p *parser) VisitMapInitializerList(ctx *gen.MapInitializerListContext) any { +func (p *antlrParser) VisitMapInitializerList(ctx *gen.MapInitializerListContext) any { if ctx == nil || ctx.GetKeys() == nil { // This is the result of a syntax error handled elswhere, return empty. return []ast.EntryExpr{} @@ -768,7 +718,7 @@ func (p *parser) VisitMapInitializerList(ctx *gen.MapInitializerListContext) any } // Visit a parse tree produced by CELParser#Int. -func (p *parser) VisitInt(ctx *gen.IntContext) any { +func (p *antlrParser) VisitInt(ctx *gen.IntContext) any { text := ctx.GetTok().GetText() base := 10 if strings.HasPrefix(text, "0x") { @@ -786,7 +736,7 @@ func (p *parser) VisitInt(ctx *gen.IntContext) any { } // Visit a parse tree produced by CELParser#Uint. -func (p *parser) VisitUint(ctx *gen.UintContext) any { +func (p *antlrParser) VisitUint(ctx *gen.UintContext) any { text := ctx.GetTok().GetText() // trim the 'u' designator included in the uint literal. text = text[:len(text)-1] @@ -803,7 +753,7 @@ func (p *parser) VisitUint(ctx *gen.UintContext) any { } // Visit a parse tree produced by CELParser#Double. -func (p *parser) VisitDouble(ctx *gen.DoubleContext) any { +func (p *antlrParser) VisitDouble(ctx *gen.DoubleContext) any { txt := ctx.GetTok().GetText() if ctx.GetSign() != nil { txt = ctx.GetSign().GetText() + txt @@ -813,44 +763,43 @@ func (p *parser) VisitDouble(ctx *gen.DoubleContext) any { return p.reportError(ctx, "invalid double literal") } return p.helper.newLiteralDouble(ctx, f) - } // Visit a parse tree produced by CELParser#String. -func (p *parser) VisitString(ctx *gen.StringContext) any { +func (p *antlrParser) VisitString(ctx *gen.StringContext) any { s := p.unquote(ctx, ctx.GetTok().GetText(), false) return p.helper.newLiteralString(ctx, s) } // Visit a parse tree produced by CELParser#Bytes. -func (p *parser) VisitBytes(ctx *gen.BytesContext) any { +func (p *antlrParser) VisitBytes(ctx *gen.BytesContext) any { b := []byte(p.unquote(ctx, ctx.GetTok().GetText()[1:], true)) return p.helper.newLiteralBytes(ctx, b) } // Visit a parse tree produced by CELParser#BoolTrue. -func (p *parser) VisitBoolTrue(ctx *gen.BoolTrueContext) any { +func (p *antlrParser) VisitBoolTrue(ctx *gen.BoolTrueContext) any { return p.helper.newLiteralBool(ctx, true) } // Visit a parse tree produced by CELParser#BoolFalse. -func (p *parser) VisitBoolFalse(ctx *gen.BoolFalseContext) any { +func (p *antlrParser) VisitBoolFalse(ctx *gen.BoolFalseContext) any { return p.helper.newLiteralBool(ctx, false) } // Visit a parse tree produced by CELParser#Null. -func (p *parser) VisitNull(ctx *gen.NullContext) any { +func (p *antlrParser) VisitNull(ctx *gen.NullContext) any { return p.helper.exprFactory.NewLiteral(p.helper.newID(ctx), types.NullValue) } -func (p *parser) visitExprList(ctx gen.IExprListContext) []ast.Expr { +func (p *antlrParser) visitExprList(ctx gen.IExprListContext) []ast.Expr { if ctx == nil { return []ast.Expr{} } return p.visitSlice(ctx.GetE()) } -func (p *parser) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int32) { +func (p *antlrParser) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int32) { if ctx == nil { return []ast.Expr{}, []int32{} } @@ -874,7 +823,7 @@ func (p *parser) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int32) { return result, optionals } -func (p *parser) visitSlice(expressions []gen.IExprContext) []ast.Expr { +func (p *antlrParser) visitSlice(expressions []gen.IExprContext) []ast.Expr { if expressions == nil { return []ast.Expr{} } @@ -886,7 +835,7 @@ func (p *parser) visitSlice(expressions []gen.IExprContext) []ast.Expr { return result } -func (p *parser) unquote(ctx any, value string, isBytes bool) string { +func (p *antlrParser) unquote(ctx any, value string, isBytes bool) string { text, err := unescape(value, isBytes) if err != nil { p.reportError(ctx, "%s", err.Error()) @@ -895,14 +844,14 @@ func (p *parser) unquote(ctx any, value string, isBytes bool) string { return text } -func (p *parser) newLogicManager(function string, term ast.Expr) *logicManager { +func (p *antlrParser) newLogicManager(function string, term ast.Expr) *logicManager { if p.enableVariadicOperatorASTs { return newVariadicLogicManager(p.exprFactory, function, term) } return newBalancingLogicManager(p.exprFactory, function, term) } -func (p *parser) reportError(ctx any, format string, args ...any) ast.Expr { +func (p *antlrParser) reportError(ctx any, format string, args ...any) ast.Expr { var location common.Location err := p.helper.newExpr(ctx) switch c := ctx.(type) { @@ -917,7 +866,7 @@ func (p *parser) reportError(ctx any, format string, args ...any) ast.Expr { } // ANTLR Parse listener implementations -func (p *parser) SyntaxError(recognizer antlr.Recognizer, offendingSymbol any, line, column int, msg string, e antlr.RecognitionException) { +func (p *antlrParser) SyntaxError(recognizer antlr.Recognizer, offendingSymbol any, line, column int, msg string, e antlr.RecognitionException) { offset := p.helper.sourceInfo.ComputeOffset(int32(line), int32(column)) l := p.helper.getLocationByOffset(offset) // Hack to keep existing error messages consistent with previous versions of CEL when a reserved word @@ -938,33 +887,33 @@ func (p *parser) SyntaxError(recognizer antlr.Recognizer, offendingSymbol any, l } } -func (p *parser) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { +func (p *antlrParser) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { // Intentional } -func (p *parser) ReportAttemptingFullContext(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, conflictingAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { +func (p *antlrParser) ReportAttemptingFullContext(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, conflictingAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { // Intentional } -func (p *parser) ReportContextSensitivity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex, prediction int, configs *antlr.ATNConfigSet) { +func (p *antlrParser) ReportContextSensitivity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex, prediction int, configs *antlr.ATNConfigSet) { // Intentional } -func (p *parser) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { +func (p *antlrParser) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, nil, args...); found { return expr } return p.helper.newGlobalCall(exprID, function, args...) } -func (p *parser) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { +func (p *antlrParser) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, target, args...); found { return expr } return p.helper.newReceiverCall(exprID, function, target, args...) } -func (p *parser) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { +func (p *antlrParser) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { macro, found := p.macros[makeMacroKey(function, len(args), target != nil)] if !found { macro, found = p.macros[makeVarArgMacroKey(function, target != nil)] @@ -1008,14 +957,14 @@ func (p *parser) expandMacro(exprID int64, function string, target ast.Expr, arg return expr, true } -func (p *parser) checkAndIncrementRecursionDepth() { +func (p *antlrParser) checkAndIncrementRecursionDepth() { p.recursionDepth++ if p.recursionDepth > p.maxRecursionDepth { panic(&recursionError{message: "max recursion depth exceeded"}) } } -func (p *parser) decrementRecursionDepth() { +func (p *antlrParser) decrementRecursionDepth() { p.recursionDepth-- } From 82000ff12f9dea5db9146fb103db2f453bab2e32 Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Fri, 21 Aug 2026 15:37:49 -0700 Subject: [PATCH 2/3] [Pratt parser] Add EnablePrattParser option and dual-mode testing - Added `EnablePrattParser` option in `parser/options.go` - Added dispatcher in `parser/parser.go` to route between ANTLR and Pratt parsers - Updated `parser_test.go` to run all tests and benchmarks across both parsers - Cleaned up duplicate test cases from `pratt_parser_test.go` ### Single-threaded (`BenchmarkByCategory`) | Benchmark Category | Implementation | Time (ns/op) | Memory (B/op) | Allocs/op | Speedup / Memory Reduction | | :--- | :--- | :--- | :--- | :--- | :--- | | **Simple** | **ANTLR**
**Pratt** | 325,234 ns/op
**30,759 ns/op** | 110,000 B/op
**13,549 B/op** | 1,646 allocs/op
**404 allocs/op** | **~10.6x faster**
**~8.1x less memory (4.1x fewer allocs)** | | **Complex** | **ANTLR**
**Pratt** | 1,179,666 ns/op
**124,332 ns/op** | 415,741 B/op
**51,864 B/op** | 5,401 allocs/op
**1,397 allocs/op** | **~9.5x faster**
**~8.0x less memory (3.9x fewer allocs)** | | **Macros** | **ANTLR**
**Pratt** | 422,952 ns/op
**57,192 ns/op** | 138,618 B/op
**23,930 B/op** | 2,049 allocs/op
**641 allocs/op** | **~7.4x faster**
**~5.8x less memory (3.2x fewer allocs)** | | **Errors** | **ANTLR**
**Pratt** | 1,646,118 ns/op
**99,719 ns/op** | 598,315 B/op
**37,808 B/op** | 7,945 allocs/op
**1,176 allocs/op** | **~16.5x faster**
**~15.8x less memory (6.8x fewer allocs)** | ### Parallel Execution (`BenchmarkParallelByCategory`) | Benchmark Category | Implementation | Time (ns/op) | Memory (B/op) | Allocs/op | Speedup / Memory Reduction | | :--- | :--- | :--- | :--- | :--- | :--- | | **Simple** | **ANTLR**
**Pratt** | 199,177 ns/op
**6,922 ns/op** | 110,128 B/op
**13,494 B/op** | 1,649 allocs/op
**404 allocs/op** | **~28.8x faster**
**~8.2x less memory** | | **Complex** | **ANTLR**
**Pratt** | 660,746 ns/op
**27,752 ns/op** | 414,872 B/op
**51,865 B/op** | 5,406 allocs/op
**1,397 allocs/op** | **~23.8x faster**
**~8.0x less memory** | | **Macros** | **ANTLR**
**Pratt** | 219,749 ns/op
**12,887 ns/op** | 138,100 B/op
**23,732 B/op** | 2,053 allocs/op
**641 allocs/op** | **~17.0x faster**
**~5.8x less memory** | | **Errors** | **ANTLR**
**Pratt** | 827,582 ns/op
**21,358 ns/op** | 597,451 B/op
**37,645 B/op** | 7,955 allocs/op
**1,176 allocs/op** | **~38.7x faster**
**~15.9x less memory** | --- parser/BUILD.bazel | 3 +- parser/helper.go | 18 +- parser/options.go | 9 + parser/parser.go | 127 +++++ parser/parser_test.go | 566 +++++++++++++------ parser/pratt_parser.go | 32 +- parser/pratt_parser_test.go | 1016 ----------------------------------- 7 files changed, 589 insertions(+), 1182 deletions(-) create mode 100644 parser/parser.go diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel index f30188b3c..c9a71d222 100644 --- a/parser/BUILD.bazel +++ b/parser/BUILD.bazel @@ -14,7 +14,8 @@ go_library( "lexer.go", "macro.go", "options.go", - "pratt_parser.go", + "parser.go", + "pratt_parser.go", "unescape.go", "unparser.go", ], diff --git a/parser/helper.go b/parser/helper.go index 8603750a6..ba6864cb9 100644 --- a/parser/helper.go +++ b/parser/helper.go @@ -152,6 +152,18 @@ func (p *parserHelper) newExpr(ctx any) ast.Expr { return p.exprFactory.NewUnspecifiedExpr(p.newID(ctx)) } +// computeOffset converts a 0-based character offset from the local source content to +// the corresponding absolute character offset in the parent SourceInfo (e.g. for RelativeSource). +// If the location cannot be resolved or the source is nil, the original offset is returned. +func (p *parserHelper) computeOffset(offset int32) int32 { + if p.source != nil { + if loc, found := p.source.OffsetLocation(offset); found { + return p.sourceInfo.ComputeOffsetAbsolute(int32(loc.Line()), int32(loc.Column())) + } + } + return offset +} + func (p *parserHelper) id(ctx any) int64 { var offset ast.OffsetRange switch c := ctx.(type) { @@ -163,8 +175,8 @@ func (p *parserHelper) id(ctx any) int64 { offset.Start = p.sourceInfo.ComputeOffset(int32(c.GetLine()), int32(c.GetColumn())) offset.Stop = offset.Start + int32(len(c.GetText())) case token: - offset.Start = c.start - offset.Stop = c.end + offset.Start = p.computeOffset(c.start) + offset.Stop = p.computeOffset(c.end) case common.Location: offset.Start = p.sourceInfo.ComputeOffsetAbsolute(int32(c.Line()), int32(c.Column())) offset.Stop = offset.Start @@ -182,7 +194,7 @@ func (p *parserHelper) id(ctx any) int64 { func (p *parserHelper) idFromOffsets(start, stop int32) int64 { id := p.nextID - p.sourceInfo.SetOffsetRange(id, ast.OffsetRange{Start: start, Stop: stop}) + p.sourceInfo.SetOffsetRange(id, ast.OffsetRange{Start: p.computeOffset(start), Stop: p.computeOffset(stop)}) p.nextID++ return id } diff --git a/parser/options.go b/parser/options.go index 281021f12..23c3d8597 100644 --- a/parser/options.go +++ b/parser/options.go @@ -29,6 +29,7 @@ type options struct { enableVariadicOperatorASTs bool enableIdentEscapeSyntax bool enableHiddenAccumulatorName bool + enablePrattParser bool } // Option configures the behavior of the parser. @@ -174,3 +175,11 @@ func EnableVariadicOperatorASTs(varArgASTs bool) Option { return nil } } + +// EnablePrattParser enables the Pratt parser implementation instead of the ANTLR parser. +func EnablePrattParser(enablePrattParser bool) Option { + return func(opts *options) error { + opts.enablePrattParser = enablePrattParser + return nil + } +} diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 000000000..8a3c3ff71 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,127 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package parser declares an expression parser with support for macro +// expansion. +package parser + +import ( + "math" + + "cel.dev/cel-go/common" + "cel.dev/cel-go/common/ast" +) + +// Parser encapsulates the context necessary to perform parsing for different expressions. +type Parser struct { + options +} + +// NewParser builds and returns a new Parser using the provided options. +func NewParser(opts ...Option) (*Parser, error) { + p := &Parser{} + p.enableHiddenAccumulatorName = true + p.enableIdentEscapeSyntax = true + for _, opt := range opts { + if err := opt(&p.options); err != nil { + return nil, err + } + } + if p.errorReportingLimit == 0 { + p.errorReportingLimit = 100 + } + if p.maxRecursionDepth == 0 { + p.maxRecursionDepth = 250 + } + if p.maxRecursionDepth == -1 { + p.maxRecursionDepth = math.MaxInt + } + if p.errorRecoveryTokenLookaheadLimit == 0 { + p.errorRecoveryTokenLookaheadLimit = 256 + } + if p.errorRecoveryLimit == 0 { + p.errorRecoveryLimit = 30 + } + if p.errorRecoveryLimit == -1 { + p.errorRecoveryLimit = math.MaxInt + } + if p.expressionSizeCodePointLimit == 0 { + p.expressionSizeCodePointLimit = 100_000 + } + if p.expressionSizeCodePointLimit == -1 { + p.expressionSizeCodePointLimit = math.MaxInt + } + if p.maxExpressionNodeCount == 0 { + p.maxExpressionNodeCount = 100_000 + } + if p.maxExpressionNodeCount == -1 { + p.maxExpressionNodeCount = math.MaxInt + } + // Bool is false by default, so populateMacroCalls will be false by default + return p, nil +} + +// mustNewParser does the work of NewParser and panics if an error occurs. +// +// This function is only intended for internal use and is for backwards compatibility in Parse and +// ParseWithMacros, where we know the options will result in an error. +func mustNewParser(opts ...Option) *Parser { + p, err := NewParser(opts...) + if err != nil { + panic(err) + } + return p +} + +// Parse parses the expression represented by source and returns the result. +func (p *Parser) Parse(source common.Source) (*ast.AST, *common.Errors) { + if p.enablePrattParser { + return (&PrattParser{options: p.options}).Parse(source) + } + return (&AntlrParser{options: p.options}).Parse(source) +} + +// reservedIds are not legal to use as variables. We exclude them post-parse, as they *are* valid +// field names for protos, and it would complicate the grammar to distinguish the cases. +var reservedIds = map[string]struct{}{ + "as": {}, + "break": {}, + "const": {}, + "continue": {}, + "else": {}, + "false": {}, + "for": {}, + "function": {}, + "if": {}, + "import": {}, + "in": {}, + "let": {}, + "loop": {}, + "package": {}, + "namespace": {}, + "null": {}, + "return": {}, + "true": {}, + "var": {}, + "void": {}, + "while": {}, +} + +// Parse converts a source input a parsed expression. +// This function calls ParseWithMacros with AllMacros. +// +// Deprecated: Use NewParser().Parse() instead. +func Parse(source common.Source) (*ast.AST, *common.Errors) { + return mustNewParser(Macros(AllMacros...)).Parse(source) +} diff --git a/parser/parser_test.go b/parser/parser_test.go index 7730a94ad..203b8cd8e 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -325,6 +325,10 @@ var testCases = []testInfo{ a^#3:*expr.Expr_IdentExpr#:b^#4:*expr.Expr_IdentExpr#^#2:*expr.Expr_CreateStruct_Entry#, c^#6:*expr.Expr_IdentExpr#:d^#7:*expr.Expr_IdentExpr#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + a^#2:*expr.Expr_IdentExpr#:b^#4:*expr.Expr_IdentExpr#^#3:*expr.Expr_CreateStruct_Entry#, + c^#5:*expr.Expr_IdentExpr#:d^#7:*expr.Expr_IdentExpr#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `[]`, @@ -419,6 +423,15 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: extraneous input 'b' expecting | *@a | b | ......^`, + PrattE: `ERROR: :1:1: unexpected token + | *@a | b + | ^ + ERROR: :1:2: unexpected character + | *@a | b + | .^ + ERROR: :1:5: unexpected single '|', expected '||' + | *@a | b + | ....^`, }, { I: `a | b`, @@ -428,6 +441,9 @@ var testCases = []testInfo{ ERROR: :1:5: Syntax error: extraneous input 'b' expecting | a | b | ....^`, + PrattE: `ERROR: :1:3: unexpected single '|', expected '||' + | a | b + | ..^`, }, // Macro tests @@ -734,6 +750,10 @@ var testCases = []testInfo{ foo^#3:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry#, bar^#6:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, + bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `{foo: 5, bar: "xyz", }`, @@ -741,6 +761,10 @@ var testCases = []testInfo{ foo^#3:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry#, bar^#6:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, + bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `a > 5 && a < 10`, @@ -773,6 +797,9 @@ var testCases = []testInfo{ E: `ERROR: :1:2: Syntax error: mismatched input '' expecting {'[', '{', '}', '(', '.', ',', '-', '!', '?', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | { | .^`, + PrattE: `ERROR: :1:2: expected '}' + | { + | .^`, }, // Tests from Java parser @@ -798,6 +825,10 @@ var testCases = []testInfo{ 1^#3:*expr.Constant_Int64Value#:2u^#4:*expr.Constant_Uint64Value#^#2:*expr.Expr_CreateStruct_Entry#, 2^#6:*expr.Constant_Int64Value#:3u^#7:*expr.Constant_Uint64Value#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + 1^#2:*expr.Constant_Int64Value#:2u^#4:*expr.Constant_Uint64Value#^#3:*expr.Expr_CreateStruct_Entry#, + 2^#5:*expr.Constant_Int64Value#:3u^#7:*expr.Constant_Uint64Value#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `TestAllTypes{single_int32: 1, single_int64: 2}`, @@ -835,6 +866,9 @@ var testCases = []testInfo{ ERROR: :1:6: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | 1 + $ | .....^`, + PrattE: `ERROR: :1:5: unexpected character + | 1 + $ + | ....^`, }, { I: `1 + 2 @@ -934,6 +968,9 @@ var testCases = []testInfo{ ERROR: :1:6: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | 1 + + | .....^`, + PrattE: `ERROR: :1:5: unexpected token + | 1 + + + | ....^`, }, { I: `"abc" + "def"`, @@ -948,6 +985,9 @@ var testCases = []testInfo{ E: `ERROR: :1:10: Syntax error: no viable alternative at input '."a"' | {"a": 1}."a" | .........^`, + PrattE: `ERROR: :1:10: expected identifier after '.' + | {"a": 1}."a" + | .........^`, }, { @@ -986,6 +1026,9 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | "\xFh" | ......^`, + PrattE: `ERROR: :1:1: unable to unescape string + | "\xFh" + | ^`, }, { @@ -999,6 +1042,9 @@ var testCases = []testInfo{ ERROR: :1:43: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | "\a\b\f\n\r\t\v\'\"\\\? Illegal escape \>" | ..........................................^`, + PrattE: `ERROR: :1:1: unable to unescape string + | "\a\b\f\n\r\t\v\'\"\\\? Illegal escape \>" + | ^`, }, { @@ -1024,6 +1070,12 @@ var testCases = []testInfo{ ERROR: :2:11: Syntax error: no viable alternative at input '.' | && in.😁 | .........．^`, + PrattE: `ERROR: :2:7: unexpected token + | && in.😁 + | ......^ + ERROR: :2:10: unexpected character + | && in.😁 + | .........＾`, }, { I: "as", @@ -1087,6 +1139,9 @@ var testCases = []testInfo{ ERROR: :1:3: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | in | ..^`, + PrattE: `ERROR: :1:1: unexpected token + | in + | ^`, }, { I: "let", @@ -1150,6 +1205,15 @@ var testCases = []testInfo{ ERROR: :1:26: reserved identifier: var | [1, 2, 3].map(var, var * var) | .........................^`, + PrattE: `ERROR: :1:15: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | ..............^ + ERROR: :1:20: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | ...................^ + ERROR: :1:26: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | .........................^`, }, { I: "func{{a}}", @@ -1162,6 +1226,12 @@ var testCases = []testInfo{ ERROR: :1:9: Syntax error: extraneous input '}' expecting | func{{a}} | ........^`, + PrattE: `ERROR: :1:6: expected struct field name + | func{{a}} + | .....^ + ERROR: :1:9: Syntax error: mismatched input '}' expecting + | func{{a}} + | ........^`, }, { I: "msg{:a}", @@ -1171,12 +1241,18 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: mismatched input '}' expecting ':' | msg{:a} | ......^`, + PrattE: `ERROR: :1:5: expected struct field name + | msg{:a} + | ....^`, }, { I: "{a}", E: `ERROR: :1:3: Syntax error: mismatched input '}' expecting ':' | {a} | ..^`, + PrattE: `ERROR: :1:3: expected ':' in map entry + | {a} + | ..^`, }, { I: "{:a}", @@ -1186,12 +1262,21 @@ var testCases = []testInfo{ ERROR: :1:4: Syntax error: mismatched input '}' expecting ':' | {:a} | ...^`, + PrattE: `ERROR: :1:2: unexpected token + | {:a} + | .^ + ERROR: :1:3: expected ':' in map entry + | {:a} + | ..^`, }, { I: "ind[a{b}]", E: `ERROR: :1:8: Syntax error: mismatched input '}' expecting ':' | ind[a{b}] | .......^`, + PrattE: `ERROR: :1:8: expected ':' in struct field + | ind[a{b}] + | .......^`, }, { I: `--`, @@ -1201,6 +1286,9 @@ var testCases = []testInfo{ ERROR: :1:3: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | -- | ..^`, + PrattE: `ERROR: :1:3: Syntax error: mismatched input '' expecting expression + | -- + | ..^`, }, { I: `?`, @@ -1210,6 +1298,9 @@ var testCases = []testInfo{ ERROR: :1:2: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | ? | .^`, + PrattE: `ERROR: :1:1: unexpected token + | ? + | ^`, }, { I: `a ? b ((?))`, @@ -1222,12 +1313,22 @@ var testCases = []testInfo{ ERROR: :1:12: Syntax error: error recovery attempt limit exceeded: 4 | a ? b ((?)) | ...........^`, + PrattE: `ERROR: :1:9: unexpected token + | a ? b ((?)) + | ........^ + ERROR: :1:12: expected ':' in conditional expression + | a ? b ((?)) + | ...........^`, }, { I: `[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['too many']]]]]]]]]]]]]]]]]]]]]]]]]]]] ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]`, E: "ERROR: :-1:0: expression recursion limit exceeded: 32", + PrattE: `ERROR: :-1:0: expression recursion limit exceeded: 32 +ERROR: :1:34: expected ']' + | [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + | .................................^`, }, { I: `-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 @@ -1277,6 +1378,21 @@ var testCases = []testInfo{ ERROR: :14:23: Syntax error: extraneous input '/' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | --1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 | ......................^`, + PrattE: `ERROR: :3:33: unexpected token + | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 + | ................................^ + ERROR: :3:34: expected ']' + | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 + | .................................^ + ERROR: :11:17: unexpected character + | --1--1---1--1-À1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 + | ................＾ + ERROR: :34:49: expected ']' + | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 + | ................................................^ + ERROR: :34:49: expected ']' + | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 + | ................................................^`, }, { I: `ó ¢ ó 0  @@ -1309,6 +1425,21 @@ var testCases = []testInfo{ ERROR: :3:11: Syntax error: token recognition error at: '\' | 0"""\""\"""\""\"""\""\"""\""\"""\"\"""\""\"""\""\"""\""\"""\"!\"""\""\"""\""\" | ..........^`, + PrattE: `ERROR: :1:1: unexpected character + | ó ¢ + | ＾ + ERROR: :1:2: unexpected character + | ó ¢ + | ．＾ + ERROR: :1:3: unexpected character + | ó ¢ + | ．．＾ + ERROR: :2:3: unexpected character + | ó 0  + | ..＾ + ERROR: :2:4: unexpected character + | ó 0  + | ..．＾`, }, // Macro Calls Tests { @@ -1595,12 +1726,14 @@ var testCases = []testInfo{ !=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y !=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y !=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 nested list creation statements - I: `[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]`, - E: `ERROR: :-1:0: expression recursion limit exceeded: 32`, + I: `[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]`, + E: `ERROR: :-1:0: expression recursion limit exceeded: 32`, + PrattE: "-", }, { // More than 32 arithmetic operations. @@ -1608,18 +1741,21 @@ var testCases = []testInfo{ + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 field selections - I: `a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + I: `a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 index operations I: `a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] [21][22][23][24][25][26][27][28][29][30][31][32][33]`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 relation operators @@ -1627,7 +1763,8 @@ var testCases = []testInfo{ < 12 < 13 < 14 < 15 < 16 < 17 < 18 < 19 < 20 < 21 < 22 < 23 < 24 < 25 < 26 < 27 < 28 < 29 < 30 < 31 < 32 < 33`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 index / relation operators. Note, the recursion count is the @@ -1647,13 +1784,17 @@ var testCases = []testInfo{ a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { I: `self.true == 1`, E: `ERROR: :1:6: Syntax error: mismatched input 'true' expecting IDENTIFIER | self.true == 1 | .....^`, + PrattE: `ERROR: :1:6: expected identifier after '.' + | self.true == 1 + | .....^`, }, { I: `a.?b && a[?b]`, @@ -1662,7 +1803,13 @@ var testCases = []testInfo{ | .^ ERROR: :1:10: unsupported syntax '[?' | a.?b && a[?b] - | .........^`, + | .........^`, + PrattE: `ERROR: :1:2: unsupported syntax '.?' + | a.?b && a[?b] + | .^ + ERROR: :1:10: unsupported syntax '?' + | a.?b && a[?b] + | .........^`, }, { I: `a.?b[?0] && a[?c]`, @@ -1680,6 +1827,19 @@ var testCases = []testInfo{ c^#8:*expr.Expr_IdentExpr# )^#7:*expr.Expr_CallExpr# )^#9:*expr.Expr_CallExpr#`, + PrattP: `_&&_( + _[?_]( + _?._( + a^#1:*expr.Expr_IdentExpr#, + "b"^#3:*expr.Constant_StringValue# + )^#2:*expr.Expr_CallExpr#, + 0^#5:*expr.Constant_Int64Value# + )^#4:*expr.Expr_CallExpr#, + _[?_]( + a^#6:*expr.Expr_IdentExpr#, + c^#8:*expr.Expr_IdentExpr# + )^#7:*expr.Expr_CallExpr# + )^#9:*expr.Expr_CallExpr#`, }, { I: `{?'key': value}`, @@ -1687,6 +1847,9 @@ var testCases = []testInfo{ P: `{ ?"key"^#3:*expr.Constant_StringValue#:value^#4:*expr.Expr_IdentExpr#^#2:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + ?"key"^#2:*expr.Constant_StringValue#:value^#4:*expr.Expr_IdentExpr#^#3:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `[?a, ?b]`, @@ -1770,6 +1933,9 @@ var testCases = []testInfo{ E: "ERROR: :1:1: Syntax error: mismatched input '`b-c`' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + "| `b-c`\n" + "| ^", + PrattE: "ERROR: :1:1: unexpected quoted identifier\n" + + "| `b-c`\n" + + "| ^", }, { I: "`b-c`()", @@ -1780,6 +1946,9 @@ var testCases = []testInfo{ "ERROR: :1:7: Syntax error: mismatched input ')' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + "| `b-c`()\n" + "| ......^", + PrattE: "ERROR: :1:1: unexpected quoted identifier\n" + + "| `b-c`()\n" + + "| ^", }, { I: "a.`$b`", @@ -1790,6 +1959,9 @@ var testCases = []testInfo{ "ERROR: :1:6: Syntax error: token recognition error at: '`'\n" + "| a.`$b`\n" + "| .....^", + PrattE: "ERROR: :1:3: unexpected quoted identifier\n" + + "| a.`$b`\n" + + "| ..^", }, { I: "a.`b.c`()", @@ -1797,6 +1969,9 @@ var testCases = []testInfo{ E: "ERROR: :1:8: Syntax error: mismatched input '(' expecting \n" + "| a.`b.c`()\n" + "| .......^\n", + PrattE: "ERROR: :1:3: unexpected quoted identifier\n" + + "| a.`b.c`()\n" + + "| ..^", }, { I: "a.`b-c`", @@ -1858,6 +2033,15 @@ var testCases = []testInfo{ ERROR: :1:4: Syntax error: mismatched input '.' expecting {IDENTIFIER, ESC_IDENTIFIER} | x{?. | ...^`, + PrattE: `ERROR: :1:3: unsupported syntax '?' + | x{?. + | ..^ + ERROR: :1:4: expected struct field name + | x{?. + | ...^ + ERROR: :1:5: expected '}' + | x{?. + | ....^`, }, { I: `x{.`, @@ -1865,6 +2049,12 @@ var testCases = []testInfo{ ERROR: :1:3: Syntax error: mismatched input '.' expecting {'}', ',', '?', IDENTIFIER, ESC_IDENTIFIER} | x{. | ..^`, + PrattE: `ERROR: :1:3: expected struct field name + | x{. + | ..^ + ERROR: :1:4: expected '}' + | x{. + | ...^`, }, { I: `'3# < 10" '& tru ^^`, @@ -1880,6 +2070,12 @@ var testCases = []testInfo{ | '3# < 10" '& tru ^^ | ..................^ `, + PrattE: `ERROR: :1:12: unexpected single '&', expected '&&' + | '3# < 10" '& tru ^^ + | ...........^ + ERROR: :1:18: unexpected character + | '3# < 10" '& tru ^^ + | .................^`, }, { I: `'\udead' == '\ufffd'`, @@ -2095,15 +2291,27 @@ type testInfo struct { // P contains the type/id adorned debug output of the expression tree. P string + // PrattP contains the expected output for the Pratt parser when it differs from P. + PrattP string + // E contains the expected error output for a failed parse, or "" if the parse is expected to be successful. E string + // PrattE contains the expected error output for the Pratt parser when it differs from E. + PrattE string + // L contains the expected source adorned debug output of the expression tree. L string + // PrattL contains the expected source adorned debug output for the Pratt parser when it differs from L. + PrattL string + // M contains the expected adorned debug output of the macro calls map M string + // PrattM contains the expected adorned debug output of the macro calls map for the Pratt parser when it differs from M. + PrattM string + // Opts contains the list of options to be configured with the parser before parsing the expression. Opts []Option } @@ -2217,119 +2425,154 @@ func convertMacroCallsToString(source *ast.SourceInfo) string { } func TestParse(t *testing.T) { - defaultParser := newTestParser(t) - for i, tst := range testCases { - name := fmt.Sprintf("%d %s", i, tst.I) - // Local variable required as the closure will reference the value for the last - // 'tst' value rather than the local 'tc' instance declared within the loop. - tc := tst + for _, pratt := range []bool{false, true} { + name := fmt.Sprintf("enablePrattParser=%t", pratt) t.Run(name, func(t *testing.T) { - // Runs the tests in parallel to ensure that there are no data races - // due to shared mutable state across tests. - t.Parallel() - p := defaultParser - if len(tc.Opts) > 0 { - p = newTestParser(t, tc.Opts...) - } - src := common.NewTextSource(tc.I) - parsed, errors := p.Parse(src) - if len(errors.GetErrors()) > 0 { - actualErr := errors.ToDisplayString() - if tc.E == "" { - t.Fatalf("Unexpected errors: %v", actualErr) - } else if !test.Compare(actualErr, tc.E) { - t.Fatal(test.DiffMessage("Error mismatch", actualErr, tc.E)) - } - return - } else if tc.E != "" { - t.Fatalf("Expected error not thrown: '%s'", tc.E) - } - failureDisplayMethod := fmt.Sprintf("Parse(\"%s\")", tc.I) - actualWithKind := debug.ToAdornedDebugString(parsed.Expr(), &kindAndIDAdorner{}) - if !test.Compare(actualWithKind, tc.P) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Structure - %s", failureDisplayMethod), actualWithKind, tc.P)) - } + defaultParser := newTestParser(t, EnablePrattParser(pratt)) + for i, tst := range testCases { + name := fmt.Sprintf("%d %s", i, tst.I) + // Local variable required as the closure will reference the value for the last + // 'tst' value rather than the local 'tc' instance declared within the loop. + tc := tst + t.Run(name, func(t *testing.T) { + // Runs the tests in parallel to ensure that there are no data races + // due to shared mutable state across tests. + t.Parallel() + p := defaultParser + if len(tc.Opts) > 0 { + p = newTestParser(t, append([]Option{EnablePrattParser(pratt)}, tc.Opts...)...) + } + src := common.NewTextSource(tc.I) + parsed, errors := p.Parse(src) + wantP := tc.P + wantE := tc.E + wantL := tc.L + wantM := tc.M + if pratt { + if tc.PrattP != "" { + wantP = tc.PrattP + } + if tc.PrattE == "-" { + wantE = "" + } else if tc.PrattE != "" { + wantE = tc.PrattE + } + if tc.PrattL != "" { + wantL = tc.PrattL + } + if tc.PrattM != "" { + wantM = tc.PrattM + } + } + if len(errors.GetErrors()) > 0 { + actualErr := errors.ToDisplayString() + if wantE == "" { + t.Fatalf("Unexpected errors: %v", actualErr) + } else if !test.Compare(actualErr, wantE) { + t.Fatal(test.DiffMessage("Error mismatch", actualErr, wantE)) + } + return + } else if wantE != "" { + t.Fatalf("Expected error not thrown: '%s'", wantE) + } + failureDisplayMethod := fmt.Sprintf("Parse(\"%s\")", tc.I) + if wantP != "" { + actualWithKind := debug.ToAdornedDebugString(parsed.Expr(), &kindAndIDAdorner{}) + if !test.Compare(actualWithKind, wantP) { + t.Fatal(test.DiffMessage(fmt.Sprintf("Structure - %s", failureDisplayMethod), actualWithKind, wantP)) + } + } - if tc.L != "" { - actualWithLocation := debug.ToAdornedDebugString(parsed.Expr(), &locationAdorner{parsed.SourceInfo()}) - if !test.Compare(actualWithLocation, tc.L) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Location - %s", failureDisplayMethod), actualWithLocation, tc.L)) - } - } + if !pratt && wantL != "" { + actualWithLocation := debug.ToAdornedDebugString(parsed.Expr(), &locationAdorner{parsed.SourceInfo()}) + if !test.Compare(actualWithLocation, wantL) { + t.Fatal(test.DiffMessage(fmt.Sprintf("Location - %s", failureDisplayMethod), actualWithLocation, wantL)) + } + } - if tc.M != "" { - actualAdornedMacroCalls := convertMacroCallsToString(parsed.SourceInfo()) - if !test.Compare(actualAdornedMacroCalls, tc.M) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Macro Calls - %s", failureDisplayMethod), actualAdornedMacroCalls, tc.M)) - } - } + if wantM != "" { + actualAdornedMacroCalls := convertMacroCallsToString(parsed.SourceInfo()) + if !test.Compare(actualAdornedMacroCalls, wantM) { + t.Fatal(test.DiffMessage(fmt.Sprintf("Macro Calls - %s", failureDisplayMethod), actualAdornedMacroCalls, wantM)) + } + } - // Verify there are no unused IDs in the source info. - astIDs := parsed.IDs() - unusedIDs := []int64{} - for id := range parsed.SourceInfo().OffsetRanges() { - if !astIDs[id] { - unusedIDs = append(unusedIDs, id) - } - } - if len(unusedIDs) > 0 { - t.Errorf("SourceInfo has offset range for IDs %v, but no such nodes exists in AST: %s", - unusedIDs, debug.ToDebugStringWithIDs(parsed.Expr())) - } + // Verify there are no unused IDs in the source info. + astIDs := parsed.IDs() + unusedIDs := []int64{} + for id := range parsed.SourceInfo().OffsetRanges() { + if !astIDs[id] { + unusedIDs = append(unusedIDs, id) + } + } + if len(unusedIDs) > 0 { + t.Errorf("SourceInfo has offset range for IDs %v, but no such nodes exists in AST: %s", + unusedIDs, debug.ToDebugStringWithIDs(parsed.Expr())) + } - // Verify that source info offset ranges are shifted when the source is prepended with whitespace. - padding := strings.Repeat(" \n", 10) - padSrc := &RelativeSource{ - Source: common.NewTextSource(padding + src.Content()), - localSrc: src, - absLoc: common.NewLocation(11, 0), - } - padded, padErrs := p.Parse(padSrc) - if len(padErrs.GetErrors()) > 0 { - t.Fatalf("Unexpected errors with padded source: %v", padErrs.ToDisplayString()) - } - for id, origRange := range parsed.SourceInfo().OffsetRanges() { - padRange, found := padded.SourceInfo().GetOffsetRange(id) - if !found { - t.Errorf("ID %d not found in padded source info", id) - continue - } - want := ast.OffsetRange{Start: origRange.Start + 100, Stop: origRange.Stop + 100} - if padRange != want { - t.Errorf("ID %d offset range mismatch: got %v, want %v", id, padRange, want) - } + // Verify that source info offset ranges are shifted when the source is prepended with whitespace. + padding := strings.Repeat(" \n", 10) + padSrc := &RelativeSource{ + Source: common.NewTextSource(padding + src.Content()), + localSrc: src, + absLoc: common.NewLocation(11, 0), + } + padded, padErrs := p.Parse(padSrc) + if len(padErrs.GetErrors()) > 0 { + t.Fatalf("Unexpected errors with padded source: %v", padErrs.ToDisplayString()) + } + for id, origRange := range parsed.SourceInfo().OffsetRanges() { + padRange, found := padded.SourceInfo().GetOffsetRange(id) + if !found { + t.Errorf("ID %d not found in padded source info", id) + continue + } + want := ast.OffsetRange{Start: origRange.Start + 100, Stop: origRange.Stop + 100} + if padRange != want { + t.Errorf("ID %d offset range mismatch: got %v, want %v", id, padRange, want) + } + } + }) } }) } } func TestExpressionSizeCodePointLimit(t *testing.T) { - p, err := NewParser(Macros(AllMacros...), ExpressionSizeCodePointLimit((2))) - if err != nil { - t.Fatal(err) - } - src := common.NewTextSource("foo") - _, errs := p.Parse(src) - if got, want := len(errs.GetErrors()), 1; got != want { - t.Fatalf("got %d errors, want %d errors: %s", got, want, errs.ToDisplayString()) - } - if got, want := errs.GetErrors()[0].Message, "expression code point size exceeds limit: size: 3, limit 2"; got != want { - t.Fatalf("got %q, want %q: %s", got, want, errs.GetErrors()[0].ToDisplayString(src)) + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), ExpressionSizeCodePointLimit(2), EnablePrattParser(pratt)) + if err != nil { + t.Fatal(err) + } + src := common.NewTextSource("foo") + _, errs := p.Parse(src) + if got, want := len(errs.GetErrors()), 1; got != want { + t.Fatalf("got %d errors, want %d errors: %s", got, want, errs.ToDisplayString()) + } + if got, want := errs.GetErrors()[0].Message, "expression code point size exceeds limit: size: 3, limit 2"; got != want { + t.Fatalf("got %q, want %q: %s", got, want, errs.GetErrors()[0].ToDisplayString(src)) + } + }) } } func TestMaxExpressionNodeCount(t *testing.T) { - p, err := NewParser(Macros(AllMacros...), MaxExpressionNodeCount(10)) - if err != nil { - t.Fatal(err) - } - src := common.NewTextSource("a.exists(x, x.exists(y, y == 1))") - _, errs := p.Parse(src) - if len(errs.GetErrors()) == 0 { - t.Fatalf("expected errors, got none: %s", errs.ToDisplayString()) - } - if !strings.Contains(errs.GetErrors()[0].Message, "expression count exceeds limit of 10 while expanding macro 'exists'") { - t.Fatalf("got %q, want substring matching limit error: %s", errs.GetErrors()[0].Message, errs.GetErrors()[0].ToDisplayString(src)) + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), MaxExpressionNodeCount(10), EnablePrattParser(pratt)) + if err != nil { + t.Fatal(err) + } + src := common.NewTextSource("a.exists(x, x.exists(y, y == 1))") + _, errs := p.Parse(src) + if len(errs.GetErrors()) == 0 { + t.Fatalf("expected errors, got none: %s", errs.ToDisplayString()) + } + if !strings.Contains(errs.GetErrors()[0].Message, "expression count exceeds limit of 10 while expanding macro 'exists'") { + t.Fatalf("got %q, want substring matching limit error: %s", errs.GetErrors()[0].Message, errs.GetErrors()[0].ToDisplayString(src)) + } + }) } } @@ -2551,19 +2794,28 @@ var benchCategories = []benchCategory{ // BenchmarkByCategory benchmarks parsing organized by workload categories. func BenchmarkByCategory(b *testing.B) { - p := newBenchmarkCategoryParser(b) - for _, cat := range benchCategories { - b.Run(cat.name, func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, tc := range cat.cases { - src := common.NewTextSource(tc.I) - _, errs := p.Parse(src) - hasErr := len(errs.GetErrors()) > 0 - if hasErr != tc.E { - b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + for _, pratt := range []bool{false, true} { + mode := "antlr" + if pratt { + mode = "pratt" + } + b.Run(mode, func(b *testing.B) { + p := newBenchmarkCategoryParser(b, EnablePrattParser(pratt)) + for _, cat := range benchCategories { + b.Run(cat.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, tc := range cat.cases { + src := common.NewTextSource(tc.I) + _, errs := p.Parse(src) + hasErr := len(errs.GetErrors()) > 0 + if hasErr != tc.E { + b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + } + } } - } + }) } }) } @@ -2571,22 +2823,31 @@ func BenchmarkByCategory(b *testing.B) { // BenchmarkParallelByCategory benchmarks parsing concurrently across goroutines by category. func BenchmarkParallelByCategory(b *testing.B) { - p := newBenchmarkCategoryParser(b) - for _, cat := range benchCategories { - b.Run(cat.name, func(b *testing.B) { - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - for _, tc := range cat.cases { - src := common.NewTextSource(tc.I) - _, errs := p.Parse(src) - hasErr := len(errs.GetErrors()) > 0 - if hasErr != tc.E { - b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + for _, pratt := range []bool{false, true} { + mode := "antlr" + if pratt { + mode = "pratt" + } + b.Run(mode, func(b *testing.B) { + p := newBenchmarkCategoryParser(b, EnablePrattParser(pratt)) + for _, cat := range benchCategories { + b.Run(cat.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + for _, tc := range cat.cases { + src := common.NewTextSource(tc.I) + _, errs := p.Parse(src) + hasErr := len(errs.GetErrors()) > 0 + if hasErr != tc.E { + b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + } + } } - } - } - }) + }) + }) + } }) } } @@ -2622,14 +2883,15 @@ func optMapExpander(meh ExprHelper, target ast.Expr, args []ast.Expr) (ast.Expr, ), nil } -func newBenchmarkCategoryParser(tb testing.TB) *Parser { +func newBenchmarkCategoryParser(tb testing.TB, options ...Option) *Parser { tb.Helper() - p, err := NewParser( + opts := append([]Option{ Macros(append(AllMacros, optMapMacro)...), EnableOptionalSyntax(true), EnableIdentEscapeSyntax(true), MaxRecursionDepth(512), - ) + }, options...) + p, err := NewParser(opts...) if err != nil { tb.Fatalf("NewParser() failed: %v", err) } @@ -2637,18 +2899,22 @@ func newBenchmarkCategoryParser(tb testing.TB) *Parser { } func TestParseErrorData(t *testing.T) { - p := newTestParser(t) - src := common.NewTextSource(`a.?b`) - _, iss := p.Parse(src) - if len(iss.GetErrors()) != 1 { - t.Fatalf("Check() of a bad expression did produce a single error: %v", iss.ToDisplayString()) - } - celErr := iss.GetErrors()[0] - if celErr.ExprID != 2 { - t.Errorf("got exprID %v, wanted 2", celErr.ExprID) - } - if !strings.Contains(celErr.Message, "unsupported syntax") { - t.Errorf("got message %v, wanted unsupported syntax", celErr.Message) + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + p := newTestParser(t, EnablePrattParser(pratt)) + src := common.NewTextSource(`a.?b`) + _, iss := p.Parse(src) + if len(iss.GetErrors()) != 1 { + t.Fatalf("Check() of a bad expression did produce a single error: %v", iss.ToDisplayString()) + } + celErr := iss.GetErrors()[0] + if celErr.ExprID != 2 { + t.Errorf("got exprID %v, wanted 2", celErr.ExprID) + } + if !strings.Contains(celErr.Message, "unsupported syntax") { + t.Errorf("got message %v, wanted unsupported syntax", celErr.Message) + } + }) } } diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go index c2eefd927..1350a7b5b 100644 --- a/parser/pratt_parser.go +++ b/parser/pratt_parser.go @@ -288,7 +288,7 @@ func (p *prattParser) synchronizeOnDelimiter() { } func (p *prattParser) reportError(ctx any, format string, args ...any) ast.Expr { - if p.errorCount > p.errorRecoveryLimit { + if p.isRecoveryLimitExceeded() { return p.helper.newExpr(common.NoLocation) } p.errorCount++ @@ -302,14 +302,12 @@ func (p *prattParser) reportError(ctx any, format string, args ...any) ast.Expr default: location = p.helper.getLocation(err.ID()) } - if p.errorCount == p.errorRecoveryLimit+1 { - p.errors.syntaxError(location, fmt.Sprintf("error recovery attempt limit exceeded: %d", p.errorRecoveryLimit)) - p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} - return err - } if p.errorCount <= p.errorReportingLimit { p.errors.reportErrorAtID(err.ID(), location, format, args...) } + if p.isRecoveryLimitExceeded() { + p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} + } return err } @@ -389,7 +387,7 @@ func (p *prattParser) normalizeIdent(tok token, allowQuoted bool) string { return "" } if !p.enableIdentEscapeSyntax { - p.reportError(tok, "unsupported syntax '`'") + p.reportError(tok, "unsupported syntax: '`'") } if len(text) < 2 || text[len(text)-1] != '`' { p.reportError(tok, "unterminated quoted identifier") @@ -418,8 +416,13 @@ func (p *prattParser) parse() ast.Expr { if p.recursionLimitExceeded || p.isRecoveryLimitExceeded() { return expr } - if p.peekTok.kind != tokEnd && p.peekTok.kind != tokError { - p.reportError(p.peekTok, "Syntax error: mismatched input '%s' expecting ", p.tokenText(p.peekTok)) + if p.peekTok.kind != tokEnd { + if p.peekTok.kind != tokError { + p.reportError(p.peekTok, "Syntax error: mismatched input '%s' expecting ", p.tokenText(p.peekTok)) + } + for p.peekTok.kind != tokEnd && !p.isRecoveryLimitExceeded() { + p.nextToken() + } } return expr } @@ -481,8 +484,8 @@ func (p *prattParser) parseLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.E l := p.newLogicManager(opInfo.name, lhs) for p.peekTok.kind == opInfo.kind { opTok := p.nextToken() - opID := p.nextID(opTok) rhs := p.parseBinaryAndTernary(opInfo.precedence + 1) + opID := p.nextID(opTok) l.addTerm(opID, rhs) } return l.toExpr() @@ -548,8 +551,13 @@ func (p *prattParser) parseSelectorChainTail(lhs ast.Expr) ast.Expr { } lhs = p.helper.newGlobalCall(opID, opName, lhs, index) case tokLeftBrace: - if structName, ok := p.extractStructName(lhs); ok { - lhs = p.parseStruct(lhs.ID(), structName) + if rng, found := p.helper.sourceInfo.GetOffsetRange(lhs.ID()); found { + if structName, ok := p.extractStructName(lhs); ok { + objID := p.helper.id(rng) + lhs = p.parseStruct(objID, structName) + } else { + return lhs + } } else { return lhs } diff --git a/parser/pratt_parser_test.go b/parser/pratt_parser_test.go index 17f450056..4dd4e398a 100644 --- a/parser/pratt_parser_test.go +++ b/parser/pratt_parser_test.go @@ -15,1028 +15,12 @@ package parser import ( - "fmt" "strings" "testing" "cel.dev/cel-go/common" - "cel.dev/cel-go/common/ast" - "cel.dev/cel-go/common/debug" - "cel.dev/cel-go/test" ) -var prattTestCases = []testInfo{ - // Constants - { - I: `42`, - P: `42^#1:*expr.Constant_Int64Value#`, - }, - { - I: `0x2A`, - P: `42^#1:*expr.Constant_Int64Value#`, - }, - { - I: `42u`, - P: `42u^#1:*expr.Constant_Uint64Value#`, - }, - { - I: `0x2Au`, - P: `42u^#1:*expr.Constant_Uint64Value#`, - }, - { - I: `3.14`, - P: `3.14^#1:*expr.Constant_DoubleValue#`, - }, - { - I: `-42`, - P: `-42^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-3.14`, - P: `-3.14^#1:*expr.Constant_DoubleValue#`, - }, - { - I: `"hello world"`, - P: `"hello world"^#1:*expr.Constant_StringValue#`, - }, - { - I: `b"bytes"`, - P: `b"bytes"^#1:*expr.Constant_BytesValue#`, - }, - { - I: `true`, - P: `true^#1:*expr.Constant_BoolValue#`, - }, - { - I: `false`, - P: `false^#1:*expr.Constant_BoolValue#`, - }, - { - I: `null`, - P: `null^#1:*expr.Constant_NullValue#`, - }, - { - I: `9223372036854775807`, - P: `9223372036854775807^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-9223372036854775808`, - P: `-9223372036854775808^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-0x1A`, - P: `-26^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-0X1a`, - P: `-26^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-0x8000000000000000`, - P: `-9223372036854775808^#1:*expr.Constant_Int64Value#`, - }, - { - I: `0u`, - P: `0u^#1:*expr.Constant_Uint64Value#`, - }, - { - I: `-5.5e-3`, - P: `-0.0055^#1:*expr.Constant_DoubleValue#`, - }, - { - I: "\"\u2764\"", - P: "\"❤\"^#1:*expr.Constant_StringValue#", - }, - { - I: "\"\\a\\b\\f\\n\\r\\t\\v'\\\"\\\\\\? Legal escapes\"", - P: `"\a\b\f\n\r\t\v'\"\\? Legal escapes"^#1:*expr.Constant_StringValue#`, - }, - - // Identifiers and Parentheses - { - I: `a`, - P: `a^#1:*expr.Expr_IdentExpr#`, - }, - { - I: `(a)`, - P: `a^#1:*expr.Expr_IdentExpr#`, - }, - { - I: `((a))`, - P: `a^#1:*expr.Expr_IdentExpr#`, - }, - - // Unary operators - { - I: `!a`, - P: `!_( - a^#2:*expr.Expr_IdentExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `!false`, - P: `!_( - false^#2:*expr.Constant_BoolValue# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `-x`, - P: `-_( - x^#2:*expr.Expr_IdentExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `---a`, - P: `-_( - -_( - -_( - a^#4:*expr.Expr_IdentExpr# - )^#3:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `- -1`, - P: `-_( - -1^#2:*expr.Constant_Int64Value# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `4--4`, - P: `_-_( - 4^#1:*expr.Constant_Int64Value#, - -4^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `!!a`, - P: `!_( - !_( - a^#3:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `-!true`, - P: `-_( - !_( - true^#3:*expr.Constant_BoolValue# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `-!-x`, - P: `-_( - !_( - -_( - x^#4:*expr.Expr_IdentExpr# - )^#3:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - - // Binary & Ternary operators - { - I: `1 + 2 * 3 - 4 / 2 % 3`, - P: `_-_( - _+_( - 1^#1:*expr.Constant_Int64Value#, - _*_( - 2^#3:*expr.Constant_Int64Value#, - 3^#5:*expr.Constant_Int64Value# - )^#4:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr#, - _%_( - _/_( - 4^#7:*expr.Constant_Int64Value#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr#, - 3^#11:*expr.Constant_Int64Value# - )^#10:*expr.Expr_CallExpr# - )^#6:*expr.Expr_CallExpr#`, - }, - { - I: `a < 10 && b <= 20 || c > 30 && d >= 40 || e == 50 && f != 60`, - P: `_||_( - _||_( - _&&_( - _<_( - a^#1:*expr.Expr_IdentExpr#, - 10^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - _<=_( - b^#5:*expr.Expr_IdentExpr#, - 20^#7:*expr.Constant_Int64Value# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#, - _&&_( - _>_( - c^#9:*expr.Expr_IdentExpr#, - 30^#11:*expr.Constant_Int64Value# - )^#10:*expr.Expr_CallExpr#, - _>=_( - d^#13:*expr.Expr_IdentExpr#, - 40^#15:*expr.Constant_Int64Value# - )^#14:*expr.Expr_CallExpr# - )^#12:*expr.Expr_CallExpr# - )^#8:*expr.Expr_CallExpr#, - _&&_( - _==_( - e^#17:*expr.Expr_IdentExpr#, - 50^#19:*expr.Constant_Int64Value# - )^#18:*expr.Expr_CallExpr#, - _!=_( - f^#21:*expr.Expr_IdentExpr#, - 60^#23:*expr.Constant_Int64Value# - )^#22:*expr.Expr_CallExpr# - )^#20:*expr.Expr_CallExpr# - )^#16:*expr.Expr_CallExpr#`, - }, - { - I: `a && b && c && d`, - P: `_&&_( - _&&_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#, - _&&_( - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `a && b && c && d`, - P: `_&&_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableVariadicOperatorASTs(true)}, - }, - { - I: `a || b || c || d`, - P: `_||_( - _||_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#, - _||_( - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `a || b || c || d`, - P: `_||_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableVariadicOperatorASTs(true)}, - }, - { - I: `10 - 3 - 2`, - P: `_-_( - _-_( - 10^#1:*expr.Constant_Int64Value#, - 3^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - 2^#5:*expr.Constant_Int64Value# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `(((10 - 3) - 2))`, - P: `_-_( - _-_( - 10^#1:*expr.Constant_Int64Value#, - 3^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - 2^#5:*expr.Constant_Int64Value# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `x in [1, 2, 3]`, - P: `@in( - x^#1:*expr.Expr_IdentExpr#, - [ - 1^#4:*expr.Constant_Int64Value#, - 2^#5:*expr.Constant_Int64Value#, - 3^#6:*expr.Constant_Int64Value# - ]^#3:*expr.Expr_ListExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a ? b : c`, - P: `_?_:_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#4:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a ? b : c ? d : e`, - P: `_?_:_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - _?_:_( - c^#4:*expr.Expr_IdentExpr#, - d^#6:*expr.Expr_IdentExpr#, - e^#7:*expr.Expr_IdentExpr# - )^#5:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `(((1 + 2))) * (3 + 4)`, - P: `_*_( - _+_( - 1^#1:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - _+_( - 3^#5:*expr.Constant_Int64Value#, - 4^#7:*expr.Constant_Int64Value# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#`, - }, - - // Members, Selects, Indexing, and Calls - { - I: `a.b()`, - P: `a^#1:*expr.Expr_IdentExpr#.b()^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a.b(1)`, - P: `a^#1:*expr.Expr_IdentExpr#.b( - 1^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a.b(1, 2)`, - P: `a^#1:*expr.Expr_IdentExpr#.b( - 1^#3:*expr.Constant_Int64Value#, - 2^#4:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `"foo".size()`, - P: `"foo"^#1:*expr.Constant_StringValue#.size()^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a.b.c`, - P: `a^#1:*expr.Expr_IdentExpr#.b^#2:*expr.Expr_SelectExpr#.c^#3:*expr.Expr_SelectExpr#`, - }, - { - I: `a[0]`, - P: `_[_]( - a^#1:*expr.Expr_IdentExpr#, - 0^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `[1, 3, 4][0]`, - P: `_[_]( - [ - 1^#2:*expr.Constant_Int64Value#, - 3^#3:*expr.Constant_Int64Value#, - 4^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - 0^#6:*expr.Constant_Int64Value# - )^#5:*expr.Expr_CallExpr#`, - }, - { - I: `a[b[c]]`, - P: `_[_]( - a^#1:*expr.Expr_IdentExpr#, - _[_]( - b^#3:*expr.Expr_IdentExpr#, - c^#5:*expr.Expr_IdentExpr# - )^#4:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a()`, - P: `a()^#1:*expr.Expr_CallExpr#`, - }, - { - I: `a(b)`, - P: `a( - b^#2:*expr.Expr_IdentExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `func(1, 2, 3)`, - P: `func( - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `.func(1, 2)`, - P: `.func( - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value# - )^#1:*expr.Expr_CallExpr#`, - }, - - // Collections & Structs - { - I: `[1, 2, 3]`, - P: `[ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#`, - }, - { - I: `[1, 2, 3,]`, - P: `[ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#`, - }, - { - I: `[]`, - P: `[]^#1:*expr.Expr_ListExpr#`, - }, - { - I: `{"a": 1, "b": 2}`, - P: `{ - "a"^#2:*expr.Constant_StringValue#:1^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - "b"^#5:*expr.Constant_StringValue#:2^#7:*expr.Constant_Int64Value#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - }, - { - I: `{foo: 5, bar: "xyz"}`, - P: `{ - foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - }, - { - I: `{}`, - P: `{}^#1:*expr.Expr_StructExpr#`, - }, - { - I: `google.protobuf.Empty{}`, - P: `google.protobuf.Empty{}^#3:*expr.Expr_StructExpr#`, - }, - { - I: `foo{ a: b }`, - P: `foo{ - a:b^#2:*expr.Expr_IdentExpr#^#1:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - }, - { - I: `pkg.Msg{field1: "val", field2: 42}`, - P: `pkg.Msg{ - field1:"val"^#3:*expr.Constant_StringValue#^#2:*expr.Expr_CreateStruct_Entry#, - field2:42^#5:*expr.Constant_Int64Value#^#4:*expr.Expr_CreateStruct_Entry# - }^#2:*expr.Expr_StructExpr#`, - }, - { - I: `.pkg.Msg{field1: "val"}`, - P: `.pkg.Msg{ - field1:"val"^#3:*expr.Constant_StringValue#^#2:*expr.Expr_CreateStruct_Entry# - }^#2:*expr.Expr_StructExpr#`, - }, - - // Macros - { - I: `has(a.b)`, - P: `a^#2:*expr.Expr_IdentExpr#.b~test-only~^#4:has#`, - }, - { - I: `[1, 2, 3].all(x, x > 0)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - true^#10:*expr.Constant_BoolValue#, - // LoopCondition - @not_strictly_false( - @result^#11:*expr.Expr_IdentExpr# - )^#12:*expr.Expr_CallExpr#, - // LoopStep - _&&_( - @result^#13:*expr.Expr_IdentExpr#, - _>_( - x^#7:*expr.Expr_IdentExpr#, - 0^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr# - )^#14:*expr.Expr_CallExpr#, - // Result - @result^#15:*expr.Expr_IdentExpr#)^#16:all#`, - }, - { - I: `[1, 2, 3].exists(x, x == 2)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - false^#10:*expr.Constant_BoolValue#, - // LoopCondition - @not_strictly_false( - !_( - @result^#11:*expr.Expr_IdentExpr# - )^#12:*expr.Expr_CallExpr# - )^#13:*expr.Expr_CallExpr#, - // LoopStep - _||_( - @result^#14:*expr.Expr_IdentExpr#, - _==_( - x^#7:*expr.Expr_IdentExpr#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr# - )^#15:*expr.Expr_CallExpr#, - // Result - @result^#16:*expr.Expr_IdentExpr#)^#17:exists#`, - }, - { - I: `[1, 2, 3].exists_one(x, x == 2)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - 0^#10:*expr.Constant_Int64Value#, - // LoopCondition - true^#11:*expr.Constant_BoolValue#, - // LoopStep - _?_:_( - _==_( - x^#7:*expr.Expr_IdentExpr#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr#, - _+_( - @result^#12:*expr.Expr_IdentExpr#, - 1^#13:*expr.Constant_Int64Value# - )^#14:*expr.Expr_CallExpr#, - @result^#15:*expr.Expr_IdentExpr# - )^#16:*expr.Expr_CallExpr#, - // Result - _==_( - @result^#17:*expr.Expr_IdentExpr#, - 1^#18:*expr.Constant_Int64Value# - )^#19:*expr.Expr_CallExpr#)^#20:exists_one#`, - }, - { - I: `[1, 2, 3].map(x, x * 2)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - []^#10:*expr.Expr_ListExpr#, - // LoopCondition - true^#11:*expr.Constant_BoolValue#, - // LoopStep - _+_( - @result^#12:*expr.Expr_IdentExpr#, - [ - _*_( - x^#7:*expr.Expr_IdentExpr#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr# - ]^#13:*expr.Expr_ListExpr# - )^#14:*expr.Expr_CallExpr#, - // Result - @result^#15:*expr.Expr_IdentExpr#)^#16:map#`, - }, - { - I: `[1, 2, 3].filter(x, x > 1)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - []^#10:*expr.Expr_ListExpr#, - // LoopCondition - true^#11:*expr.Constant_BoolValue#, - // LoopStep - _?_:_( - _>_( - x^#7:*expr.Expr_IdentExpr#, - 1^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr#, - _+_( - @result^#12:*expr.Expr_IdentExpr#, - [ - x^#6:*expr.Expr_IdentExpr# - ]^#13:*expr.Expr_ListExpr# - )^#14:*expr.Expr_CallExpr#, - @result^#15:*expr.Expr_IdentExpr# - )^#16:*expr.Expr_CallExpr#, - // Result - @result^#17:*expr.Expr_IdentExpr#)^#18:filter#`, - }, - - // Optional Syntax - { - I: `a.?b`, - P: `_?._( - a^#1:*expr.Expr_IdentExpr#, - "b"^#3:*expr.Constant_StringValue# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `a[?0]`, - P: `_[?_]( - a^#1:*expr.Expr_IdentExpr#, - 0^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `[?a, b, ?c]`, - P: `[ - a^#2:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#4:*expr.Expr_IdentExpr# - ]^#1:*expr.Expr_ListExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `{?a: 1, b: 2}`, - P: `{ - ?a^#2:*expr.Expr_IdentExpr#:1^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - b^#5:*expr.Expr_IdentExpr#:2^#7:*expr.Constant_Int64Value#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `pkg.Msg{?field: 42}`, - P: `pkg.Msg{ - ?field:42^#3:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry# - }^#2:*expr.Expr_StructExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - - // Escaped Identifier Syntax - { - I: "msg.`field-name`", - P: `msg^#1:*expr.Expr_IdentExpr#.field-name^#2:*expr.Expr_SelectExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "msg.`a.b.c`", - P: `msg^#1:*expr.Expr_IdentExpr#.a.b.c^#2:*expr.Expr_SelectExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "msg.`field name`", - P: `msg^#1:*expr.Expr_IdentExpr#.field name^#2:*expr.Expr_SelectExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "Msg{`field-1`: 42}", - P: `Msg{ - field-1:42^#2:*expr.Constant_Int64Value#^#1:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - - // Error Cases - { - I: "(1 + 2", - E: "ERROR: :1:7: expected ')'\n" + - " | (1 + 2\n" + - " | ......^", - }, - { - I: "[1, 2", - E: "ERROR: :1:6: expected ']'\n" + - " | [1, 2\n" + - " | .....^", - }, - { - I: "{1: 2", - E: "ERROR: :1:6: expected '}'\n" + - " | {1: 2\n" + - " | .....^", - }, - { - I: "a ? b", - E: "ERROR: :1:6: expected ':' in conditional expression\n" + - " | a ? b\n" + - " | .....^", - }, - { - I: "1 + 2 3", - E: "ERROR: :1:7: Syntax error: mismatched input '3' expecting \n" + - " | 1 + 2 3\n" + - " | ......^", - }, - { - I: "0xFFFFFFFFFFFFFFFFF", - E: "ERROR: :1:1: invalid int literal\n" + - " | 0xFFFFFFFFFFFFFFFFF\n" + - " | ^", - }, - { - I: "0xFFFFFFFFFFFFFFFFFu", - E: "ERROR: :1:1: invalid uint literal\n" + - " | 0xFFFFFFFFFFFFFFFFFu\n" + - " | ^", - }, - { - I: "1.99e90000009", - E: "ERROR: :1:1: invalid double literal\n" + - " | 1.99e90000009\n" + - " | ^", - }, - { - I: "as", - E: "ERROR: :1:1: reserved identifier: as\n" + - " | as\n" + - " | ^", - }, - { - I: "msg.`ident` + msg.`other`", - E: "ERROR: :1:5: unsupported syntax '`'\n" + - " | msg.`ident` + msg.`other`\n" + - " | ....^\n" + - "ERROR: :1:19: unsupported syntax '`'\n" + - " | msg.`ident` + msg.`other`\n" + - " | ..................^", - Opts: []Option{EnableIdentEscapeSyntax(false)}, - }, - { - I: "a.?b", - E: "ERROR: :1:2: unsupported syntax '.?'\n" + - " | a.?b\n" + - " | .^", - }, - { - I: "has(m)", - E: "ERROR: :1:5: invalid argument to has() macro\n" + - " | has(m)\n" + - " | ....^", - Opts: []Option{Macros(AllMacros...)}, - }, - { - I: "[1, 2].all(1 + 2, true)", - E: "ERROR: :1:14: argument must be a simple name\n" + - " | [1, 2].all(1 + 2, true)\n" + - " | .............^", - Opts: []Option{Macros(AllMacros...)}, - }, - { - I: "[1, 2].all(__result__, true)", - E: "ERROR: :1:12: iteration variable overwrites accumulator variable\n" + - " | [1, 2].all(__result__, true)\n" + - " | ...........^", - Opts: []Option{Macros(AllMacros...)}, - }, - { - I: "1{}", - E: "ERROR: :1:2: Syntax error: mismatched input '{' expecting \n" + - " | 1{}\n" + - " | .^", - }, - { - I: "a.", - E: "ERROR: :1:3: expected identifier after '.'\n" + - " | a.\n" + - " | ..^", - }, - { - I: ". *", - E: "ERROR: :1:3: expected identifier\n" + - " | . *\n" + - " | ..^", - }, - { - I: ".as", - E: "ERROR: :1:2: reserved identifier: as\n" + - " | .as\n" + - " | .^", - }, - { - I: "* 2", - E: "ERROR: :1:1: unexpected token\n" + - " | * 2\n" + - " | ^\n" + - "ERROR: :1:3: Syntax error: mismatched input '2' expecting \n" + - " | * 2\n" + - " | ..^", - }, - { - I: "{'k' 'v'}", - E: "ERROR: :1:6: expected ':' in map entry\n" + - " | {'k' 'v'}\n" + - " | .....^", - }, - { - I: "Msg{1: 2}", - E: "ERROR: :1:5: expected struct field name\n" + - " | Msg{1: 2}\n" + - " | ....^", - }, - { - I: "Msg{f 10}", - E: "ERROR: :1:7: expected ':' in struct field\n" + - " | Msg{f 10}\n" + - " | ......^", - }, - { - I: "Msg{f: 10", - E: "ERROR: :1:10: expected '}'\n" + - " | Msg{f: 10\n" + - " | .........^", - }, - { - I: "f(1, 2", - E: "ERROR: :1:7: Syntax error: mismatched input expecting ')'\n" + - " | f(1, 2\n" + - " | ......^", - }, - { - I: "1e", - E: "ERROR: :1:1: floating point literal missing digits after exponent separator\n" + - " | 1e\n" + - " | ^", - }, - { - I: "\"unterminated", - E: "ERROR: :1:1: unterminated string literal\n" + - " | \"unterminated\n" + - " | ^", - }, - { - I: "b\"unterminated", - E: "ERROR: :1:1: unterminated bytes literal\n" + - " | b\"unterminated\n" + - " | ^", - }, - { - I: "a.`foo`()", - E: "ERROR: :1:3: unexpected quoted identifier\n" + - " | a.`foo`()\n" + - " | ..^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "`foo`", - E: "ERROR: :1:1: unexpected quoted identifier\n" + - " | `foo`\n" + - " | ^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "a.`b@c`", - E: "ERROR: :1:3: unexpected quoted identifier\n" + - " | a.`b@c`\n" + - " | ..^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "a.``", - E: "ERROR: :1:3: unexpected quoted identifier\n" + - " | a.``\n" + - " | ..^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "`foo", - E: "ERROR: :1:1: unterminated quoted identifier\n" + - " | `foo\n" + - " | ^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "-0x8000000000000001", - E: "ERROR: :1:2: invalid int literal\n" + - " | -0x8000000000000001\n" + - " | .^", - }, - { - I: "-9223372036854775809", - E: "ERROR: :1:2: invalid int literal\n" + - " | -9223372036854775809\n" + - " | .^", - }, - { - I: "a[?0]", - E: "ERROR: :1:2: unsupported syntax '?'\n" + - " | a[?0]\n" + - " | .^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "[?1]", - E: "ERROR: :1:2: unsupported syntax '?'\n" + - " | [?1]\n" + - " | .^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "{?'k': 'v'}", - E: "ERROR: :1:2: unsupported syntax '?'\n" + - " | {?'k': 'v'}\n" + - " | .^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "Msg{?f: 1}", - E: "ERROR: :1:5: unsupported syntax '?'\n" + - " | Msg{?f: 1}\n" + - " | ....^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "a.?`foo`", - E: "ERROR: :1:4: unsupported syntax '`'\n" + - " | a.?`foo`\n" + - " | ...^", - Opts: []Option{EnableOptionalSyntax(true), EnableIdentEscapeSyntax(false)}, - }, -} - -func parse(source common.Source, opts ...Option) (*ast.AST, *common.Errors) { - p, err := NewPrattParser(opts...) - if err != nil { - panic(err) - } - return p.Parse(source) -} - -func TestPrattParser(t *testing.T) { - for i, tst := range prattTestCases { - name := fmt.Sprintf("%d %s", i, tst.I) - // Local variable required as the closure will reference the value for the last - // 'tst' value rather than the local 'tc' instance declared within the loop. - tc := tst - t.Run(name, func(t *testing.T) { - t.Parallel() - opts := tc.Opts - if len(opts) == 0 { - opts = []Option{Macros(AllMacros...), PopulateMacroCalls(true)} - } - src := common.NewTextSource(tc.I) - parsed, errors := parse(src, opts...) - if len(errors.GetErrors()) > 0 { - actualErr := errors.ToDisplayString() - if tc.E == "" { - t.Fatalf("Unexpected errors: %v", actualErr) - } else if !test.Compare(actualErr, tc.E) { - t.Fatal(test.DiffMessage("Error mismatch", actualErr, tc.E)) - } - return - } else if tc.E != "" { - t.Fatalf("Expected error not thrown: '%s'", tc.E) - } - failureDisplayMethod := fmt.Sprintf("Parse(\"%s\")", tc.I) - actualWithKind := debug.ToAdornedDebugString(parsed.Expr(), &kindAndIDAdorner{parsed.SourceInfo()}) - if !test.Compare(actualWithKind, tc.P) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Structure - %s", failureDisplayMethod), actualWithKind, tc.P)) - } - }) - } -} func TestPrattParserSourceInfoPositions(t *testing.T) { src := common.NewTextSource("a + b") p, err := NewPrattParser() From 5248063f8a8f4cd18f2ee212815eae6a9b994f85 Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Mon, 24 Aug 2026 10:42:09 -0700 Subject: [PATCH 3/3] [Pratt parser] Make antlrParser and prattParser unexported and fold unit tests into parser_test.go --- parser/antlr_parser.go | 146 ++++++++-------------- parser/parser.go | 4 +- parser/parser_test.go | 239 +++++++++++++++++++++++++++++++++++ parser/pratt_parser.go | 129 +++++++------------ parser/pratt_parser_test.go | 242 ------------------------------------ 5 files changed, 336 insertions(+), 424 deletions(-) delete mode 100644 parser/pratt_parser_test.go diff --git a/parser/antlr_parser.go b/parser/antlr_parser.go index bab466f27..e755b7ac7 100644 --- a/parser/antlr_parser.go +++ b/parser/antlr_parser.go @@ -17,7 +17,6 @@ package parser import ( "errors" "fmt" - "math" "regexp" "strconv" "strings" @@ -32,63 +31,20 @@ import ( "cel.dev/cel-go/parser/gen" ) -// AntlrParser encapsulates the context necessary to perform ANTLR parsing for different expressions. -type AntlrParser struct { +// antlrParser encapsulates the context necessary to perform ANTLR parsing for different expressions. +type antlrParser struct { options } -// NewAntlrParser builds and returns a new AntlrParser using the provided options. -func NewAntlrParser(opts ...Option) (*AntlrParser, error) { - p := &AntlrParser{} - p.enableHiddenAccumulatorName = true - p.enableIdentEscapeSyntax = true - for _, opt := range opts { - if err := opt(&p.options); err != nil { - return nil, err - } - } - if p.errorReportingLimit == 0 { - p.errorReportingLimit = 100 - } - if p.maxRecursionDepth == 0 { - p.maxRecursionDepth = 250 - } - if p.maxRecursionDepth == -1 { - p.maxRecursionDepth = math.MaxInt - } - if p.errorRecoveryTokenLookaheadLimit == 0 { - p.errorRecoveryTokenLookaheadLimit = 256 - } - if p.errorRecoveryLimit == 0 { - p.errorRecoveryLimit = 30 - } - if p.errorRecoveryLimit == -1 { - p.errorRecoveryLimit = math.MaxInt - } - if p.expressionSizeCodePointLimit == 0 { - p.expressionSizeCodePointLimit = 100_000 - } - if p.expressionSizeCodePointLimit == -1 { - p.expressionSizeCodePointLimit = math.MaxInt - } - if p.maxExpressionNodeCount == 0 { - p.maxExpressionNodeCount = 100_000 - } - if p.maxExpressionNodeCount == -1 { - p.maxExpressionNodeCount = math.MaxInt - } - return p, nil -} - // Parse parses the expression represented by source and returns the result. -func (p *AntlrParser) Parse(source common.Source) (*ast.AST, *common.Errors) { +func (p *antlrParser) Parse(source common.Source) (*ast.AST, *common.Errors) { errs := common.NewErrors(source) accu := AccumulatorName if p.enableHiddenAccumulatorName { accu = HiddenAccumulatorName } fac := ast.NewExprFactoryWithAccumulator(accu) - impl := antlrParser{ + impl := antlrVisitor{ errors: &parseErrors{errs}, exprFactory: fac, helper: newParserHelper(source, fac), @@ -255,7 +211,7 @@ func (rl *recoveryLimitErrorStrategy) checkAttempts(recognizer antlr.Parser) { var _ antlr.ErrorStrategy = &recoveryLimitErrorStrategy{} -type antlrParser struct { +type antlrVisitor struct { gen.BaseCELVisitor errors *parseErrors exprFactory ast.ExprFactory @@ -274,10 +230,10 @@ type antlrParser struct { enableIdentEscapeSyntax bool } -var _ gen.CELVisitor = (*antlrParser)(nil) +var _ gen.CELVisitor = (*antlrVisitor)(nil) // normalizeIdent returns the interpreted identifier. -func (p *antlrParser) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error) { +func (p *antlrVisitor) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error) { switch ident := ctx.(type) { case *gen.SimpleIdentifierContext: return ident.GetId().GetText(), nil @@ -290,7 +246,7 @@ func (p *antlrParser) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error return "", errors.New("unsupported ident kind") } -func (p *antlrParser) parse(expr runes.Buffer, desc string) ast.Expr { +func (p *antlrVisitor) parse(expr runes.Buffer, desc string) ast.Expr { lexer := gen.NewCELLexer(newCharStream(expr, desc)) lexer.RemoveErrorListeners() lexer.AddErrorListener(p) @@ -333,7 +289,7 @@ func (p *antlrParser) parse(expr runes.Buffer, desc string) ast.Expr { } // Visitor implementations. -func (p *antlrParser) Visit(tree antlr.ParseTree) any { +func (p *antlrVisitor) Visit(tree antlr.ParseTree) any { t := unnest(tree) switch tree := t.(type) { case *gen.StartContext: @@ -421,12 +377,12 @@ func (p *antlrParser) Visit(tree antlr.ParseTree) any { } // Visit a parse tree produced by CELParser#start. -func (p *antlrParser) VisitStart(ctx *gen.StartContext) any { +func (p *antlrVisitor) VisitStart(ctx *gen.StartContext) any { return p.Visit(ctx.Expr()) } // Visit a parse tree produced by CELParser#expr. -func (p *antlrParser) VisitExpr(ctx *gen.ExprContext) any { +func (p *antlrVisitor) VisitExpr(ctx *gen.ExprContext) any { result := p.Visit(ctx.GetE()).(ast.Expr) if ctx.GetOp() == nil { return result @@ -438,7 +394,7 @@ func (p *antlrParser) VisitExpr(ctx *gen.ExprContext) any { } // Visit a parse tree produced by CELParser#conditionalOr. -func (p *antlrParser) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { +func (p *antlrVisitor) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { result := p.Visit(ctx.GetE()).(ast.Expr) l := p.newLogicManager(operators.LogicalOr, result) rest := ctx.GetE1() @@ -454,7 +410,7 @@ func (p *antlrParser) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { } // Visit a parse tree produced by CELParser#conditionalAnd. -func (p *antlrParser) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { +func (p *antlrVisitor) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { result := p.Visit(ctx.GetE()).(ast.Expr) l := p.newLogicManager(operators.LogicalAnd, result) rest := ctx.GetE1() @@ -470,7 +426,7 @@ func (p *antlrParser) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { } // Visit a parse tree produced by CELParser#relation. -func (p *antlrParser) VisitRelation(ctx *gen.RelationContext) any { +func (p *antlrVisitor) VisitRelation(ctx *gen.RelationContext) any { opText := "" if ctx.GetOp() != nil { opText = ctx.GetOp().GetText() @@ -485,7 +441,7 @@ func (p *antlrParser) VisitRelation(ctx *gen.RelationContext) any { } // Visit a parse tree produced by CELParser#calc. -func (p *antlrParser) VisitCalc(ctx *gen.CalcContext) any { +func (p *antlrVisitor) VisitCalc(ctx *gen.CalcContext) any { opText := "" if ctx.GetOp() != nil { opText = ctx.GetOp().GetText() @@ -499,12 +455,12 @@ func (p *antlrParser) VisitCalc(ctx *gen.CalcContext) any { return p.reportError(ctx, "operator not found") } -func (p *antlrParser) VisitUnary(ctx *gen.UnaryContext) any { +func (p *antlrVisitor) VisitUnary(ctx *gen.UnaryContext) any { return p.helper.newLiteralString(ctx, "<>") } // Visit a parse tree produced by CELParser#LogicalNot. -func (p *antlrParser) VisitLogicalNot(ctx *gen.LogicalNotContext) any { +func (p *antlrVisitor) VisitLogicalNot(ctx *gen.LogicalNotContext) any { if len(ctx.GetOps())%2 == 0 { return p.Visit(ctx.Member()) } @@ -513,7 +469,7 @@ func (p *antlrParser) VisitLogicalNot(ctx *gen.LogicalNotContext) any { return p.globalCallOrMacro(opID, operators.LogicalNot, target) } -func (p *antlrParser) VisitNegate(ctx *gen.NegateContext) any { +func (p *antlrVisitor) VisitNegate(ctx *gen.NegateContext) any { if len(ctx.GetOps())%2 == 0 { return p.Visit(ctx.Member()) } @@ -523,7 +479,7 @@ func (p *antlrParser) VisitNegate(ctx *gen.NegateContext) any { } // VisitSelect visits a parse tree produced by CELParser#Select. -func (p *antlrParser) VisitSelect(ctx *gen.SelectContext) any { +func (p *antlrVisitor) VisitSelect(ctx *gen.SelectContext) any { operand := p.Visit(ctx.Member()).(ast.Expr) // Handle the error case where no valid identifier is specified. if ctx.GetId() == nil || ctx.GetOp() == nil { @@ -547,7 +503,7 @@ func (p *antlrParser) VisitSelect(ctx *gen.SelectContext) any { } // VisitMemberCall visits a parse tree produced by CELParser#MemberCall. -func (p *antlrParser) VisitMemberCall(ctx *gen.MemberCallContext) any { +func (p *antlrVisitor) VisitMemberCall(ctx *gen.MemberCallContext) any { operand := p.Visit(ctx.Member()).(ast.Expr) // Handle the error case where no valid identifier is specified. if ctx.GetId() == nil { @@ -559,7 +515,7 @@ func (p *antlrParser) VisitMemberCall(ctx *gen.MemberCallContext) any { } // Visit a parse tree produced by CELParser#Index. -func (p *antlrParser) VisitIndex(ctx *gen.IndexContext) any { +func (p *antlrVisitor) VisitIndex(ctx *gen.IndexContext) any { target := p.Visit(ctx.Member()).(ast.Expr) // Handle the error case where no valid identifier is specified. if ctx.GetOp() == nil { @@ -578,7 +534,7 @@ func (p *antlrParser) VisitIndex(ctx *gen.IndexContext) any { } // Visit a parse tree produced by CELParser#CreateMessage. -func (p *antlrParser) VisitCreateMessage(ctx *gen.CreateMessageContext) any { +func (p *antlrVisitor) VisitCreateMessage(ctx *gen.CreateMessageContext) any { messageName := "" for _, id := range ctx.GetIds() { if len(messageName) != 0 { @@ -595,7 +551,7 @@ func (p *antlrParser) VisitCreateMessage(ctx *gen.CreateMessageContext) any { } // Visit a parse tree of field initializers. -func (p *antlrParser) VisitIFieldInitializerList(ctx gen.IFieldInitializerListContext) any { +func (p *antlrVisitor) VisitIFieldInitializerList(ctx gen.IFieldInitializerListContext) any { if ctx == nil || ctx.GetFields() == nil { // This is the result of a syntax error handled elswhere, return empty. return []ast.EntryExpr{} @@ -632,7 +588,7 @@ func (p *antlrParser) VisitIFieldInitializerList(ctx gen.IFieldInitializerListCo } // Visit a parse tree produced by CELParser#Ident. -func (p *antlrParser) VisitIdent(ctx *gen.IdentContext) any { +func (p *antlrVisitor) VisitIdent(ctx *gen.IdentContext) any { identName := "" if ctx.GetLeadingDot() != nil { identName = "." @@ -651,7 +607,7 @@ func (p *antlrParser) VisitIdent(ctx *gen.IdentContext) any { } // Visit a parse tree produced by CELParser#GlobalCallContext. -func (p *antlrParser) VisitGlobalCall(ctx *gen.GlobalCallContext) any { +func (p *antlrVisitor) VisitGlobalCall(ctx *gen.GlobalCallContext) any { identName := "" if ctx.GetLeadingDot() != nil { identName = "." @@ -671,14 +627,14 @@ func (p *antlrParser) VisitGlobalCall(ctx *gen.GlobalCallContext) any { } // Visit a parse tree produced by CELParser#CreateList. -func (p *antlrParser) VisitCreateList(ctx *gen.CreateListContext) any { +func (p *antlrVisitor) VisitCreateList(ctx *gen.CreateListContext) any { listID := p.helper.id(ctx.GetOp()) elems, optionals := p.visitListInit(ctx.GetElems()) return p.helper.newList(listID, elems, optionals...) } // Visit a parse tree produced by CELParser#CreateStruct. -func (p *antlrParser) VisitCreateStruct(ctx *gen.CreateStructContext) any { +func (p *antlrVisitor) VisitCreateStruct(ctx *gen.CreateStructContext) any { structID := p.helper.id(ctx.GetOp()) entries := []ast.EntryExpr{} if ctx.GetEntries() != nil { @@ -688,7 +644,7 @@ func (p *antlrParser) VisitCreateStruct(ctx *gen.CreateStructContext) any { } // Visit a parse tree produced by CELParser#mapInitializerList. -func (p *antlrParser) VisitMapInitializerList(ctx *gen.MapInitializerListContext) any { +func (p *antlrVisitor) VisitMapInitializerList(ctx *gen.MapInitializerListContext) any { if ctx == nil || ctx.GetKeys() == nil { // This is the result of a syntax error handled elswhere, return empty. return []ast.EntryExpr{} @@ -718,7 +674,7 @@ func (p *antlrParser) VisitMapInitializerList(ctx *gen.MapInitializerListContext } // Visit a parse tree produced by CELParser#Int. -func (p *antlrParser) VisitInt(ctx *gen.IntContext) any { +func (p *antlrVisitor) VisitInt(ctx *gen.IntContext) any { text := ctx.GetTok().GetText() base := 10 if strings.HasPrefix(text, "0x") { @@ -736,7 +692,7 @@ func (p *antlrParser) VisitInt(ctx *gen.IntContext) any { } // Visit a parse tree produced by CELParser#Uint. -func (p *antlrParser) VisitUint(ctx *gen.UintContext) any { +func (p *antlrVisitor) VisitUint(ctx *gen.UintContext) any { text := ctx.GetTok().GetText() // trim the 'u' designator included in the uint literal. text = text[:len(text)-1] @@ -753,7 +709,7 @@ func (p *antlrParser) VisitUint(ctx *gen.UintContext) any { } // Visit a parse tree produced by CELParser#Double. -func (p *antlrParser) VisitDouble(ctx *gen.DoubleContext) any { +func (p *antlrVisitor) VisitDouble(ctx *gen.DoubleContext) any { txt := ctx.GetTok().GetText() if ctx.GetSign() != nil { txt = ctx.GetSign().GetText() + txt @@ -766,40 +722,40 @@ func (p *antlrParser) VisitDouble(ctx *gen.DoubleContext) any { } // Visit a parse tree produced by CELParser#String. -func (p *antlrParser) VisitString(ctx *gen.StringContext) any { +func (p *antlrVisitor) VisitString(ctx *gen.StringContext) any { s := p.unquote(ctx, ctx.GetTok().GetText(), false) return p.helper.newLiteralString(ctx, s) } // Visit a parse tree produced by CELParser#Bytes. -func (p *antlrParser) VisitBytes(ctx *gen.BytesContext) any { +func (p *antlrVisitor) VisitBytes(ctx *gen.BytesContext) any { b := []byte(p.unquote(ctx, ctx.GetTok().GetText()[1:], true)) return p.helper.newLiteralBytes(ctx, b) } // Visit a parse tree produced by CELParser#BoolTrue. -func (p *antlrParser) VisitBoolTrue(ctx *gen.BoolTrueContext) any { +func (p *antlrVisitor) VisitBoolTrue(ctx *gen.BoolTrueContext) any { return p.helper.newLiteralBool(ctx, true) } // Visit a parse tree produced by CELParser#BoolFalse. -func (p *antlrParser) VisitBoolFalse(ctx *gen.BoolFalseContext) any { +func (p *antlrVisitor) VisitBoolFalse(ctx *gen.BoolFalseContext) any { return p.helper.newLiteralBool(ctx, false) } // Visit a parse tree produced by CELParser#Null. -func (p *antlrParser) VisitNull(ctx *gen.NullContext) any { +func (p *antlrVisitor) VisitNull(ctx *gen.NullContext) any { return p.helper.exprFactory.NewLiteral(p.helper.newID(ctx), types.NullValue) } -func (p *antlrParser) visitExprList(ctx gen.IExprListContext) []ast.Expr { +func (p *antlrVisitor) visitExprList(ctx gen.IExprListContext) []ast.Expr { if ctx == nil { return []ast.Expr{} } return p.visitSlice(ctx.GetE()) } -func (p *antlrParser) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int32) { +func (p *antlrVisitor) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int32) { if ctx == nil { return []ast.Expr{}, []int32{} } @@ -823,7 +779,7 @@ func (p *antlrParser) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int return result, optionals } -func (p *antlrParser) visitSlice(expressions []gen.IExprContext) []ast.Expr { +func (p *antlrVisitor) visitSlice(expressions []gen.IExprContext) []ast.Expr { if expressions == nil { return []ast.Expr{} } @@ -835,7 +791,7 @@ func (p *antlrParser) visitSlice(expressions []gen.IExprContext) []ast.Expr { return result } -func (p *antlrParser) unquote(ctx any, value string, isBytes bool) string { +func (p *antlrVisitor) unquote(ctx any, value string, isBytes bool) string { text, err := unescape(value, isBytes) if err != nil { p.reportError(ctx, "%s", err.Error()) @@ -844,14 +800,14 @@ func (p *antlrParser) unquote(ctx any, value string, isBytes bool) string { return text } -func (p *antlrParser) newLogicManager(function string, term ast.Expr) *logicManager { +func (p *antlrVisitor) newLogicManager(function string, term ast.Expr) *logicManager { if p.enableVariadicOperatorASTs { return newVariadicLogicManager(p.exprFactory, function, term) } return newBalancingLogicManager(p.exprFactory, function, term) } -func (p *antlrParser) reportError(ctx any, format string, args ...any) ast.Expr { +func (p *antlrVisitor) reportError(ctx any, format string, args ...any) ast.Expr { var location common.Location err := p.helper.newExpr(ctx) switch c := ctx.(type) { @@ -866,7 +822,7 @@ func (p *antlrParser) reportError(ctx any, format string, args ...any) ast.Expr } // ANTLR Parse listener implementations -func (p *antlrParser) SyntaxError(recognizer antlr.Recognizer, offendingSymbol any, line, column int, msg string, e antlr.RecognitionException) { +func (p *antlrVisitor) SyntaxError(recognizer antlr.Recognizer, offendingSymbol any, line, column int, msg string, e antlr.RecognitionException) { offset := p.helper.sourceInfo.ComputeOffset(int32(line), int32(column)) l := p.helper.getLocationByOffset(offset) // Hack to keep existing error messages consistent with previous versions of CEL when a reserved word @@ -887,33 +843,33 @@ func (p *antlrParser) SyntaxError(recognizer antlr.Recognizer, offendingSymbol a } } -func (p *antlrParser) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { +func (p *antlrVisitor) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { // Intentional } -func (p *antlrParser) ReportAttemptingFullContext(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, conflictingAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { +func (p *antlrVisitor) ReportAttemptingFullContext(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, conflictingAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { // Intentional } -func (p *antlrParser) ReportContextSensitivity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex, prediction int, configs *antlr.ATNConfigSet) { +func (p *antlrVisitor) ReportContextSensitivity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex, prediction int, configs *antlr.ATNConfigSet) { // Intentional } -func (p *antlrParser) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { +func (p *antlrVisitor) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, nil, args...); found { return expr } return p.helper.newGlobalCall(exprID, function, args...) } -func (p *antlrParser) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { +func (p *antlrVisitor) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, target, args...); found { return expr } return p.helper.newReceiverCall(exprID, function, target, args...) } -func (p *antlrParser) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { +func (p *antlrVisitor) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { macro, found := p.macros[makeMacroKey(function, len(args), target != nil)] if !found { macro, found = p.macros[makeVarArgMacroKey(function, target != nil)] @@ -957,14 +913,14 @@ func (p *antlrParser) expandMacro(exprID int64, function string, target ast.Expr return expr, true } -func (p *antlrParser) checkAndIncrementRecursionDepth() { +func (p *antlrVisitor) checkAndIncrementRecursionDepth() { p.recursionDepth++ if p.recursionDepth > p.maxRecursionDepth { panic(&recursionError{message: "max recursion depth exceeded"}) } } -func (p *antlrParser) decrementRecursionDepth() { +func (p *antlrVisitor) decrementRecursionDepth() { p.recursionDepth-- } diff --git a/parser/parser.go b/parser/parser.go index 8a3c3ff71..8fe906a0b 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -87,9 +87,9 @@ func mustNewParser(opts ...Option) *Parser { // Parse parses the expression represented by source and returns the result. func (p *Parser) Parse(source common.Source) (*ast.AST, *common.Errors) { if p.enablePrattParser { - return (&PrattParser{options: p.options}).Parse(source) + return (&prattParser{options: p.options}).Parse(source) } - return (&AntlrParser{options: p.options}).Parse(source) + return (&antlrParser{options: p.options}).Parse(source) } // reservedIds are not legal to use as variables. We exclude them post-parse, as they *are* valid diff --git a/parser/parser_test.go b/parser/parser_test.go index 203b8cd8e..00dda7c5a 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -2597,6 +2597,245 @@ func TestParserOptionErrors(t *testing.T) { } } +func TestSourceInfoPositions(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("ASCII", func(t *testing.T) { + src := common.NewTextSource("a + b") + p, err := NewParser(EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(src) + if len(errs.GetErrors()) > 0 { + t.Fatalf("Parse() failed: %s", errs.ToDisplayString()) + } + sourceInfo := parsed.SourceInfo() + root := parsed.Expr() + if sourceInfo.GetStartLocation(root.ID()).Column() != 2 { + t.Errorf("expected root column 2, got %d", sourceInfo.GetStartLocation(root.ID()).Column()) + } + args := root.AsCall().Args() + if len(args) != 2 { + t.Fatalf("expected 2 args, got %d", len(args)) + } + if sourceInfo.GetStartLocation(args[0].ID()).Column() != 0 { + t.Errorf("expected arg[0] column 0, got %d", sourceInfo.GetStartLocation(args[0].ID()).Column()) + } + if sourceInfo.GetStartLocation(args[1].ID()).Column() != 4 { + t.Errorf("expected arg[1] column 4, got %d", sourceInfo.GetStartLocation(args[1].ID()).Column()) + } + }) + + t.Run("MixedUnicodeMultiByteAndMultiLine", func(t *testing.T) { + // Mix of 1-byte ASCII (a, b, +), 2-byte Unicode ("ñ"), 3-byte Unicode ("❤"), and 4-byte Unicode ("🚀") + expr := "a + \"ñ\" +\n\"🚀\" + \"❤\" + b" + src := common.NewTextSource(expr) + p, err := NewParser(EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(src) + if len(errs.GetErrors()) > 0 { + t.Fatalf("Parse() failed: %s", errs.ToDisplayString()) + } + sinfo := parsed.SourceInfo() + + // AST hierarchy: + // expr4: [expr3] + [b] (line 2, col 10) + // expr3: [expr2] + ["❤"] (line 2, col 4) + // expr2: [expr1] + ["🚀"] (line 1, col 8) + // expr1: [a] + ["ñ"] (line 1, col 2) + expr4 := parsed.Expr() + call4 := expr4.AsCall() + bExpr := call4.Args()[1] + + expr3 := call4.Args()[0] + call3 := expr3.AsCall() + heartExpr := call3.Args()[1] + + expr2 := call3.Args()[0] + call2 := expr2.AsCall() + rocketExpr := call2.Args()[1] + + expr1 := call2.Args()[0] + call1 := expr1.AsCall() + aExpr := call1.Args()[0] + enyeExpr := call1.Args()[1] + + assertLoc := func(name string, id int64, wantLine, wantCol int32) { + t.Helper() + loc := sinfo.GetStartLocation(id) + if int32(loc.Line()) != wantLine || int32(loc.Column()) != wantCol { + t.Errorf("%s location mismatch: got (%d, %d), want (%d, %d)", + name, loc.Line(), loc.Column(), wantLine, wantCol) + } + } + + assertLoc("call1 (+)", expr1.ID(), 1, 2) + assertLoc("a", aExpr.ID(), 1, 0) + assertLoc("\"ñ\" (2-byte)", enyeExpr.ID(), 1, 4) + assertLoc("call2 (+)", expr2.ID(), 1, 8) + assertLoc("\"🚀\" (4-byte)", rocketExpr.ID(), 2, 0) + assertLoc("call3 (+)", expr3.ID(), 2, 4) + assertLoc("\"❤\" (3-byte)", heartExpr.ID(), 2, 6) + assertLoc("call4 (+)", expr4.ID(), 2, 10) + assertLoc("b", bExpr.ID(), 2, 12) + }) + }) + } +} + +func TestPopulateMacroCalls(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("DisabledByDefault", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(false), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + if len(parsed.SourceInfo().MacroCalls()) != 0 { + t.Errorf("expected 0 macro calls, got %d", len(parsed.SourceInfo().MacroCalls())) + } + }) + + t.Run("GlobalMacroCallRecorded", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 1 { + t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) + } + }) + + t.Run("ReceiverMacroCallRecorded", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("[1, 2].exists(x, x > 0)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 1 { + t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) + } + }) + + t.Run("NestedMacroCallsRecorded", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("[1, 2].all(x, has(x.b))")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 2 { + t.Fatalf("expected 2 macro calls, got %d", len(macroCalls)) + } + }) + }) + } +} + +func TestErrorRecoveryLimits(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("LimitZero", func(t *testing.T) { + p, err := NewParser(ErrorRecoveryLimit(0), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("......")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected error recovery limit error, got none") + } + }) + + t.Run("LimitOne", func(t *testing.T) { + p, err := NewParser(ErrorRecoveryLimit(1), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("......")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected error recovery limit error, got none") + } + }) + }) + } +} + +func TestRecursionLimit(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("DeeplyNestedBracketsLimitExceeded", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(5), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("[[[[[[1]]]]]]")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected recursion limit error, got none") + } + }) + }) + } + + t.Run("PrattSequentialScopesDoNotAccumulateDepth", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(2), EnablePrattParser(true)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("[1] + [2] + [3]")) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on sequential scopes: %s", errs.ToDisplayString()) + } + }) + + t.Run("PrattIgnoreExtraParens", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(1), EnablePrattParser(true)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("((((1))))")) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error: %s", errs.ToDisplayString()) + } + }) + + t.Run("PrattDeeplyNestedParens1000", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(1), EnablePrattParser(true)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + expr1 := strings.Repeat("(", 1000) + "42" + strings.Repeat(")", 1000) + _, errs := p.Parse(common.NewTextSource(expr1)) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on 1000 parens literal: %s", errs.ToDisplayString()) + } + + expr2 := strings.Repeat("(", 1000) + "1 + 2" + strings.Repeat(")", 1000) + _, errs = p.Parse(common.NewTextSource(expr2)) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on 1000 parens binary: %s", errs.ToDisplayString()) + } + }) +} + func BenchmarkParse(b *testing.B) { p, err := NewParser( Macros(AllMacros...), diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go index 1350a7b5b..f9163322d 100644 --- a/parser/pratt_parser.go +++ b/parser/pratt_parser.go @@ -16,7 +16,6 @@ package parser import ( "fmt" - "math" "strconv" "strings" @@ -86,7 +85,7 @@ func getBinaryOpInfo(kind tokenKind) binaryOpInfo { } } -type prattParser struct { +type prattParserWorker struct { content runes.Buffer length int32 helper *parserHelper @@ -109,53 +108,13 @@ type prattParser struct { enableIdentEscapeSyntax bool } -// PrattParser encapsulates the context necessary to perform Pratt parsing for different expressions. -type PrattParser struct { +// prattParser encapsulates the context necessary to perform Pratt parsing for different expressions. +type prattParser struct { options } -// NewPrattParser builds and returns a new PrattParser using the provided options. -func NewPrattParser(opts ...Option) (*PrattParser, error) { - p := &PrattParser{} - p.enableHiddenAccumulatorName = true - p.enableIdentEscapeSyntax = true - for _, opt := range opts { - if err := opt(&p.options); err != nil { - return nil, err - } - } - if p.errorReportingLimit == 0 { - p.errorReportingLimit = 100 - } - if p.maxRecursionDepth == 0 { - p.maxRecursionDepth = 250 - } - if p.maxRecursionDepth == -1 { - p.maxRecursionDepth = math.MaxInt - } - if p.errorRecoveryLimit == 0 { - p.errorRecoveryLimit = 30 - } - if p.errorRecoveryLimit == -1 { - p.errorRecoveryLimit = math.MaxInt - } - if p.expressionSizeCodePointLimit == 0 { - p.expressionSizeCodePointLimit = 100_000 - } - if p.expressionSizeCodePointLimit == -1 { - p.expressionSizeCodePointLimit = math.MaxInt - } - if p.maxExpressionNodeCount == 0 { - p.maxExpressionNodeCount = 100_000 - } - if p.maxExpressionNodeCount == -1 { - p.maxExpressionNodeCount = math.MaxInt - } - return p, nil -} - // Parse parses the expression represented by source using the Pratt parser and returns the result. -func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { +func (p *prattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { errs := common.NewErrors(source) buf, ok := source.(runes.Buffer) if !ok { @@ -172,7 +131,7 @@ func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { accu = HiddenAccumulatorName } fac := ast.NewExprFactoryWithAccumulator(accu) - pratt := &prattParser{ + pratt := &prattParserWorker{ content: buf, length: int32(buf.Len()), helper: newParserHelper(source, fac), @@ -197,16 +156,16 @@ func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { return ast.NewAST(out, pratt.helper.getSourceInfo()), errs } -func (p *prattParser) initTokenStream() { +func (p *prattParserWorker) initTokenStream() { p.currTok = token{kind: tokError, start: 0, end: 0} p.peekTok = p.nextSignificantToken(true) } -func (p *prattParser) isRecoveryLimitExceeded() bool { +func (p *prattParserWorker) isRecoveryLimitExceeded() bool { return p.errorCount > p.errorRecoveryLimit } -func (p *prattParser) nextSignificantToken(reportError bool) token { +func (p *prattParserWorker) nextSignificantToken(reportError bool) token { if p.isRecoveryLimitExceeded() { return token{kind: tokEnd, start: p.length, end: p.length} } @@ -225,7 +184,7 @@ func (p *prattParser) nextSignificantToken(reportError bool) token { } } -func (p *prattParser) nextToken() token { +func (p *prattParserWorker) nextToken() token { p.currTok = p.peekTok if p.isRecoveryLimitExceeded() { p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} @@ -237,18 +196,18 @@ func (p *prattParser) nextToken() token { return p.currTok } -func (p *prattParser) tokenText(tok token) string { +func (p *prattParserWorker) tokenText(tok token) string { if tok.start >= 0 && tok.end >= tok.start && tok.end <= p.length { return p.content.Slice(int(tok.start), int(tok.end)) } return "" } -func (p *prattParser) nextID(tok token) int64 { +func (p *prattParserWorker) nextID(tok token) int64 { return p.helper.idFromOffsets(tok.start, tok.end) } -func (p *prattParser) expect(kind tokenKind, msg string) bool { +func (p *prattParserWorker) expect(kind tokenKind, msg string) bool { if p.peekTok.kind == kind { p.nextToken() return true @@ -271,7 +230,7 @@ func (p *prattParser) expect(kind tokenKind, msg string) bool { return false } -func (p *prattParser) synchronizeOnDelimiter() { +func (p *prattParserWorker) synchronizeOnDelimiter() { if p.isRecoveryLimitExceeded() { p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} return @@ -287,7 +246,7 @@ func (p *prattParser) synchronizeOnDelimiter() { } } -func (p *prattParser) reportError(ctx any, format string, args ...any) ast.Expr { +func (p *prattParserWorker) reportError(ctx any, format string, args ...any) ast.Expr { if p.isRecoveryLimitExceeded() { return p.helper.newExpr(common.NoLocation) } @@ -311,28 +270,28 @@ func (p *prattParser) reportError(ctx any, format string, args ...any) ast.Expr return err } -func (p *prattParser) newLogicManager(function string, term ast.Expr) *logicManager { +func (p *prattParserWorker) newLogicManager(function string, term ast.Expr) *logicManager { if p.enableVariadicOperatorASTs { return newVariadicLogicManager(p.exprFactory, function, term) } return newBalancingLogicManager(p.exprFactory, function, term) } -func (p *prattParser) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { +func (p *prattParserWorker) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, nil, args...); found { return expr } return p.helper.newGlobalCall(exprID, function, args...) } -func (p *prattParser) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { +func (p *prattParserWorker) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, target, args...); found { return expr } return p.helper.newReceiverCall(exprID, function, target, args...) } -func (p *prattParser) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { +func (p *prattParserWorker) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { if len(p.macros) == 0 { return nil, false } @@ -376,7 +335,7 @@ func (p *prattParser) expandMacro(exprID int64, function string, target ast.Expr return expr, true } -func (p *prattParser) normalizeIdent(tok token, allowQuoted bool) string { +func (p *prattParserWorker) normalizeIdent(tok token, allowQuoted bool) string { text := p.tokenText(tok) if len(text) == 0 { return "" @@ -411,7 +370,7 @@ func (p *prattParser) normalizeIdent(tok token, allowQuoted bool) string { return text } -func (p *prattParser) parse() ast.Expr { +func (p *prattParserWorker) parse() ast.Expr { expr := p.parseExpr() if p.recursionLimitExceeded || p.isRecoveryLimitExceeded() { return expr @@ -427,7 +386,7 @@ func (p *prattParser) parse() ast.Expr { return expr } -func (p *prattParser) parseExpr() ast.Expr { +func (p *prattParserWorker) parseExpr() ast.Expr { if p.recursionLimitExceeded || p.isRecoveryLimitExceeded() { return p.helper.newExpr(common.NoLocation) } @@ -442,7 +401,7 @@ func (p *prattParser) parseExpr() ast.Expr { return expr } -func (p *prattParser) parseBinaryAndTernary(minPrec int) ast.Expr { +func (p *prattParserWorker) parseBinaryAndTernary(minPrec int) ast.Expr { lhs := p.parseSelectorChain() for { tok := p.peekTok.kind @@ -469,7 +428,7 @@ func (p *prattParser) parseBinaryAndTernary(minPrec int) ast.Expr { return lhs } -func (p *prattParser) parseTernary(lhs ast.Expr) ast.Expr { +func (p *prattParserWorker) parseTernary(lhs ast.Expr) ast.Expr { qTok := p.nextToken() opID := p.nextID(qTok) trueExpr := p.parseBinaryAndTernary(1) @@ -480,7 +439,7 @@ func (p *prattParser) parseTernary(lhs ast.Expr) ast.Expr { return p.helper.newGlobalCall(opID, operators.Conditional, lhs, trueExpr, falseExpr) } -func (p *prattParser) parseLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.Expr { +func (p *prattParserWorker) parseLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.Expr { l := p.newLogicManager(opInfo.name, lhs) for p.peekTok.kind == opInfo.kind { opTok := p.nextToken() @@ -491,12 +450,12 @@ func (p *prattParser) parseLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.E return l.toExpr() } -func (p *prattParser) parseSelectorChain() ast.Expr { +func (p *prattParserWorker) parseSelectorChain() ast.Expr { lhs := p.parseUnary() return p.parseSelectorChainTail(lhs) } -func (p *prattParser) parseSelectorChainTail(lhs ast.Expr) ast.Expr { +func (p *prattParserWorker) parseSelectorChainTail(lhs ast.Expr) ast.Expr { for { switch p.peekTok.kind { case tokDot: @@ -567,7 +526,7 @@ func (p *prattParser) parseSelectorChainTail(lhs ast.Expr) ast.Expr { } } -func (p *prattParser) extractStructName(expr ast.Expr) (string, bool) { +func (p *prattParserWorker) extractStructName(expr ast.Expr) (string, bool) { if expr == nil || expr.Kind() == ast.LiteralKind { return "", false } @@ -591,7 +550,7 @@ func (p *prattParser) extractStructName(expr ast.Expr) (string, bool) { return "", false } -func (p *prattParser) parseStruct(objID int64, structName string) ast.Expr { +func (p *prattParserWorker) parseStruct(objID int64, structName string) ast.Expr { p.nextToken() // consumes { var fields []ast.EntryExpr for p.peekTok.kind != tokRightBrace && p.peekTok.kind != tokEnd { @@ -627,7 +586,7 @@ func (p *prattParser) parseStruct(objID int64, structName string) ast.Expr { return p.helper.newObject(objID, structName, fields...) } -func (p *prattParser) parseUnary() ast.Expr { +func (p *prattParserWorker) parseUnary() ast.Expr { tok := p.peekTok.kind if tok == tokExclamation || tok == tokMinus { return p.parseUnaryOps() @@ -635,7 +594,7 @@ func (p *prattParser) parseUnary() ast.Expr { return p.parsePrimary() } -func (p *prattParser) parseUnaryOps() ast.Expr { +func (p *prattParserWorker) parseUnaryOps() ast.Expr { op := p.nextToken() if p.peekTok.kind == tokExclamation || p.peekTok.kind == tokMinus { return p.parseUnaryOpsChain(op) @@ -659,7 +618,7 @@ func (p *prattParser) parseUnaryOps() ast.Expr { return p.globalCallOrMacro(opID, opName, operand) } -func (p *prattParser) parseUnaryOpsChain(firstOp token) ast.Expr { +func (p *prattParserWorker) parseUnaryOpsChain(firstOp token) ast.Expr { type unaryOpInfo struct { kind tokenKind id int64 @@ -699,7 +658,7 @@ func (p *prattParser) parseUnaryOpsChain(firstOp token) ast.Expr { return operand } -func (p *prattParser) countGroupingParentheses() int { +func (p *prattParserWorker) countGroupingParentheses() int { if p.peekTok.kind != tokLeftParen { return 0 } @@ -748,7 +707,7 @@ func (p *prattParser) countGroupingParentheses() int { return 1 } -func (p *prattParser) parsePrimary() ast.Expr { +func (p *prattParserWorker) parsePrimary() ast.Expr { switch p.peekTok.kind { case tokLeftParen: groupingCount := p.countGroupingParentheses() @@ -797,7 +756,7 @@ func (p *prattParser) parsePrimary() ast.Expr { } } -func (p *prattParser) parseList() ast.Expr { +func (p *prattParserWorker) parseList() ast.Expr { openTok := p.nextToken() listID := p.nextID(openTok) var elems []ast.Expr @@ -829,7 +788,7 @@ func (p *prattParser) parseList() ast.Expr { return p.helper.newList(listID, elems, optionals...) } -func (p *prattParser) parseMap() ast.Expr { +func (p *prattParserWorker) parseMap() ast.Expr { openTok := p.nextToken() mapID := p.nextID(openTok) var entries []ast.EntryExpr @@ -863,7 +822,7 @@ func (p *prattParser) parseMap() ast.Expr { return p.helper.newMap(mapID, entries...) } -func (p *prattParser) parseIdentOrCall() ast.Expr { +func (p *prattParserWorker) parseIdentOrCall() ast.Expr { leadingDot := false firstTok := p.peekTok if p.peekTok.kind == tokDot { @@ -896,7 +855,7 @@ func (p *prattParser) parseIdentOrCall() ast.Expr { return p.helper.newIdent(id, name) } -func (p *prattParser) parseArguments(closeTok tokenKind) []ast.Expr { +func (p *prattParserWorker) parseArguments(closeTok tokenKind) []ast.Expr { var args []ast.Expr if p.peekTok.kind != closeTok && p.peekTok.kind != tokEnd { for { @@ -915,7 +874,7 @@ func (p *prattParser) parseArguments(closeTok tokenKind) []ast.Expr { return args } -func (p *prattParser) parseIntLiteral() ast.Expr { +func (p *prattParserWorker) parseIntLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -931,7 +890,7 @@ func (p *prattParser) parseIntLiteral() ast.Expr { return p.helper.newLiteralInt(id, val) } -func (p *prattParser) parseNegativeIntLiteral(opID int64) ast.Expr { +func (p *prattParserWorker) parseNegativeIntLiteral(opID int64) ast.Expr { tok := p.nextToken() text := p.tokenText(tok) base := 10 @@ -946,7 +905,7 @@ func (p *prattParser) parseNegativeIntLiteral(opID int64) ast.Expr { return p.helper.newLiteralInt(opID, val) } -func (p *prattParser) parseUintLiteral() ast.Expr { +func (p *prattParserWorker) parseUintLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -963,7 +922,7 @@ func (p *prattParser) parseUintLiteral() ast.Expr { return p.helper.newLiteralUint(id, val) } -func (p *prattParser) parseDoubleLiteral() ast.Expr { +func (p *prattParserWorker) parseDoubleLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -974,7 +933,7 @@ func (p *prattParser) parseDoubleLiteral() ast.Expr { return p.helper.newLiteralDouble(id, val) } -func (p *prattParser) parseNegativeDoubleLiteral(opID int64) ast.Expr { +func (p *prattParserWorker) parseNegativeDoubleLiteral(opID int64) ast.Expr { tok := p.nextToken() text := p.tokenText(tok) val, err := strconv.ParseFloat(text, 64) @@ -984,7 +943,7 @@ func (p *prattParser) parseNegativeDoubleLiteral(opID int64) ast.Expr { return p.helper.newLiteralDouble(opID, -val) } -func (p *prattParser) parseStringLiteral() ast.Expr { +func (p *prattParserWorker) parseStringLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -995,7 +954,7 @@ func (p *prattParser) parseStringLiteral() ast.Expr { return p.helper.newLiteralString(id, unescaped) } -func (p *prattParser) parseBytesLiteral() ast.Expr { +func (p *prattParserWorker) parseBytesLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) diff --git a/parser/pratt_parser_test.go b/parser/pratt_parser_test.go deleted file mode 100644 index 4dd4e398a..000000000 --- a/parser/pratt_parser_test.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package parser - -import ( - "strings" - "testing" - - "cel.dev/cel-go/common" -) - -func TestPrattParserSourceInfoPositions(t *testing.T) { - src := common.NewTextSource("a + b") - p, err := NewPrattParser() - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(src) - if len(errs.GetErrors()) > 0 { - t.Fatalf("Parse() failed: %s", errs.ToDisplayString()) - } - sourceInfo := parsed.SourceInfo() - root := parsed.Expr() - if sourceInfo.GetStartLocation(root.ID()).Column() != 2 { - t.Errorf("expected root column 2, got %d", sourceInfo.GetStartLocation(root.ID()).Column()) - } - args := root.AsCall().Args() - if len(args) != 2 { - t.Fatalf("expected 2 args, got %d", len(args)) - } - if sourceInfo.GetStartLocation(args[0].ID()).Column() != 0 { - t.Errorf("expected arg[0] column 0, got %d", sourceInfo.GetStartLocation(args[0].ID()).Column()) - } - if sourceInfo.GetStartLocation(args[1].ID()).Column() != 4 { - t.Errorf("expected arg[1] column 4, got %d", sourceInfo.GetStartLocation(args[1].ID()).Column()) - } -} - -func TestPrattParserRecursionDepth(t *testing.T) { - t.Run("DeeplyNestedBracketsLimitExceeded", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(5)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("[[[[[[1]]]]]]")) - if len(errs.GetErrors()) == 0 { - t.Errorf("expected recursion limit error, got none") - } - }) - - t.Run("IgnoreExtraParens", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(1)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("((((1))))")) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error: %s", errs.ToDisplayString()) - } - }) - - t.Run("DeeplyNestedParens1000", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(1)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - expr1 := strings.Repeat("(", 1000) + "42" + strings.Repeat(")", 1000) - _, errs := p.Parse(common.NewTextSource(expr1)) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error on 1000 parens literal: %s", errs.ToDisplayString()) - } - - expr2 := strings.Repeat("(", 1000) + "1 + 2" + strings.Repeat(")", 1000) - _, errs = p.Parse(common.NewTextSource(expr2)) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error on 1000 parens binary: %s", errs.ToDisplayString()) - } - }) - - t.Run("SequentialScopesDoNotAccumulateDepth", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(2)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("[1] + [2] + [3]")) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error on sequential scopes: %s", errs.ToDisplayString()) - } - }) -} - -func TestPrattParserMacroCalls(t *testing.T) { - t.Run("DisabledByDefault", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(false)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - if len(parsed.SourceInfo().MacroCalls()) != 0 { - t.Errorf("expected 0 macro calls, got %d", len(parsed.SourceInfo().MacroCalls())) - } - }) - - t.Run("GlobalMacroCallRecorded", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - macroCalls := parsed.SourceInfo().MacroCalls() - if len(macroCalls) != 1 { - t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) - } - }) - - t.Run("ReceiverMacroCallRecorded", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("[1, 2].exists(x, x > 0)")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - macroCalls := parsed.SourceInfo().MacroCalls() - if len(macroCalls) != 1 { - t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) - } - }) - - t.Run("NestedMacroCallsRecorded", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("[1, 2].all(x, has(x.b))")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - macroCalls := parsed.SourceInfo().MacroCalls() - if len(macroCalls) != 2 { - t.Fatalf("expected 2 macro calls, got %d", len(macroCalls)) - } - }) -} - -func TestPrattParserErrorRecoveryLimits(t *testing.T) { - t.Run("LimitZero", func(t *testing.T) { - p, err := NewPrattParser(ErrorRecoveryLimit(0)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("......")) - if len(errs.GetErrors()) == 0 { - t.Errorf("expected error recovery limit error, got none") - } - }) - - t.Run("LimitOne", func(t *testing.T) { - p, err := NewPrattParser(ErrorRecoveryLimit(1)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("......")) - if len(errs.GetErrors()) == 0 { - t.Errorf("expected error recovery limit error, got none") - } - }) -} - -func TestPrattParserExpressionSizeCodePointLimit(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), ExpressionSizeCodePointLimit(2)) - if err != nil { - t.Fatal(err) - } - src := common.NewTextSource("foo") - _, errs := p.Parse(src) - if got, want := len(errs.GetErrors()), 1; got != want { - t.Fatalf("got %d errors, want %d errors: %s", got, want, errs.ToDisplayString()) - } - if got, want := errs.GetErrors()[0].Message, "expression code point size exceeds limit: size: 3, limit 2"; got != want { - t.Fatalf("got %q, want %q: %s", got, want, errs.GetErrors()[0].ToDisplayString(src)) - } -} - -func BenchmarkParsers(b *testing.B) { - exprs := []string{ - `42`, - `a > 5 && b < 10 || c == "xyz"`, - `[1, 2, 3].all(x, x > 0) && [4, 5, 6].exists(y, y == 5)`, - `pkg.Msg{field1: "value", field2: 123, list_field: [1, 2, 3], map_field: {"a": true, "b": false}}`, - `a.b.c.d.e.f(1, 2, [3, ?4], {?5: 6}) ? (x + y * z - w / v) : (!p && !q || r.s)`, - } - - antlrParser, _ := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnableOptionalSyntax(true)) - prattParser, _ := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true), EnableOptionalSyntax(true)) - - for _, expr := range exprs { - src := common.NewTextSource(expr) - - b.Run("ANTLR/"+expr[:min(len(expr), 20)], func(b *testing.B) { - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = antlrParser.Parse(src) - } - }) - - b.Run("Pratt/"+expr[:min(len(expr), 20)], func(b *testing.B) { - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = prattParser.Parse(src) - } - }) - } -} - -func min(a, b int) int { - if a < b { - return a - } - return b -}