diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel index e4ff679d1..c9a71d222 100644 --- a/parser/BUILD.bazel +++ b/parser/BUILD.bazel @@ -7,6 +7,7 @@ package( go_library( name = "go_default_library", srcs = [ + "antlr_parser.go", "errors.go", "helper.go", "input.go", diff --git a/parser/antlr_parser.go b/parser/antlr_parser.go new file mode 100644 index 000000000..e755b7ac7 --- /dev/null +++ b/parser/antlr_parser.go @@ -0,0 +1,989 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + antlr "github.com/antlr4-go/antlr/v4" + + "cel.dev/cel-go/common" + "cel.dev/cel-go/common/ast" + "cel.dev/cel-go/common/operators" + "cel.dev/cel-go/common/runes" + "cel.dev/cel-go/common/types" + "cel.dev/cel-go/parser/gen" +) + +// antlrParser encapsulates the context necessary to perform ANTLR parsing for different expressions. +type antlrParser struct { + options +} + +// Parse parses the expression represented by source and returns the result. +func (p *antlrParser) Parse(source common.Source) (*ast.AST, *common.Errors) { + errs := common.NewErrors(source) + accu := AccumulatorName + if p.enableHiddenAccumulatorName { + accu = HiddenAccumulatorName + } + fac := ast.NewExprFactoryWithAccumulator(accu) + impl := antlrVisitor{ + errors: &parseErrors{errs}, + exprFactory: fac, + helper: newParserHelper(source, fac), + macros: p.macros, + maxRecursionDepth: p.maxRecursionDepth, + maxExpressionNodeCount: p.maxExpressionNodeCount, + errorReportingLimit: p.errorReportingLimit, + errorRecoveryLimit: p.errorRecoveryLimit, + errorRecoveryLookaheadTokenLimit: p.errorRecoveryTokenLookaheadLimit, + populateMacroCalls: p.populateMacroCalls, + enableOptionalSyntax: p.enableOptionalSyntax, + enableVariadicOperatorASTs: p.enableVariadicOperatorASTs, + enableIdentEscapeSyntax: p.enableIdentEscapeSyntax, + } + buf, ok := source.(runes.Buffer) + if !ok { + buf = runes.NewBuffer(source.Content()) + } + var out ast.Expr + if buf.Len() > p.expressionSizeCodePointLimit { + out = impl.reportError(common.NoLocation, + "expression code point size exceeds limit: size: %d, limit %d", + buf.Len(), p.expressionSizeCodePointLimit) + } else { + out = impl.parse(buf, source.Description()) + } + return ast.NewAST(out, impl.helper.getSourceInfo()), errs +} + +func unescapeIdent(in string) (string, error) { + if len(in) <= 2 { + return "", errors.New("invalid escaped identifier: underflow") + } + return in[1 : len(in)-1], nil +} + +type recursionError struct { + message string +} + +// Error implements error. +func (re *recursionError) Error() string { + return re.message +} + +var _ error = &recursionError{} + +type recursionListener struct { + maxDepth int + ruleTypeDepth map[int]*int +} + +func (rl *recursionListener) VisitTerminal(node antlr.TerminalNode) {} + +func (rl *recursionListener) VisitErrorNode(node antlr.ErrorNode) {} + +func (rl *recursionListener) EnterEveryRule(ctx antlr.ParserRuleContext) { + if ctx == nil { + return + } + ruleIndex := ctx.GetRuleIndex() + depth, found := rl.ruleTypeDepth[ruleIndex] + if !found { + var counter = 1 + rl.ruleTypeDepth[ruleIndex] = &counter + depth = &counter + } else { + *depth++ + } + if *depth > rl.maxDepth { + panic(&recursionError{ + message: fmt.Sprintf("expression recursion limit exceeded: %d", rl.maxDepth), + }) + } +} + +func (rl *recursionListener) ExitEveryRule(ctx antlr.ParserRuleContext) { + if ctx == nil { + return + } + ruleIndex := ctx.GetRuleIndex() + if depth, found := rl.ruleTypeDepth[ruleIndex]; found && *depth > 0 { + *depth-- + } +} + +var _ antlr.ParseTreeListener = &recursionListener{} + +type tooManyErrors struct { + errorReportingLimit int +} + +func (t *tooManyErrors) Error() string { + return fmt.Sprintf("More than %d syntax errors", t.errorReportingLimit) +} + +var _ error = &tooManyErrors{} + +type recoveryLimitError struct { + message string +} + +// Error implements error. +func (rl *recoveryLimitError) Error() string { + return rl.message +} + +type lookaheadLimitError struct { + message string +} + +func (ll *lookaheadLimitError) Error() string { + return ll.message +} + +var _ error = &recoveryLimitError{} + +type recoveryLimitErrorStrategy struct { + *antlr.DefaultErrorStrategy + errorRecoveryLimit int + errorRecoveryTokenLookaheadLimit int + recoveryAttempts int +} + +type lookaheadConsumer struct { + antlr.Parser + errorRecoveryTokenLookaheadLimit int + lookaheadAttempts int +} + +func (lc *lookaheadConsumer) Consume() antlr.Token { + if lc.lookaheadAttempts >= lc.errorRecoveryTokenLookaheadLimit { + panic(&lookaheadLimitError{ + message: fmt.Sprintf("error recovery token lookahead limit exceeded: %d", lc.errorRecoveryTokenLookaheadLimit), + }) + } + lc.lookaheadAttempts++ + return lc.Parser.Consume() +} + +func (rl *recoveryLimitErrorStrategy) Recover(recognizer antlr.Parser, e antlr.RecognitionException) { + rl.checkAttempts(recognizer) + lc := &lookaheadConsumer{Parser: recognizer, errorRecoveryTokenLookaheadLimit: rl.errorRecoveryTokenLookaheadLimit} + rl.DefaultErrorStrategy.Recover(lc, e) +} + +func (rl *recoveryLimitErrorStrategy) RecoverInline(recognizer antlr.Parser) antlr.Token { + rl.checkAttempts(recognizer) + lc := &lookaheadConsumer{Parser: recognizer, errorRecoveryTokenLookaheadLimit: rl.errorRecoveryTokenLookaheadLimit} + return rl.DefaultErrorStrategy.RecoverInline(lc) +} + +func (rl *recoveryLimitErrorStrategy) checkAttempts(recognizer antlr.Parser) { + if rl.recoveryAttempts == rl.errorRecoveryLimit { + rl.recoveryAttempts++ + msg := fmt.Sprintf("error recovery attempt limit exceeded: %d", rl.errorRecoveryLimit) + recognizer.NotifyErrorListeners(msg, nil, nil) + panic(&recoveryLimitError{ + message: msg, + }) + } + rl.recoveryAttempts++ +} + +var _ antlr.ErrorStrategy = &recoveryLimitErrorStrategy{} + +type antlrVisitor struct { + gen.BaseCELVisitor + errors *parseErrors + exprFactory ast.ExprFactory + helper *parserHelper + macros map[string]Macro + recursionDepth int + errorReports int + maxRecursionDepth int + maxExpressionNodeCount int + errorReportingLimit int + errorRecoveryLimit int + errorRecoveryLookaheadTokenLimit int + populateMacroCalls bool + enableOptionalSyntax bool + enableVariadicOperatorASTs bool + enableIdentEscapeSyntax bool +} + +var _ gen.CELVisitor = (*antlrVisitor)(nil) + +// normalizeIdent returns the interpreted identifier. +func (p *antlrVisitor) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error) { + switch ident := ctx.(type) { + case *gen.SimpleIdentifierContext: + return ident.GetId().GetText(), nil + case *gen.EscapedIdentifierContext: + if !p.enableIdentEscapeSyntax { + return "", errors.New("unsupported syntax: '`'") + } + return unescapeIdent(ident.GetId().GetText()) + } + return "", errors.New("unsupported ident kind") +} + +func (p *antlrVisitor) parse(expr runes.Buffer, desc string) ast.Expr { + lexer := gen.NewCELLexer(newCharStream(expr, desc)) + lexer.RemoveErrorListeners() + lexer.AddErrorListener(p) + + prsr := gen.NewCELParser(antlr.NewCommonTokenStream(lexer, 0)) + prsr.RemoveErrorListeners() + + prsrListener := &recursionListener{ + maxDepth: p.maxRecursionDepth, + ruleTypeDepth: map[int]*int{}, + } + + prsr.AddErrorListener(p) + prsr.AddParseListener(prsrListener) + + prsr.SetErrorHandler(&recoveryLimitErrorStrategy{ + DefaultErrorStrategy: antlr.NewDefaultErrorStrategy(), + errorRecoveryLimit: p.errorRecoveryLimit, + errorRecoveryTokenLookaheadLimit: p.errorRecoveryLookaheadTokenLimit, + }) + + defer func() { + if val := recover(); val != nil { + switch err := val.(type) { + case *lookaheadLimitError: + p.errors.internalError(err.Error()) + case *recursionError: + p.errors.internalError(err.Error()) + case *tooManyErrors: + // do nothing + case *recoveryLimitError: + // do nothing, listeners already notified and error reported. + default: + panic(val) + } + } + }() + + return p.Visit(prsr.Start_()).(ast.Expr) +} + +// Visitor implementations. +func (p *antlrVisitor) Visit(tree antlr.ParseTree) any { + t := unnest(tree) + switch tree := t.(type) { + case *gen.StartContext: + return p.VisitStart(tree) + case *gen.ExprContext: + p.checkAndIncrementRecursionDepth() + out := p.VisitExpr(tree) + p.decrementRecursionDepth() + return out + case *gen.ConditionalAndContext: + return p.VisitConditionalAnd(tree) + case *gen.ConditionalOrContext: + return p.VisitConditionalOr(tree) + case *gen.RelationContext: + p.checkAndIncrementRecursionDepth() + out := p.VisitRelation(tree) + p.decrementRecursionDepth() + return out + case *gen.CalcContext: + p.checkAndIncrementRecursionDepth() + out := p.VisitCalc(tree) + p.decrementRecursionDepth() + return out + case *gen.LogicalNotContext: + return p.VisitLogicalNot(tree) + case *gen.IdentContext: + return p.VisitIdent(tree) + case *gen.GlobalCallContext: + return p.VisitGlobalCall(tree) + case *gen.SelectContext: + p.checkAndIncrementRecursionDepth() + out := p.VisitSelect(tree) + p.decrementRecursionDepth() + return out + case *gen.MemberCallContext: + p.checkAndIncrementRecursionDepth() + out := p.VisitMemberCall(tree) + p.decrementRecursionDepth() + return out + case *gen.MapInitializerListContext: + return p.VisitMapInitializerList(tree) + case *gen.NegateContext: + return p.VisitNegate(tree) + case *gen.IndexContext: + p.checkAndIncrementRecursionDepth() + out := p.VisitIndex(tree) + p.decrementRecursionDepth() + return out + case *gen.UnaryContext: + return p.VisitUnary(tree) + case *gen.CreateListContext: + return p.VisitCreateList(tree) + case *gen.CreateMessageContext: + return p.VisitCreateMessage(tree) + case *gen.CreateStructContext: + return p.VisitCreateStruct(tree) + case *gen.IntContext: + return p.VisitInt(tree) + case *gen.UintContext: + return p.VisitUint(tree) + case *gen.DoubleContext: + return p.VisitDouble(tree) + case *gen.StringContext: + return p.VisitString(tree) + case *gen.BytesContext: + return p.VisitBytes(tree) + case *gen.BoolFalseContext: + return p.VisitBoolFalse(tree) + case *gen.BoolTrueContext: + return p.VisitBoolTrue(tree) + case *gen.NullContext: + return p.VisitNull(tree) + } + + // Report at least one error if the parser reaches an unknown parse element. + // Typically, this happens if the parser has already encountered a syntax error elsewhere. + if p.errors.errorCount() == 0 { + txt := "<>" + if t != nil { + txt = fmt.Sprintf("<<%T>>", t) + } + return p.reportError(common.NoLocation, "unknown parse element encountered: %s", txt) + } + return p.helper.newExpr(common.NoLocation) +} + +// Visit a parse tree produced by CELParser#start. +func (p *antlrVisitor) VisitStart(ctx *gen.StartContext) any { + return p.Visit(ctx.Expr()) +} + +// Visit a parse tree produced by CELParser#expr. +func (p *antlrVisitor) VisitExpr(ctx *gen.ExprContext) any { + result := p.Visit(ctx.GetE()).(ast.Expr) + if ctx.GetOp() == nil { + return result + } + opID := p.helper.id(ctx.GetOp()) + ifTrue := p.Visit(ctx.GetE1()).(ast.Expr) + ifFalse := p.Visit(ctx.GetE2()).(ast.Expr) + return p.globalCallOrMacro(opID, operators.Conditional, result, ifTrue, ifFalse) +} + +// Visit a parse tree produced by CELParser#conditionalOr. +func (p *antlrVisitor) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { + result := p.Visit(ctx.GetE()).(ast.Expr) + l := p.newLogicManager(operators.LogicalOr, result) + rest := ctx.GetE1() + for i, op := range ctx.GetOps() { + if i >= len(rest) { + return p.reportError(ctx, "unexpected character, wanted '||'") + } + next := p.Visit(rest[i]).(ast.Expr) + opID := p.helper.id(op) + l.addTerm(opID, next) + } + return l.toExpr() +} + +// Visit a parse tree produced by CELParser#conditionalAnd. +func (p *antlrVisitor) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { + result := p.Visit(ctx.GetE()).(ast.Expr) + l := p.newLogicManager(operators.LogicalAnd, result) + rest := ctx.GetE1() + for i, op := range ctx.GetOps() { + if i >= len(rest) { + return p.reportError(ctx, "unexpected character, wanted '&&'") + } + next := p.Visit(rest[i]).(ast.Expr) + opID := p.helper.id(op) + l.addTerm(opID, next) + } + return l.toExpr() +} + +// Visit a parse tree produced by CELParser#relation. +func (p *antlrVisitor) VisitRelation(ctx *gen.RelationContext) any { + opText := "" + if ctx.GetOp() != nil { + opText = ctx.GetOp().GetText() + } + if op, found := operators.Find(opText); found { + lhs := p.Visit(ctx.Relation(0)).(ast.Expr) + opID := p.helper.id(ctx.GetOp()) + rhs := p.Visit(ctx.Relation(1)).(ast.Expr) + return p.globalCallOrMacro(opID, op, lhs, rhs) + } + return p.reportError(ctx, "operator not found") +} + +// Visit a parse tree produced by CELParser#calc. +func (p *antlrVisitor) VisitCalc(ctx *gen.CalcContext) any { + opText := "" + if ctx.GetOp() != nil { + opText = ctx.GetOp().GetText() + } + if op, found := operators.Find(opText); found { + lhs := p.Visit(ctx.Calc(0)).(ast.Expr) + opID := p.helper.id(ctx.GetOp()) + rhs := p.Visit(ctx.Calc(1)).(ast.Expr) + return p.globalCallOrMacro(opID, op, lhs, rhs) + } + return p.reportError(ctx, "operator not found") +} + +func (p *antlrVisitor) VisitUnary(ctx *gen.UnaryContext) any { + return p.helper.newLiteralString(ctx, "<>") +} + +// Visit a parse tree produced by CELParser#LogicalNot. +func (p *antlrVisitor) VisitLogicalNot(ctx *gen.LogicalNotContext) any { + if len(ctx.GetOps())%2 == 0 { + return p.Visit(ctx.Member()) + } + opID := p.helper.id(ctx.GetOps()[0]) + target := p.Visit(ctx.Member()).(ast.Expr) + return p.globalCallOrMacro(opID, operators.LogicalNot, target) +} + +func (p *antlrVisitor) VisitNegate(ctx *gen.NegateContext) any { + if len(ctx.GetOps())%2 == 0 { + return p.Visit(ctx.Member()) + } + opID := p.helper.id(ctx.GetOps()[0]) + target := p.Visit(ctx.Member()).(ast.Expr) + return p.globalCallOrMacro(opID, operators.Negate, target) +} + +// VisitSelect visits a parse tree produced by CELParser#Select. +func (p *antlrVisitor) VisitSelect(ctx *gen.SelectContext) any { + operand := p.Visit(ctx.Member()).(ast.Expr) + // Handle the error case where no valid identifier is specified. + if ctx.GetId() == nil || ctx.GetOp() == nil { + return p.helper.newExpr(ctx) + } + id, err := p.normalizeIdent(ctx.GetId()) + if err != nil { + p.reportError(ctx.GetId(), "%v", err) + } + if ctx.GetOpt() != nil { + if !p.enableOptionalSyntax { + return p.reportError(ctx.GetOp(), "unsupported syntax '.?'") + } + return p.helper.newGlobalCall( + ctx.GetOp(), + operators.OptSelect, + operand, + p.helper.newLiteralString(ctx.GetId(), id)) + } + return p.helper.newSelect(ctx.GetOp(), operand, id) +} + +// VisitMemberCall visits a parse tree produced by CELParser#MemberCall. +func (p *antlrVisitor) VisitMemberCall(ctx *gen.MemberCallContext) any { + operand := p.Visit(ctx.Member()).(ast.Expr) + // Handle the error case where no valid identifier is specified. + if ctx.GetId() == nil { + return p.helper.newExpr(ctx) + } + id := ctx.GetId().GetText() + opID := p.helper.id(ctx.GetOpen()) + return p.receiverCallOrMacro(opID, id, operand, p.visitExprList(ctx.GetArgs())...) +} + +// Visit a parse tree produced by CELParser#Index. +func (p *antlrVisitor) VisitIndex(ctx *gen.IndexContext) any { + target := p.Visit(ctx.Member()).(ast.Expr) + // Handle the error case where no valid identifier is specified. + if ctx.GetOp() == nil { + return p.helper.newExpr(ctx) + } + opID := p.helper.id(ctx.GetOp()) + index := p.Visit(ctx.GetIndex()).(ast.Expr) + operator := operators.Index + if ctx.GetOpt() != nil { + if !p.enableOptionalSyntax { + return p.reportError(ctx.GetOp(), "unsupported syntax '[?'") + } + operator = operators.OptIndex + } + return p.globalCallOrMacro(opID, operator, target, index) +} + +// Visit a parse tree produced by CELParser#CreateMessage. +func (p *antlrVisitor) VisitCreateMessage(ctx *gen.CreateMessageContext) any { + messageName := "" + for _, id := range ctx.GetIds() { + if len(messageName) != 0 { + messageName += "." + } + messageName += id.GetText() + } + if ctx.GetLeadingDot() != nil { + messageName = "." + messageName + } + objID := p.helper.id(ctx.GetOp()) + entries := p.VisitIFieldInitializerList(ctx.GetEntries()).([]ast.EntryExpr) + return p.helper.newObject(objID, messageName, entries...) +} + +// Visit a parse tree of field initializers. +func (p *antlrVisitor) VisitIFieldInitializerList(ctx gen.IFieldInitializerListContext) any { + if ctx == nil || ctx.GetFields() == nil { + // This is the result of a syntax error handled elswhere, return empty. + return []ast.EntryExpr{} + } + + result := make([]ast.EntryExpr, len(ctx.GetFields())) + cols := ctx.GetCols() + vals := ctx.GetValues() + for i, f := range ctx.GetFields() { + if i >= len(cols) || i >= len(vals) { + // This is the result of a syntax error detected elsewhere. + return []ast.EntryExpr{} + } + initID := p.helper.id(cols[i]) + optField := f.(*gen.OptFieldContext) + optional := optField.GetOpt() != nil + if !p.enableOptionalSyntax && optional { + p.reportError(optField, "unsupported syntax '?'") + continue + } + + // The field may be empty due to a prior error. + fieldName, err := p.normalizeIdent(optField.EscapeIdent()) + if err != nil { + p.reportError(ctx, "%v", err) + continue + } + + value := p.Visit(vals[i]).(ast.Expr) + field := p.helper.newObjectField(initID, fieldName, value, optional) + result[i] = field + } + return result +} + +// Visit a parse tree produced by CELParser#Ident. +func (p *antlrVisitor) VisitIdent(ctx *gen.IdentContext) any { + identName := "" + if ctx.GetLeadingDot() != nil { + identName = "." + } + // Handle the error case where no valid identifier is specified. + if ctx.GetId() == nil { + return p.helper.newExpr(ctx) + } + // Handle reserved identifiers. + id := ctx.GetId().GetText() + if _, ok := reservedIds[id]; ok { + return p.reportError(ctx, "reserved identifier: %s", id) + } + identName += id + return p.helper.newIdent(ctx.GetId(), identName) +} + +// Visit a parse tree produced by CELParser#GlobalCallContext. +func (p *antlrVisitor) VisitGlobalCall(ctx *gen.GlobalCallContext) any { + identName := "" + if ctx.GetLeadingDot() != nil { + identName = "." + } + // Handle the error case where no valid identifier is specified. + if ctx.GetId() == nil { + return p.helper.newExpr(ctx) + } + // Handle reserved identifiers. + id := ctx.GetId().GetText() + if _, ok := reservedIds[id]; ok { + return p.reportError(ctx, "reserved identifier: %s", id) + } + identName += id + opID := p.helper.id(ctx.GetOp()) + return p.globalCallOrMacro(opID, identName, p.visitExprList(ctx.GetArgs())...) +} + +// Visit a parse tree produced by CELParser#CreateList. +func (p *antlrVisitor) VisitCreateList(ctx *gen.CreateListContext) any { + listID := p.helper.id(ctx.GetOp()) + elems, optionals := p.visitListInit(ctx.GetElems()) + return p.helper.newList(listID, elems, optionals...) +} + +// Visit a parse tree produced by CELParser#CreateStruct. +func (p *antlrVisitor) VisitCreateStruct(ctx *gen.CreateStructContext) any { + structID := p.helper.id(ctx.GetOp()) + entries := []ast.EntryExpr{} + if ctx.GetEntries() != nil { + entries = p.Visit(ctx.GetEntries()).([]ast.EntryExpr) + } + return p.helper.newMap(structID, entries...) +} + +// Visit a parse tree produced by CELParser#mapInitializerList. +func (p *antlrVisitor) VisitMapInitializerList(ctx *gen.MapInitializerListContext) any { + if ctx == nil || ctx.GetKeys() == nil { + // This is the result of a syntax error handled elswhere, return empty. + return []ast.EntryExpr{} + } + + result := make([]ast.EntryExpr, len(ctx.GetCols())) + keys := ctx.GetKeys() + vals := ctx.GetValues() + for i, col := range ctx.GetCols() { + colID := p.helper.id(col) + if i >= len(keys) || i >= len(vals) { + // This is the result of a syntax error detected elsewhere. + return []ast.EntryExpr{} + } + optKey := keys[i] + optional := optKey.GetOpt() != nil + if !p.enableOptionalSyntax && optional { + p.reportError(optKey, "unsupported syntax '?'") + continue + } + key := p.Visit(optKey.GetE()).(ast.Expr) + value := p.Visit(vals[i]).(ast.Expr) + entry := p.helper.newMapEntry(colID, key, value, optional) + result[i] = entry + } + return result +} + +// Visit a parse tree produced by CELParser#Int. +func (p *antlrVisitor) VisitInt(ctx *gen.IntContext) any { + text := ctx.GetTok().GetText() + base := 10 + if strings.HasPrefix(text, "0x") { + base = 16 + text = text[2:] + } + if ctx.GetSign() != nil { + text = ctx.GetSign().GetText() + text + } + i, err := strconv.ParseInt(text, base, 64) + if err != nil { + return p.reportError(ctx, "invalid int literal") + } + return p.helper.newLiteralInt(ctx, i) +} + +// Visit a parse tree produced by CELParser#Uint. +func (p *antlrVisitor) VisitUint(ctx *gen.UintContext) any { + text := ctx.GetTok().GetText() + // trim the 'u' designator included in the uint literal. + text = text[:len(text)-1] + base := 10 + if strings.HasPrefix(text, "0x") { + base = 16 + text = text[2:] + } + i, err := strconv.ParseUint(text, base, 64) + if err != nil { + return p.reportError(ctx, "invalid uint literal") + } + return p.helper.newLiteralUint(ctx, i) +} + +// Visit a parse tree produced by CELParser#Double. +func (p *antlrVisitor) VisitDouble(ctx *gen.DoubleContext) any { + txt := ctx.GetTok().GetText() + if ctx.GetSign() != nil { + txt = ctx.GetSign().GetText() + txt + } + f, err := strconv.ParseFloat(txt, 64) + if err != nil { + return p.reportError(ctx, "invalid double literal") + } + return p.helper.newLiteralDouble(ctx, f) +} + +// Visit a parse tree produced by CELParser#String. +func (p *antlrVisitor) VisitString(ctx *gen.StringContext) any { + s := p.unquote(ctx, ctx.GetTok().GetText(), false) + return p.helper.newLiteralString(ctx, s) +} + +// Visit a parse tree produced by CELParser#Bytes. +func (p *antlrVisitor) VisitBytes(ctx *gen.BytesContext) any { + b := []byte(p.unquote(ctx, ctx.GetTok().GetText()[1:], true)) + return p.helper.newLiteralBytes(ctx, b) +} + +// Visit a parse tree produced by CELParser#BoolTrue. +func (p *antlrVisitor) VisitBoolTrue(ctx *gen.BoolTrueContext) any { + return p.helper.newLiteralBool(ctx, true) +} + +// Visit a parse tree produced by CELParser#BoolFalse. +func (p *antlrVisitor) VisitBoolFalse(ctx *gen.BoolFalseContext) any { + return p.helper.newLiteralBool(ctx, false) +} + +// Visit a parse tree produced by CELParser#Null. +func (p *antlrVisitor) VisitNull(ctx *gen.NullContext) any { + return p.helper.exprFactory.NewLiteral(p.helper.newID(ctx), types.NullValue) +} + +func (p *antlrVisitor) visitExprList(ctx gen.IExprListContext) []ast.Expr { + if ctx == nil { + return []ast.Expr{} + } + return p.visitSlice(ctx.GetE()) +} + +func (p *antlrVisitor) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int32) { + if ctx == nil { + return []ast.Expr{}, []int32{} + } + elements := ctx.GetElems() + result := make([]ast.Expr, len(elements)) + optionals := []int32{} + for i, e := range elements { + ex := p.Visit(e.GetE()).(ast.Expr) + if ex == nil { + return []ast.Expr{}, []int32{} + } + result[i] = ex + if e.GetOpt() != nil { + if !p.enableOptionalSyntax { + p.reportError(e.GetOpt(), "unsupported syntax '?'") + continue + } + optionals = append(optionals, int32(i)) + } + } + return result, optionals +} + +func (p *antlrVisitor) visitSlice(expressions []gen.IExprContext) []ast.Expr { + if expressions == nil { + return []ast.Expr{} + } + result := make([]ast.Expr, len(expressions)) + for i, e := range expressions { + ex := p.Visit(e).(ast.Expr) + result[i] = ex + } + return result +} + +func (p *antlrVisitor) unquote(ctx any, value string, isBytes bool) string { + text, err := unescape(value, isBytes) + if err != nil { + p.reportError(ctx, "%s", err.Error()) + return value + } + return text +} + +func (p *antlrVisitor) newLogicManager(function string, term ast.Expr) *logicManager { + if p.enableVariadicOperatorASTs { + return newVariadicLogicManager(p.exprFactory, function, term) + } + return newBalancingLogicManager(p.exprFactory, function, term) +} + +func (p *antlrVisitor) reportError(ctx any, format string, args ...any) ast.Expr { + var location common.Location + err := p.helper.newExpr(ctx) + switch c := ctx.(type) { + case common.Location: + location = c + case antlr.Token, antlr.ParserRuleContext: + location = p.helper.getLocation(err.ID()) + } + // Provide arguments to the report error. + p.errors.reportErrorAtID(err.ID(), location, format, args...) + return err +} + +// ANTLR Parse listener implementations +func (p *antlrVisitor) SyntaxError(recognizer antlr.Recognizer, offendingSymbol any, line, column int, msg string, e antlr.RecognitionException) { + offset := p.helper.sourceInfo.ComputeOffset(int32(line), int32(column)) + l := p.helper.getLocationByOffset(offset) + // Hack to keep existing error messages consistent with previous versions of CEL when a reserved word + // is used as an identifier. This behavior needs to be overhauled to provide consistent, normalized error + // messages out of ANTLR to prevent future breaking changes related to error message content. + if strings.Contains(msg, "no viable alternative") { + msg = reservedIdentifier.ReplaceAllString(msg, mismatchedReservedIdentifier) + } + // Ensure that no more than 100 syntax errors are reported as this will halt attempts to recover from a + // seriously broken expression. + if p.errorReports < p.errorReportingLimit { + p.errorReports++ + p.errors.syntaxError(l, msg) + } else { + tme := &tooManyErrors{errorReportingLimit: p.errorReportingLimit} + p.errors.syntaxError(l, tme.Error()) + panic(tme) + } +} + +func (p *antlrVisitor) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { + // Intentional +} + +func (p *antlrVisitor) ReportAttemptingFullContext(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, conflictingAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { + // Intentional +} + +func (p *antlrVisitor) ReportContextSensitivity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex, prediction int, configs *antlr.ATNConfigSet) { + // Intentional +} + +func (p *antlrVisitor) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { + if expr, found := p.expandMacro(exprID, function, nil, args...); found { + return expr + } + return p.helper.newGlobalCall(exprID, function, args...) +} + +func (p *antlrVisitor) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { + if expr, found := p.expandMacro(exprID, function, target, args...); found { + return expr + } + return p.helper.newReceiverCall(exprID, function, target, args...) +} + +func (p *antlrVisitor) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { + macro, found := p.macros[makeMacroKey(function, len(args), target != nil)] + if !found { + macro, found = p.macros[makeVarArgMacroKey(function, target != nil)] + if !found { + return nil, false + } + } + if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { + loc := p.helper.getLocation(exprID) + p.helper.deleteID(exprID) + return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true + } + eh := exprHelperPool.Get().(*exprHelper) + defer exprHelperPool.Put(eh) + eh.parserHelper = p.helper + eh.id = exprID + expr, err := macro.Expander()(eh, target, args) + if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { + loc := p.helper.getLocation(exprID) + p.helper.deleteID(exprID) + return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true + } + // An error indicates that the macro was matched, but the arguments were not well-formed. + if err != nil { + loc := err.Location + if loc == nil { + loc = p.helper.getLocation(exprID) + } + p.helper.deleteID(exprID) + return p.reportError(loc, "%s", err.Message), true + } + // A nil value from the macro indicates that the macro implementation decided that + // an expansion should not be performed. + if expr == nil { + return nil, false + } + if p.populateMacroCalls { + p.helper.addMacroCall(expr.ID(), function, target, args...) + } + p.helper.deleteID(exprID) + return expr, true +} + +func (p *antlrVisitor) checkAndIncrementRecursionDepth() { + p.recursionDepth++ + if p.recursionDepth > p.maxRecursionDepth { + panic(&recursionError{message: "max recursion depth exceeded"}) + } +} + +func (p *antlrVisitor) decrementRecursionDepth() { + p.recursionDepth-- +} + +// unnest traverses down the left-hand side of the parse graph until it encounters the first compound +// parse node or the first leaf in the parse graph. +func unnest(tree antlr.ParseTree) antlr.ParseTree { + for tree != nil { + switch t := tree.(type) { + case *gen.ExprContext: + // conditionalOr op='?' conditionalOr : expr + if t.GetOp() != nil { + return t + } + // conditionalOr + tree = t.GetE() + case *gen.ConditionalOrContext: + // conditionalAnd (ops=|| conditionalAnd)* + if t.GetOps() != nil && len(t.GetOps()) > 0 { + return t + } + // conditionalAnd + tree = t.GetE() + case *gen.ConditionalAndContext: + // relation (ops=&& relation)* + if t.GetOps() != nil && len(t.GetOps()) > 0 { + return t + } + // relation + tree = t.GetE() + case *gen.RelationContext: + // relation op relation + if t.GetOp() != nil { + return t + } + // calc + tree = t.Calc() + case *gen.CalcContext: + // calc op calc + if t.GetOp() != nil { + return t + } + // unary + tree = t.Unary() + case *gen.MemberExprContext: + // member expands to one of: primary, select, index, or create message + tree = t.Member() + case *gen.PrimaryExprContext: + // primary expands to one of identifier, nested, create list, create struct, literal + tree = t.Primary() + case *gen.NestedContext: + // contains a nested 'expr' + tree = t.GetE() + case *gen.ConstantLiteralContext: + // expands to a primitive literal + tree = t.Literal() + default: + return t + } + } + return tree +} + +var ( + reservedIdentifier = regexp.MustCompile("no viable alternative at input '.(true|false|null)'") + mismatchedReservedIdentifier = "mismatched input '$1' expecting IDENTIFIER" +) diff --git a/parser/helper.go b/parser/helper.go index 8603750a6..ba6864cb9 100644 --- a/parser/helper.go +++ b/parser/helper.go @@ -152,6 +152,18 @@ func (p *parserHelper) newExpr(ctx any) ast.Expr { return p.exprFactory.NewUnspecifiedExpr(p.newID(ctx)) } +// computeOffset converts a 0-based character offset from the local source content to +// the corresponding absolute character offset in the parent SourceInfo (e.g. for RelativeSource). +// If the location cannot be resolved or the source is nil, the original offset is returned. +func (p *parserHelper) computeOffset(offset int32) int32 { + if p.source != nil { + if loc, found := p.source.OffsetLocation(offset); found { + return p.sourceInfo.ComputeOffsetAbsolute(int32(loc.Line()), int32(loc.Column())) + } + } + return offset +} + func (p *parserHelper) id(ctx any) int64 { var offset ast.OffsetRange switch c := ctx.(type) { @@ -163,8 +175,8 @@ func (p *parserHelper) id(ctx any) int64 { offset.Start = p.sourceInfo.ComputeOffset(int32(c.GetLine()), int32(c.GetColumn())) offset.Stop = offset.Start + int32(len(c.GetText())) case token: - offset.Start = c.start - offset.Stop = c.end + offset.Start = p.computeOffset(c.start) + offset.Stop = p.computeOffset(c.end) case common.Location: offset.Start = p.sourceInfo.ComputeOffsetAbsolute(int32(c.Line()), int32(c.Column())) offset.Stop = offset.Start @@ -182,7 +194,7 @@ func (p *parserHelper) id(ctx any) int64 { func (p *parserHelper) idFromOffsets(start, stop int32) int64 { id := p.nextID - p.sourceInfo.SetOffsetRange(id, ast.OffsetRange{Start: start, Stop: stop}) + p.sourceInfo.SetOffsetRange(id, ast.OffsetRange{Start: p.computeOffset(start), Stop: p.computeOffset(stop)}) p.nextID++ return id } diff --git a/parser/options.go b/parser/options.go index 281021f12..23c3d8597 100644 --- a/parser/options.go +++ b/parser/options.go @@ -29,6 +29,7 @@ type options struct { enableVariadicOperatorASTs bool enableIdentEscapeSyntax bool enableHiddenAccumulatorName bool + enablePrattParser bool } // Option configures the behavior of the parser. @@ -174,3 +175,11 @@ func EnableVariadicOperatorASTs(varArgASTs bool) Option { return nil } } + +// EnablePrattParser enables the Pratt parser implementation instead of the ANTLR parser. +func EnablePrattParser(enablePrattParser bool) Option { + return func(opts *options) error { + opts.enablePrattParser = enablePrattParser + return nil + } +} diff --git a/parser/parser.go b/parser/parser.go index 2df20a704..8fe906a0b 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -17,20 +17,10 @@ package parser import ( - "errors" - "fmt" - "regexp" - "strconv" - "strings" - - antlr "github.com/antlr4-go/antlr/v4" + "math" "cel.dev/cel-go/common" "cel.dev/cel-go/common/ast" - "cel.dev/cel-go/common/operators" - "cel.dev/cel-go/common/runes" - "cel.dev/cel-go/common/types" - "cel.dev/cel-go/parser/gen" ) // Parser encapsulates the context necessary to perform parsing for different expressions. @@ -55,7 +45,7 @@ func NewParser(opts ...Option) (*Parser, error) { p.maxRecursionDepth = 250 } if p.maxRecursionDepth == -1 { - p.maxRecursionDepth = int((^uint(0)) >> 1) + p.maxRecursionDepth = math.MaxInt } if p.errorRecoveryTokenLookaheadLimit == 0 { p.errorRecoveryTokenLookaheadLimit = 256 @@ -64,19 +54,19 @@ func NewParser(opts ...Option) (*Parser, error) { p.errorRecoveryLimit = 30 } if p.errorRecoveryLimit == -1 { - p.errorRecoveryLimit = int((^uint(0)) >> 1) + p.errorRecoveryLimit = math.MaxInt } if p.expressionSizeCodePointLimit == 0 { p.expressionSizeCodePointLimit = 100_000 } if p.expressionSizeCodePointLimit == -1 { - p.expressionSizeCodePointLimit = int((^uint(0)) >> 1) + p.expressionSizeCodePointLimit = math.MaxInt } if p.maxExpressionNodeCount == 0 { p.maxExpressionNodeCount = 100_000 } if p.maxExpressionNodeCount == -1 { - p.maxExpressionNodeCount = int((^uint(0)) >> 1) + p.maxExpressionNodeCount = math.MaxInt } // Bool is false by default, so populateMacroCalls will be false by default return p, nil @@ -96,40 +86,10 @@ func mustNewParser(opts ...Option) *Parser { // Parse parses the expression represented by source and returns the result. func (p *Parser) Parse(source common.Source) (*ast.AST, *common.Errors) { - errs := common.NewErrors(source) - accu := AccumulatorName - if p.enableHiddenAccumulatorName { - accu = HiddenAccumulatorName - } - fac := ast.NewExprFactoryWithAccumulator(accu) - impl := parser{ - errors: &parseErrors{errs}, - exprFactory: fac, - helper: newParserHelper(source, fac), - macros: p.macros, - maxRecursionDepth: p.maxRecursionDepth, - maxExpressionNodeCount: p.maxExpressionNodeCount, - errorReportingLimit: p.errorReportingLimit, - errorRecoveryLimit: p.errorRecoveryLimit, - errorRecoveryLookaheadTokenLimit: p.errorRecoveryTokenLookaheadLimit, - populateMacroCalls: p.populateMacroCalls, - enableOptionalSyntax: p.enableOptionalSyntax, - enableVariadicOperatorASTs: p.enableVariadicOperatorASTs, - enableIdentEscapeSyntax: p.enableIdentEscapeSyntax, - } - buf, ok := source.(runes.Buffer) - if !ok { - buf = runes.NewBuffer(source.Content()) - } - var out ast.Expr - if buf.Len() > p.expressionSizeCodePointLimit { - out = impl.reportError(common.NoLocation, - "expression code point size exceeds limit: size: %d, limit %d", - buf.Len(), p.expressionSizeCodePointLimit) - } else { - out = impl.parse(buf, source.Description()) + if p.enablePrattParser { + return (&prattParser{options: p.options}).Parse(source) } - return ast.NewAST(out, impl.helper.getSourceInfo()), errs + return (&antlrParser{options: p.options}).Parse(source) } // reservedIds are not legal to use as variables. We exclude them post-parse, as they *are* valid @@ -158,27 +118,6 @@ var reservedIds = map[string]struct{}{ "while": {}, } -func unescapeIdent(in string) (string, error) { - if len(in) <= 2 { - return "", errors.New("invalid escaped identifier: underflow") - } - return in[1 : len(in)-1], nil -} - -// normalizeIdent returns the interpreted identifier. -func (p *parser) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error) { - switch ident := ctx.(type) { - case *gen.SimpleIdentifierContext: - return ident.GetId().GetText(), nil - case *gen.EscapedIdentifierContext: - if !p.enableIdentEscapeSyntax { - return "", errors.New("unsupported syntax: '`'") - } - return unescapeIdent(ident.GetId().GetText()) - } - return "", errors.New("unsupported ident kind") -} - // Parse converts a source input a parsed expression. // This function calls ParseWithMacros with AllMacros. // @@ -186,899 +125,3 @@ func (p *parser) normalizeIdent(ctx gen.IEscapeIdentContext) (string, error) { func Parse(source common.Source) (*ast.AST, *common.Errors) { return mustNewParser(Macros(AllMacros...)).Parse(source) } - -type recursionError struct { - message string -} - -// Error implements error. -func (re *recursionError) Error() string { - return re.message -} - -var _ error = &recursionError{} - -type recursionListener struct { - maxDepth int - ruleTypeDepth map[int]*int -} - -func (rl *recursionListener) VisitTerminal(node antlr.TerminalNode) {} - -func (rl *recursionListener) VisitErrorNode(node antlr.ErrorNode) {} - -func (rl *recursionListener) EnterEveryRule(ctx antlr.ParserRuleContext) { - if ctx == nil { - return - } - ruleIndex := ctx.GetRuleIndex() - depth, found := rl.ruleTypeDepth[ruleIndex] - if !found { - var counter = 1 - rl.ruleTypeDepth[ruleIndex] = &counter - depth = &counter - } else { - *depth++ - } - if *depth > rl.maxDepth { - panic(&recursionError{ - message: fmt.Sprintf("expression recursion limit exceeded: %d", rl.maxDepth), - }) - } -} - -func (rl *recursionListener) ExitEveryRule(ctx antlr.ParserRuleContext) { - if ctx == nil { - return - } - ruleIndex := ctx.GetRuleIndex() - if depth, found := rl.ruleTypeDepth[ruleIndex]; found && *depth > 0 { - *depth-- - } -} - -var _ antlr.ParseTreeListener = &recursionListener{} - -type tooManyErrors struct { - errorReportingLimit int -} - -func (t *tooManyErrors) Error() string { - return fmt.Sprintf("More than %d syntax errors", t.errorReportingLimit) -} - -var _ error = &tooManyErrors{} - -type recoveryLimitError struct { - message string -} - -// Error implements error. -func (rl *recoveryLimitError) Error() string { - return rl.message -} - -type lookaheadLimitError struct { - message string -} - -func (ll *lookaheadLimitError) Error() string { - return ll.message -} - -var _ error = &recoveryLimitError{} - -type recoveryLimitErrorStrategy struct { - *antlr.DefaultErrorStrategy - errorRecoveryLimit int - errorRecoveryTokenLookaheadLimit int - recoveryAttempts int -} - -type lookaheadConsumer struct { - antlr.Parser - errorRecoveryTokenLookaheadLimit int - lookaheadAttempts int -} - -func (lc *lookaheadConsumer) Consume() antlr.Token { - if lc.lookaheadAttempts >= lc.errorRecoveryTokenLookaheadLimit { - panic(&lookaheadLimitError{ - message: fmt.Sprintf("error recovery token lookahead limit exceeded: %d", lc.errorRecoveryTokenLookaheadLimit), - }) - } - lc.lookaheadAttempts++ - return lc.Parser.Consume() -} - -func (rl *recoveryLimitErrorStrategy) Recover(recognizer antlr.Parser, e antlr.RecognitionException) { - rl.checkAttempts(recognizer) - lc := &lookaheadConsumer{Parser: recognizer, errorRecoveryTokenLookaheadLimit: rl.errorRecoveryTokenLookaheadLimit} - rl.DefaultErrorStrategy.Recover(lc, e) -} - -func (rl *recoveryLimitErrorStrategy) RecoverInline(recognizer antlr.Parser) antlr.Token { - rl.checkAttempts(recognizer) - lc := &lookaheadConsumer{Parser: recognizer, errorRecoveryTokenLookaheadLimit: rl.errorRecoveryTokenLookaheadLimit} - return rl.DefaultErrorStrategy.RecoverInline(lc) -} - -func (rl *recoveryLimitErrorStrategy) checkAttempts(recognizer antlr.Parser) { - if rl.recoveryAttempts == rl.errorRecoveryLimit { - rl.recoveryAttempts++ - msg := fmt.Sprintf("error recovery attempt limit exceeded: %d", rl.errorRecoveryLimit) - recognizer.NotifyErrorListeners(msg, nil, nil) - panic(&recoveryLimitError{ - message: msg, - }) - } - rl.recoveryAttempts++ -} - -var _ antlr.ErrorStrategy = &recoveryLimitErrorStrategy{} - -type parser struct { - gen.BaseCELVisitor - errors *parseErrors - exprFactory ast.ExprFactory - helper *parserHelper - macros map[string]Macro - recursionDepth int - errorReports int - maxRecursionDepth int - maxExpressionNodeCount int - errorReportingLimit int - errorRecoveryLimit int - errorRecoveryLookaheadTokenLimit int - populateMacroCalls bool - enableOptionalSyntax bool - enableVariadicOperatorASTs bool - enableIdentEscapeSyntax bool -} - -var _ gen.CELVisitor = (*parser)(nil) - -func (p *parser) parse(expr runes.Buffer, desc string) ast.Expr { - lexer := gen.NewCELLexer(newCharStream(expr, desc)) - lexer.RemoveErrorListeners() - lexer.AddErrorListener(p) - - prsr := gen.NewCELParser(antlr.NewCommonTokenStream(lexer, 0)) - prsr.RemoveErrorListeners() - - prsrListener := &recursionListener{ - maxDepth: p.maxRecursionDepth, - ruleTypeDepth: map[int]*int{}, - } - - prsr.AddErrorListener(p) - prsr.AddParseListener(prsrListener) - - prsr.SetErrorHandler(&recoveryLimitErrorStrategy{ - DefaultErrorStrategy: antlr.NewDefaultErrorStrategy(), - errorRecoveryLimit: p.errorRecoveryLimit, - errorRecoveryTokenLookaheadLimit: p.errorRecoveryLookaheadTokenLimit, - }) - - defer func() { - if val := recover(); val != nil { - switch err := val.(type) { - case *lookaheadLimitError: - p.errors.internalError(err.Error()) - case *recursionError: - p.errors.internalError(err.Error()) - case *tooManyErrors: - // do nothing - case *recoveryLimitError: - // do nothing, listeners already notified and error reported. - default: - panic(val) - } - } - }() - - return p.Visit(prsr.Start_()).(ast.Expr) -} - -// Visitor implementations. -func (p *parser) Visit(tree antlr.ParseTree) any { - t := unnest(tree) - switch tree := t.(type) { - case *gen.StartContext: - return p.VisitStart(tree) - case *gen.ExprContext: - p.checkAndIncrementRecursionDepth() - out := p.VisitExpr(tree) - p.decrementRecursionDepth() - return out - case *gen.ConditionalAndContext: - return p.VisitConditionalAnd(tree) - case *gen.ConditionalOrContext: - return p.VisitConditionalOr(tree) - case *gen.RelationContext: - p.checkAndIncrementRecursionDepth() - out := p.VisitRelation(tree) - p.decrementRecursionDepth() - return out - case *gen.CalcContext: - p.checkAndIncrementRecursionDepth() - out := p.VisitCalc(tree) - p.decrementRecursionDepth() - return out - case *gen.LogicalNotContext: - return p.VisitLogicalNot(tree) - case *gen.IdentContext: - return p.VisitIdent(tree) - case *gen.GlobalCallContext: - return p.VisitGlobalCall(tree) - case *gen.SelectContext: - p.checkAndIncrementRecursionDepth() - out := p.VisitSelect(tree) - p.decrementRecursionDepth() - return out - case *gen.MemberCallContext: - p.checkAndIncrementRecursionDepth() - out := p.VisitMemberCall(tree) - p.decrementRecursionDepth() - return out - case *gen.MapInitializerListContext: - return p.VisitMapInitializerList(tree) - case *gen.NegateContext: - return p.VisitNegate(tree) - case *gen.IndexContext: - p.checkAndIncrementRecursionDepth() - out := p.VisitIndex(tree) - p.decrementRecursionDepth() - return out - case *gen.UnaryContext: - return p.VisitUnary(tree) - case *gen.CreateListContext: - return p.VisitCreateList(tree) - case *gen.CreateMessageContext: - return p.VisitCreateMessage(tree) - case *gen.CreateStructContext: - return p.VisitCreateStruct(tree) - case *gen.IntContext: - return p.VisitInt(tree) - case *gen.UintContext: - return p.VisitUint(tree) - case *gen.DoubleContext: - return p.VisitDouble(tree) - case *gen.StringContext: - return p.VisitString(tree) - case *gen.BytesContext: - return p.VisitBytes(tree) - case *gen.BoolFalseContext: - return p.VisitBoolFalse(tree) - case *gen.BoolTrueContext: - return p.VisitBoolTrue(tree) - case *gen.NullContext: - return p.VisitNull(tree) - } - - // Report at least one error if the parser reaches an unknown parse element. - // Typically, this happens if the parser has already encountered a syntax error elsewhere. - if p.errors.errorCount() == 0 { - txt := "<>" - if t != nil { - txt = fmt.Sprintf("<<%T>>", t) - } - return p.reportError(common.NoLocation, "unknown parse element encountered: %s", txt) - } - return p.helper.newExpr(common.NoLocation) - -} - -// Visit a parse tree produced by CELParser#start. -func (p *parser) VisitStart(ctx *gen.StartContext) any { - return p.Visit(ctx.Expr()) -} - -// Visit a parse tree produced by CELParser#expr. -func (p *parser) VisitExpr(ctx *gen.ExprContext) any { - result := p.Visit(ctx.GetE()).(ast.Expr) - if ctx.GetOp() == nil { - return result - } - opID := p.helper.id(ctx.GetOp()) - ifTrue := p.Visit(ctx.GetE1()).(ast.Expr) - ifFalse := p.Visit(ctx.GetE2()).(ast.Expr) - return p.globalCallOrMacro(opID, operators.Conditional, result, ifTrue, ifFalse) -} - -// Visit a parse tree produced by CELParser#conditionalOr. -func (p *parser) VisitConditionalOr(ctx *gen.ConditionalOrContext) any { - result := p.Visit(ctx.GetE()).(ast.Expr) - l := p.newLogicManager(operators.LogicalOr, result) - rest := ctx.GetE1() - for i, op := range ctx.GetOps() { - if i >= len(rest) { - return p.reportError(ctx, "unexpected character, wanted '||'") - } - next := p.Visit(rest[i]).(ast.Expr) - opID := p.helper.id(op) - l.addTerm(opID, next) - } - return l.toExpr() -} - -// Visit a parse tree produced by CELParser#conditionalAnd. -func (p *parser) VisitConditionalAnd(ctx *gen.ConditionalAndContext) any { - result := p.Visit(ctx.GetE()).(ast.Expr) - l := p.newLogicManager(operators.LogicalAnd, result) - rest := ctx.GetE1() - for i, op := range ctx.GetOps() { - if i >= len(rest) { - return p.reportError(ctx, "unexpected character, wanted '&&'") - } - next := p.Visit(rest[i]).(ast.Expr) - opID := p.helper.id(op) - l.addTerm(opID, next) - } - return l.toExpr() -} - -// Visit a parse tree produced by CELParser#relation. -func (p *parser) VisitRelation(ctx *gen.RelationContext) any { - opText := "" - if ctx.GetOp() != nil { - opText = ctx.GetOp().GetText() - } - if op, found := operators.Find(opText); found { - lhs := p.Visit(ctx.Relation(0)).(ast.Expr) - opID := p.helper.id(ctx.GetOp()) - rhs := p.Visit(ctx.Relation(1)).(ast.Expr) - return p.globalCallOrMacro(opID, op, lhs, rhs) - } - return p.reportError(ctx, "operator not found") -} - -// Visit a parse tree produced by CELParser#calc. -func (p *parser) VisitCalc(ctx *gen.CalcContext) any { - opText := "" - if ctx.GetOp() != nil { - opText = ctx.GetOp().GetText() - } - if op, found := operators.Find(opText); found { - lhs := p.Visit(ctx.Calc(0)).(ast.Expr) - opID := p.helper.id(ctx.GetOp()) - rhs := p.Visit(ctx.Calc(1)).(ast.Expr) - return p.globalCallOrMacro(opID, op, lhs, rhs) - } - return p.reportError(ctx, "operator not found") -} - -func (p *parser) VisitUnary(ctx *gen.UnaryContext) any { - return p.helper.newLiteralString(ctx, "<>") -} - -// Visit a parse tree produced by CELParser#LogicalNot. -func (p *parser) VisitLogicalNot(ctx *gen.LogicalNotContext) any { - if len(ctx.GetOps())%2 == 0 { - return p.Visit(ctx.Member()) - } - opID := p.helper.id(ctx.GetOps()[0]) - target := p.Visit(ctx.Member()).(ast.Expr) - return p.globalCallOrMacro(opID, operators.LogicalNot, target) -} - -func (p *parser) VisitNegate(ctx *gen.NegateContext) any { - if len(ctx.GetOps())%2 == 0 { - return p.Visit(ctx.Member()) - } - opID := p.helper.id(ctx.GetOps()[0]) - target := p.Visit(ctx.Member()).(ast.Expr) - return p.globalCallOrMacro(opID, operators.Negate, target) -} - -// VisitSelect visits a parse tree produced by CELParser#Select. -func (p *parser) VisitSelect(ctx *gen.SelectContext) any { - operand := p.Visit(ctx.Member()).(ast.Expr) - // Handle the error case where no valid identifier is specified. - if ctx.GetId() == nil || ctx.GetOp() == nil { - return p.helper.newExpr(ctx) - } - id, err := p.normalizeIdent(ctx.GetId()) - if err != nil { - p.reportError(ctx.GetId(), "%v", err) - } - if ctx.GetOpt() != nil { - if !p.enableOptionalSyntax { - return p.reportError(ctx.GetOp(), "unsupported syntax '.?'") - } - return p.helper.newGlobalCall( - ctx.GetOp(), - operators.OptSelect, - operand, - p.helper.newLiteralString(ctx.GetId(), id)) - } - return p.helper.newSelect(ctx.GetOp(), operand, id) -} - -// VisitMemberCall visits a parse tree produced by CELParser#MemberCall. -func (p *parser) VisitMemberCall(ctx *gen.MemberCallContext) any { - operand := p.Visit(ctx.Member()).(ast.Expr) - // Handle the error case where no valid identifier is specified. - if ctx.GetId() == nil { - return p.helper.newExpr(ctx) - } - id := ctx.GetId().GetText() - opID := p.helper.id(ctx.GetOpen()) - return p.receiverCallOrMacro(opID, id, operand, p.visitExprList(ctx.GetArgs())...) -} - -// Visit a parse tree produced by CELParser#Index. -func (p *parser) VisitIndex(ctx *gen.IndexContext) any { - target := p.Visit(ctx.Member()).(ast.Expr) - // Handle the error case where no valid identifier is specified. - if ctx.GetOp() == nil { - return p.helper.newExpr(ctx) - } - opID := p.helper.id(ctx.GetOp()) - index := p.Visit(ctx.GetIndex()).(ast.Expr) - operator := operators.Index - if ctx.GetOpt() != nil { - if !p.enableOptionalSyntax { - return p.reportError(ctx.GetOp(), "unsupported syntax '[?'") - } - operator = operators.OptIndex - } - return p.globalCallOrMacro(opID, operator, target, index) -} - -// Visit a parse tree produced by CELParser#CreateMessage. -func (p *parser) VisitCreateMessage(ctx *gen.CreateMessageContext) any { - messageName := "" - for _, id := range ctx.GetIds() { - if len(messageName) != 0 { - messageName += "." - } - messageName += id.GetText() - } - if ctx.GetLeadingDot() != nil { - messageName = "." + messageName - } - objID := p.helper.id(ctx.GetOp()) - entries := p.VisitIFieldInitializerList(ctx.GetEntries()).([]ast.EntryExpr) - return p.helper.newObject(objID, messageName, entries...) -} - -// Visit a parse tree of field initializers. -func (p *parser) VisitIFieldInitializerList(ctx gen.IFieldInitializerListContext) any { - if ctx == nil || ctx.GetFields() == nil { - // This is the result of a syntax error handled elswhere, return empty. - return []ast.EntryExpr{} - } - - result := make([]ast.EntryExpr, len(ctx.GetFields())) - cols := ctx.GetCols() - vals := ctx.GetValues() - for i, f := range ctx.GetFields() { - if i >= len(cols) || i >= len(vals) { - // This is the result of a syntax error detected elsewhere. - return []ast.EntryExpr{} - } - initID := p.helper.id(cols[i]) - optField := f.(*gen.OptFieldContext) - optional := optField.GetOpt() != nil - if !p.enableOptionalSyntax && optional { - p.reportError(optField, "unsupported syntax '?'") - continue - } - - // The field may be empty due to a prior error. - fieldName, err := p.normalizeIdent(optField.EscapeIdent()) - if err != nil { - p.reportError(ctx, "%v", err) - continue - } - - value := p.Visit(vals[i]).(ast.Expr) - field := p.helper.newObjectField(initID, fieldName, value, optional) - result[i] = field - } - return result -} - -// Visit a parse tree produced by CELParser#Ident. -func (p *parser) VisitIdent(ctx *gen.IdentContext) any { - identName := "" - if ctx.GetLeadingDot() != nil { - identName = "." - } - // Handle the error case where no valid identifier is specified. - if ctx.GetId() == nil { - return p.helper.newExpr(ctx) - } - // Handle reserved identifiers. - id := ctx.GetId().GetText() - if _, ok := reservedIds[id]; ok { - return p.reportError(ctx, "reserved identifier: %s", id) - } - identName += id - return p.helper.newIdent(ctx.GetId(), identName) -} - -// Visit a parse tree produced by CELParser#GlobalCallContext. -func (p *parser) VisitGlobalCall(ctx *gen.GlobalCallContext) any { - identName := "" - if ctx.GetLeadingDot() != nil { - identName = "." - } - // Handle the error case where no valid identifier is specified. - if ctx.GetId() == nil { - return p.helper.newExpr(ctx) - } - // Handle reserved identifiers. - id := ctx.GetId().GetText() - if _, ok := reservedIds[id]; ok { - return p.reportError(ctx, "reserved identifier: %s", id) - } - identName += id - opID := p.helper.id(ctx.GetOp()) - return p.globalCallOrMacro(opID, identName, p.visitExprList(ctx.GetArgs())...) - -} - -// Visit a parse tree produced by CELParser#CreateList. -func (p *parser) VisitCreateList(ctx *gen.CreateListContext) any { - listID := p.helper.id(ctx.GetOp()) - elems, optionals := p.visitListInit(ctx.GetElems()) - return p.helper.newList(listID, elems, optionals...) -} - -// Visit a parse tree produced by CELParser#CreateStruct. -func (p *parser) VisitCreateStruct(ctx *gen.CreateStructContext) any { - structID := p.helper.id(ctx.GetOp()) - entries := []ast.EntryExpr{} - if ctx.GetEntries() != nil { - entries = p.Visit(ctx.GetEntries()).([]ast.EntryExpr) - } - return p.helper.newMap(structID, entries...) -} - -// Visit a parse tree produced by CELParser#mapInitializerList. -func (p *parser) VisitMapInitializerList(ctx *gen.MapInitializerListContext) any { - if ctx == nil || ctx.GetKeys() == nil { - // This is the result of a syntax error handled elswhere, return empty. - return []ast.EntryExpr{} - } - - result := make([]ast.EntryExpr, len(ctx.GetCols())) - keys := ctx.GetKeys() - vals := ctx.GetValues() - for i, col := range ctx.GetCols() { - colID := p.helper.id(col) - if i >= len(keys) || i >= len(vals) { - // This is the result of a syntax error detected elsewhere. - return []ast.EntryExpr{} - } - optKey := keys[i] - optional := optKey.GetOpt() != nil - if !p.enableOptionalSyntax && optional { - p.reportError(optKey, "unsupported syntax '?'") - continue - } - key := p.Visit(optKey.GetE()).(ast.Expr) - value := p.Visit(vals[i]).(ast.Expr) - entry := p.helper.newMapEntry(colID, key, value, optional) - result[i] = entry - } - return result -} - -// Visit a parse tree produced by CELParser#Int. -func (p *parser) VisitInt(ctx *gen.IntContext) any { - text := ctx.GetTok().GetText() - base := 10 - if strings.HasPrefix(text, "0x") { - base = 16 - text = text[2:] - } - if ctx.GetSign() != nil { - text = ctx.GetSign().GetText() + text - } - i, err := strconv.ParseInt(text, base, 64) - if err != nil { - return p.reportError(ctx, "invalid int literal") - } - return p.helper.newLiteralInt(ctx, i) -} - -// Visit a parse tree produced by CELParser#Uint. -func (p *parser) VisitUint(ctx *gen.UintContext) any { - text := ctx.GetTok().GetText() - // trim the 'u' designator included in the uint literal. - text = text[:len(text)-1] - base := 10 - if strings.HasPrefix(text, "0x") { - base = 16 - text = text[2:] - } - i, err := strconv.ParseUint(text, base, 64) - if err != nil { - return p.reportError(ctx, "invalid uint literal") - } - return p.helper.newLiteralUint(ctx, i) -} - -// Visit a parse tree produced by CELParser#Double. -func (p *parser) VisitDouble(ctx *gen.DoubleContext) any { - txt := ctx.GetTok().GetText() - if ctx.GetSign() != nil { - txt = ctx.GetSign().GetText() + txt - } - f, err := strconv.ParseFloat(txt, 64) - if err != nil { - return p.reportError(ctx, "invalid double literal") - } - return p.helper.newLiteralDouble(ctx, f) - -} - -// Visit a parse tree produced by CELParser#String. -func (p *parser) VisitString(ctx *gen.StringContext) any { - s := p.unquote(ctx, ctx.GetTok().GetText(), false) - return p.helper.newLiteralString(ctx, s) -} - -// Visit a parse tree produced by CELParser#Bytes. -func (p *parser) VisitBytes(ctx *gen.BytesContext) any { - b := []byte(p.unquote(ctx, ctx.GetTok().GetText()[1:], true)) - return p.helper.newLiteralBytes(ctx, b) -} - -// Visit a parse tree produced by CELParser#BoolTrue. -func (p *parser) VisitBoolTrue(ctx *gen.BoolTrueContext) any { - return p.helper.newLiteralBool(ctx, true) -} - -// Visit a parse tree produced by CELParser#BoolFalse. -func (p *parser) VisitBoolFalse(ctx *gen.BoolFalseContext) any { - return p.helper.newLiteralBool(ctx, false) -} - -// Visit a parse tree produced by CELParser#Null. -func (p *parser) VisitNull(ctx *gen.NullContext) any { - return p.helper.exprFactory.NewLiteral(p.helper.newID(ctx), types.NullValue) -} - -func (p *parser) visitExprList(ctx gen.IExprListContext) []ast.Expr { - if ctx == nil { - return []ast.Expr{} - } - return p.visitSlice(ctx.GetE()) -} - -func (p *parser) visitListInit(ctx gen.IListInitContext) ([]ast.Expr, []int32) { - if ctx == nil { - return []ast.Expr{}, []int32{} - } - elements := ctx.GetElems() - result := make([]ast.Expr, len(elements)) - optionals := []int32{} - for i, e := range elements { - ex := p.Visit(e.GetE()).(ast.Expr) - if ex == nil { - return []ast.Expr{}, []int32{} - } - result[i] = ex - if e.GetOpt() != nil { - if !p.enableOptionalSyntax { - p.reportError(e.GetOpt(), "unsupported syntax '?'") - continue - } - optionals = append(optionals, int32(i)) - } - } - return result, optionals -} - -func (p *parser) visitSlice(expressions []gen.IExprContext) []ast.Expr { - if expressions == nil { - return []ast.Expr{} - } - result := make([]ast.Expr, len(expressions)) - for i, e := range expressions { - ex := p.Visit(e).(ast.Expr) - result[i] = ex - } - return result -} - -func (p *parser) unquote(ctx any, value string, isBytes bool) string { - text, err := unescape(value, isBytes) - if err != nil { - p.reportError(ctx, "%s", err.Error()) - return value - } - return text -} - -func (p *parser) newLogicManager(function string, term ast.Expr) *logicManager { - if p.enableVariadicOperatorASTs { - return newVariadicLogicManager(p.exprFactory, function, term) - } - return newBalancingLogicManager(p.exprFactory, function, term) -} - -func (p *parser) reportError(ctx any, format string, args ...any) ast.Expr { - var location common.Location - err := p.helper.newExpr(ctx) - switch c := ctx.(type) { - case common.Location: - location = c - case antlr.Token, antlr.ParserRuleContext: - location = p.helper.getLocation(err.ID()) - } - // Provide arguments to the report error. - p.errors.reportErrorAtID(err.ID(), location, format, args...) - return err -} - -// ANTLR Parse listener implementations -func (p *parser) SyntaxError(recognizer antlr.Recognizer, offendingSymbol any, line, column int, msg string, e antlr.RecognitionException) { - offset := p.helper.sourceInfo.ComputeOffset(int32(line), int32(column)) - l := p.helper.getLocationByOffset(offset) - // Hack to keep existing error messages consistent with previous versions of CEL when a reserved word - // is used as an identifier. This behavior needs to be overhauled to provide consistent, normalized error - // messages out of ANTLR to prevent future breaking changes related to error message content. - if strings.Contains(msg, "no viable alternative") { - msg = reservedIdentifier.ReplaceAllString(msg, mismatchedReservedIdentifier) - } - // Ensure that no more than 100 syntax errors are reported as this will halt attempts to recover from a - // seriously broken expression. - if p.errorReports < p.errorReportingLimit { - p.errorReports++ - p.errors.syntaxError(l, msg) - } else { - tme := &tooManyErrors{errorReportingLimit: p.errorReportingLimit} - p.errors.syntaxError(l, tme.Error()) - panic(tme) - } -} - -func (p *parser) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { - // Intentional -} - -func (p *parser) ReportAttemptingFullContext(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, conflictingAlts *antlr.BitSet, configs *antlr.ATNConfigSet) { - // Intentional -} - -func (p *parser) ReportContextSensitivity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex, prediction int, configs *antlr.ATNConfigSet) { - // Intentional -} - -func (p *parser) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { - if expr, found := p.expandMacro(exprID, function, nil, args...); found { - return expr - } - return p.helper.newGlobalCall(exprID, function, args...) -} - -func (p *parser) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { - if expr, found := p.expandMacro(exprID, function, target, args...); found { - return expr - } - return p.helper.newReceiverCall(exprID, function, target, args...) -} - -func (p *parser) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { - macro, found := p.macros[makeMacroKey(function, len(args), target != nil)] - if !found { - macro, found = p.macros[makeVarArgMacroKey(function, target != nil)] - if !found { - return nil, false - } - } - if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { - loc := p.helper.getLocation(exprID) - p.helper.deleteID(exprID) - return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true - } - eh := exprHelperPool.Get().(*exprHelper) - defer exprHelperPool.Put(eh) - eh.parserHelper = p.helper - eh.id = exprID - expr, err := macro.Expander()(eh, target, args) - if int(p.helper.expressionCount()) > p.maxExpressionNodeCount { - loc := p.helper.getLocation(exprID) - p.helper.deleteID(exprID) - return p.reportError(loc, "expression count exceeds limit of %d while expanding macro '%s'", p.maxExpressionNodeCount, function), true - } - // An error indicates that the macro was matched, but the arguments were not well-formed. - if err != nil { - loc := err.Location - if loc == nil { - loc = p.helper.getLocation(exprID) - } - p.helper.deleteID(exprID) - return p.reportError(loc, "%s", err.Message), true - } - // A nil value from the macro indicates that the macro implementation decided that - // an expansion should not be performed. - if expr == nil { - return nil, false - } - if p.populateMacroCalls { - p.helper.addMacroCall(expr.ID(), function, target, args...) - } - p.helper.deleteID(exprID) - return expr, true -} - -func (p *parser) checkAndIncrementRecursionDepth() { - p.recursionDepth++ - if p.recursionDepth > p.maxRecursionDepth { - panic(&recursionError{message: "max recursion depth exceeded"}) - } -} - -func (p *parser) decrementRecursionDepth() { - p.recursionDepth-- -} - -// unnest traverses down the left-hand side of the parse graph until it encounters the first compound -// parse node or the first leaf in the parse graph. -func unnest(tree antlr.ParseTree) antlr.ParseTree { - for tree != nil { - switch t := tree.(type) { - case *gen.ExprContext: - // conditionalOr op='?' conditionalOr : expr - if t.GetOp() != nil { - return t - } - // conditionalOr - tree = t.GetE() - case *gen.ConditionalOrContext: - // conditionalAnd (ops=|| conditionalAnd)* - if t.GetOps() != nil && len(t.GetOps()) > 0 { - return t - } - // conditionalAnd - tree = t.GetE() - case *gen.ConditionalAndContext: - // relation (ops=&& relation)* - if t.GetOps() != nil && len(t.GetOps()) > 0 { - return t - } - // relation - tree = t.GetE() - case *gen.RelationContext: - // relation op relation - if t.GetOp() != nil { - return t - } - // calc - tree = t.Calc() - case *gen.CalcContext: - // calc op calc - if t.GetOp() != nil { - return t - } - // unary - tree = t.Unary() - case *gen.MemberExprContext: - // member expands to one of: primary, select, index, or create message - tree = t.Member() - case *gen.PrimaryExprContext: - // primary expands to one of identifier, nested, create list, create struct, literal - tree = t.Primary() - case *gen.NestedContext: - // contains a nested 'expr' - tree = t.GetE() - case *gen.ConstantLiteralContext: - // expands to a primitive literal - tree = t.Literal() - default: - return t - } - } - return tree -} - -var ( - reservedIdentifier = regexp.MustCompile("no viable alternative at input '.(true|false|null)'") - mismatchedReservedIdentifier = "mismatched input '$1' expecting IDENTIFIER" -) diff --git a/parser/parser_test.go b/parser/parser_test.go index 7730a94ad..00dda7c5a 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -325,6 +325,10 @@ var testCases = []testInfo{ a^#3:*expr.Expr_IdentExpr#:b^#4:*expr.Expr_IdentExpr#^#2:*expr.Expr_CreateStruct_Entry#, c^#6:*expr.Expr_IdentExpr#:d^#7:*expr.Expr_IdentExpr#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + a^#2:*expr.Expr_IdentExpr#:b^#4:*expr.Expr_IdentExpr#^#3:*expr.Expr_CreateStruct_Entry#, + c^#5:*expr.Expr_IdentExpr#:d^#7:*expr.Expr_IdentExpr#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `[]`, @@ -419,6 +423,15 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: extraneous input 'b' expecting | *@a | b | ......^`, + PrattE: `ERROR: :1:1: unexpected token + | *@a | b + | ^ + ERROR: :1:2: unexpected character + | *@a | b + | .^ + ERROR: :1:5: unexpected single '|', expected '||' + | *@a | b + | ....^`, }, { I: `a | b`, @@ -428,6 +441,9 @@ var testCases = []testInfo{ ERROR: :1:5: Syntax error: extraneous input 'b' expecting | a | b | ....^`, + PrattE: `ERROR: :1:3: unexpected single '|', expected '||' + | a | b + | ..^`, }, // Macro tests @@ -734,6 +750,10 @@ var testCases = []testInfo{ foo^#3:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry#, bar^#6:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, + bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `{foo: 5, bar: "xyz", }`, @@ -741,6 +761,10 @@ var testCases = []testInfo{ foo^#3:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry#, bar^#6:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, + bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `a > 5 && a < 10`, @@ -773,6 +797,9 @@ var testCases = []testInfo{ E: `ERROR: :1:2: Syntax error: mismatched input '' expecting {'[', '{', '}', '(', '.', ',', '-', '!', '?', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | { | .^`, + PrattE: `ERROR: :1:2: expected '}' + | { + | .^`, }, // Tests from Java parser @@ -798,6 +825,10 @@ var testCases = []testInfo{ 1^#3:*expr.Constant_Int64Value#:2u^#4:*expr.Constant_Uint64Value#^#2:*expr.Expr_CreateStruct_Entry#, 2^#6:*expr.Constant_Int64Value#:3u^#7:*expr.Constant_Uint64Value#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + 1^#2:*expr.Constant_Int64Value#:2u^#4:*expr.Constant_Uint64Value#^#3:*expr.Expr_CreateStruct_Entry#, + 2^#5:*expr.Constant_Int64Value#:3u^#7:*expr.Constant_Uint64Value#^#6:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `TestAllTypes{single_int32: 1, single_int64: 2}`, @@ -835,6 +866,9 @@ var testCases = []testInfo{ ERROR: :1:6: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | 1 + $ | .....^`, + PrattE: `ERROR: :1:5: unexpected character + | 1 + $ + | ....^`, }, { I: `1 + 2 @@ -934,6 +968,9 @@ var testCases = []testInfo{ ERROR: :1:6: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | 1 + + | .....^`, + PrattE: `ERROR: :1:5: unexpected token + | 1 + + + | ....^`, }, { I: `"abc" + "def"`, @@ -948,6 +985,9 @@ var testCases = []testInfo{ E: `ERROR: :1:10: Syntax error: no viable alternative at input '."a"' | {"a": 1}."a" | .........^`, + PrattE: `ERROR: :1:10: expected identifier after '.' + | {"a": 1}."a" + | .........^`, }, { @@ -986,6 +1026,9 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | "\xFh" | ......^`, + PrattE: `ERROR: :1:1: unable to unescape string + | "\xFh" + | ^`, }, { @@ -999,6 +1042,9 @@ var testCases = []testInfo{ ERROR: :1:43: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | "\a\b\f\n\r\t\v\'\"\\\? Illegal escape \>" | ..........................................^`, + PrattE: `ERROR: :1:1: unable to unescape string + | "\a\b\f\n\r\t\v\'\"\\\? Illegal escape \>" + | ^`, }, { @@ -1024,6 +1070,12 @@ var testCases = []testInfo{ ERROR: :2:11: Syntax error: no viable alternative at input '.' | && in.😁 | .........．^`, + PrattE: `ERROR: :2:7: unexpected token + | && in.😁 + | ......^ + ERROR: :2:10: unexpected character + | && in.😁 + | .........＾`, }, { I: "as", @@ -1087,6 +1139,9 @@ var testCases = []testInfo{ ERROR: :1:3: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | in | ..^`, + PrattE: `ERROR: :1:1: unexpected token + | in + | ^`, }, { I: "let", @@ -1150,6 +1205,15 @@ var testCases = []testInfo{ ERROR: :1:26: reserved identifier: var | [1, 2, 3].map(var, var * var) | .........................^`, + PrattE: `ERROR: :1:15: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | ..............^ + ERROR: :1:20: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | ...................^ + ERROR: :1:26: reserved identifier: var + | [1, 2, 3].map(var, var * var) + | .........................^`, }, { I: "func{{a}}", @@ -1162,6 +1226,12 @@ var testCases = []testInfo{ ERROR: :1:9: Syntax error: extraneous input '}' expecting | func{{a}} | ........^`, + PrattE: `ERROR: :1:6: expected struct field name + | func{{a}} + | .....^ + ERROR: :1:9: Syntax error: mismatched input '}' expecting + | func{{a}} + | ........^`, }, { I: "msg{:a}", @@ -1171,12 +1241,18 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: mismatched input '}' expecting ':' | msg{:a} | ......^`, + PrattE: `ERROR: :1:5: expected struct field name + | msg{:a} + | ....^`, }, { I: "{a}", E: `ERROR: :1:3: Syntax error: mismatched input '}' expecting ':' | {a} | ..^`, + PrattE: `ERROR: :1:3: expected ':' in map entry + | {a} + | ..^`, }, { I: "{:a}", @@ -1186,12 +1262,21 @@ var testCases = []testInfo{ ERROR: :1:4: Syntax error: mismatched input '}' expecting ':' | {:a} | ...^`, + PrattE: `ERROR: :1:2: unexpected token + | {:a} + | .^ + ERROR: :1:3: expected ':' in map entry + | {:a} + | ..^`, }, { I: "ind[a{b}]", E: `ERROR: :1:8: Syntax error: mismatched input '}' expecting ':' | ind[a{b}] | .......^`, + PrattE: `ERROR: :1:8: expected ':' in struct field + | ind[a{b}] + | .......^`, }, { I: `--`, @@ -1201,6 +1286,9 @@ var testCases = []testInfo{ ERROR: :1:3: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | -- | ..^`, + PrattE: `ERROR: :1:3: Syntax error: mismatched input '' expecting expression + | -- + | ..^`, }, { I: `?`, @@ -1210,6 +1298,9 @@ var testCases = []testInfo{ ERROR: :1:2: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | ? | .^`, + PrattE: `ERROR: :1:1: unexpected token + | ? + | ^`, }, { I: `a ? b ((?))`, @@ -1222,12 +1313,22 @@ var testCases = []testInfo{ ERROR: :1:12: Syntax error: error recovery attempt limit exceeded: 4 | a ? b ((?)) | ...........^`, + PrattE: `ERROR: :1:9: unexpected token + | a ? b ((?)) + | ........^ + ERROR: :1:12: expected ':' in conditional expression + | a ? b ((?)) + | ...........^`, }, { I: `[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['too many']]]]]]]]]]]]]]]]]]]]]]]]]]]] ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]`, E: "ERROR: :-1:0: expression recursion limit exceeded: 32", + PrattE: `ERROR: :-1:0: expression recursion limit exceeded: 32 +ERROR: :1:34: expected ']' + | [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + | .................................^`, }, { I: `-[-1--1--1--1---1--1--1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 @@ -1277,6 +1378,21 @@ var testCases = []testInfo{ ERROR: :14:23: Syntax error: extraneous input '/' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | --1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 | ......................^`, + PrattE: `ERROR: :3:33: unexpected token + | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 + | ................................^ + ERROR: :3:34: expected ']' + | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 + | .................................^ + ERROR: :11:17: unexpected character + | --1--1---1--1-À1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 + | ................＾ + ERROR: :34:49: expected ']' + | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 + | ................................................^ + ERROR: :34:49: expected ']' + | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 + | ................................................^`, }, { I: `ó ¢ ó 0  @@ -1309,6 +1425,21 @@ var testCases = []testInfo{ ERROR: :3:11: Syntax error: token recognition error at: '\' | 0"""\""\"""\""\"""\""\"""\""\"""\"\"""\""\"""\""\"""\""\"""\"!\"""\""\"""\""\" | ..........^`, + PrattE: `ERROR: :1:1: unexpected character + | ó ¢ + | ＾ + ERROR: :1:2: unexpected character + | ó ¢ + | ．＾ + ERROR: :1:3: unexpected character + | ó ¢ + | ．．＾ + ERROR: :2:3: unexpected character + | ó 0  + | ..＾ + ERROR: :2:4: unexpected character + | ó 0  + | ..．＾`, }, // Macro Calls Tests { @@ -1595,12 +1726,14 @@ var testCases = []testInfo{ !=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y !=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y !=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y!=-y!=-y-y!=-y`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 nested list creation statements - I: `[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]`, - E: `ERROR: :-1:0: expression recursion limit exceeded: 32`, + I: `[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[['not fine']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]`, + E: `ERROR: :-1:0: expression recursion limit exceeded: 32`, + PrattE: "-", }, { // More than 32 arithmetic operations. @@ -1608,18 +1741,21 @@ var testCases = []testInfo{ + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 field selections - I: `a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + I: `a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z.A.B.C.D.E.F.G.H`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 index operations I: `a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] [21][22][23][24][25][26][27][28][29][30][31][32][33]`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 relation operators @@ -1627,7 +1763,8 @@ var testCases = []testInfo{ < 12 < 13 < 14 < 15 < 16 < 17 < 18 < 19 < 20 < 21 < 22 < 23 < 24 < 25 < 26 < 27 < 28 < 29 < 30 < 31 < 32 < 33`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { // More than 32 index / relation operators. Note, the recursion count is the @@ -1647,13 +1784,17 @@ var testCases = []testInfo{ a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20] != a[1][2][3][4][5][6][7][8][9][10][11][12][13][14][15][16][17][18][19][20]`, - E: `ERROR: :-1:0: max recursion depth exceeded`, + E: `ERROR: :-1:0: max recursion depth exceeded`, + PrattE: "-", }, { I: `self.true == 1`, E: `ERROR: :1:6: Syntax error: mismatched input 'true' expecting IDENTIFIER | self.true == 1 | .....^`, + PrattE: `ERROR: :1:6: expected identifier after '.' + | self.true == 1 + | .....^`, }, { I: `a.?b && a[?b]`, @@ -1662,7 +1803,13 @@ var testCases = []testInfo{ | .^ ERROR: :1:10: unsupported syntax '[?' | a.?b && a[?b] - | .........^`, + | .........^`, + PrattE: `ERROR: :1:2: unsupported syntax '.?' + | a.?b && a[?b] + | .^ + ERROR: :1:10: unsupported syntax '?' + | a.?b && a[?b] + | .........^`, }, { I: `a.?b[?0] && a[?c]`, @@ -1680,6 +1827,19 @@ var testCases = []testInfo{ c^#8:*expr.Expr_IdentExpr# )^#7:*expr.Expr_CallExpr# )^#9:*expr.Expr_CallExpr#`, + PrattP: `_&&_( + _[?_]( + _?._( + a^#1:*expr.Expr_IdentExpr#, + "b"^#3:*expr.Constant_StringValue# + )^#2:*expr.Expr_CallExpr#, + 0^#5:*expr.Constant_Int64Value# + )^#4:*expr.Expr_CallExpr#, + _[?_]( + a^#6:*expr.Expr_IdentExpr#, + c^#8:*expr.Expr_IdentExpr# + )^#7:*expr.Expr_CallExpr# + )^#9:*expr.Expr_CallExpr#`, }, { I: `{?'key': value}`, @@ -1687,6 +1847,9 @@ var testCases = []testInfo{ P: `{ ?"key"^#3:*expr.Constant_StringValue#:value^#4:*expr.Expr_IdentExpr#^#2:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, + PrattP: `{ + ?"key"^#2:*expr.Constant_StringValue#:value^#4:*expr.Expr_IdentExpr#^#3:*expr.Expr_CreateStruct_Entry# + }^#1:*expr.Expr_StructExpr#`, }, { I: `[?a, ?b]`, @@ -1770,6 +1933,9 @@ var testCases = []testInfo{ E: "ERROR: :1:1: Syntax error: mismatched input '`b-c`' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + "| `b-c`\n" + "| ^", + PrattE: "ERROR: :1:1: unexpected quoted identifier\n" + + "| `b-c`\n" + + "| ^", }, { I: "`b-c`()", @@ -1780,6 +1946,9 @@ var testCases = []testInfo{ "ERROR: :1:7: Syntax error: mismatched input ')' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" + "| `b-c`()\n" + "| ......^", + PrattE: "ERROR: :1:1: unexpected quoted identifier\n" + + "| `b-c`()\n" + + "| ^", }, { I: "a.`$b`", @@ -1790,6 +1959,9 @@ var testCases = []testInfo{ "ERROR: :1:6: Syntax error: token recognition error at: '`'\n" + "| a.`$b`\n" + "| .....^", + PrattE: "ERROR: :1:3: unexpected quoted identifier\n" + + "| a.`$b`\n" + + "| ..^", }, { I: "a.`b.c`()", @@ -1797,6 +1969,9 @@ var testCases = []testInfo{ E: "ERROR: :1:8: Syntax error: mismatched input '(' expecting \n" + "| a.`b.c`()\n" + "| .......^\n", + PrattE: "ERROR: :1:3: unexpected quoted identifier\n" + + "| a.`b.c`()\n" + + "| ..^", }, { I: "a.`b-c`", @@ -1858,6 +2033,15 @@ var testCases = []testInfo{ ERROR: :1:4: Syntax error: mismatched input '.' expecting {IDENTIFIER, ESC_IDENTIFIER} | x{?. | ...^`, + PrattE: `ERROR: :1:3: unsupported syntax '?' + | x{?. + | ..^ + ERROR: :1:4: expected struct field name + | x{?. + | ...^ + ERROR: :1:5: expected '}' + | x{?. + | ....^`, }, { I: `x{.`, @@ -1865,6 +2049,12 @@ var testCases = []testInfo{ ERROR: :1:3: Syntax error: mismatched input '.' expecting {'}', ',', '?', IDENTIFIER, ESC_IDENTIFIER} | x{. | ..^`, + PrattE: `ERROR: :1:3: expected struct field name + | x{. + | ..^ + ERROR: :1:4: expected '}' + | x{. + | ...^`, }, { I: `'3# < 10" '& tru ^^`, @@ -1880,6 +2070,12 @@ var testCases = []testInfo{ | '3# < 10" '& tru ^^ | ..................^ `, + PrattE: `ERROR: :1:12: unexpected single '&', expected '&&' + | '3# < 10" '& tru ^^ + | ...........^ + ERROR: :1:18: unexpected character + | '3# < 10" '& tru ^^ + | .................^`, }, { I: `'\udead' == '\ufffd'`, @@ -2095,15 +2291,27 @@ type testInfo struct { // P contains the type/id adorned debug output of the expression tree. P string + // PrattP contains the expected output for the Pratt parser when it differs from P. + PrattP string + // E contains the expected error output for a failed parse, or "" if the parse is expected to be successful. E string + // PrattE contains the expected error output for the Pratt parser when it differs from E. + PrattE string + // L contains the expected source adorned debug output of the expression tree. L string + // PrattL contains the expected source adorned debug output for the Pratt parser when it differs from L. + PrattL string + // M contains the expected adorned debug output of the macro calls map M string + // PrattM contains the expected adorned debug output of the macro calls map for the Pratt parser when it differs from M. + PrattM string + // Opts contains the list of options to be configured with the parser before parsing the expression. Opts []Option } @@ -2217,119 +2425,154 @@ func convertMacroCallsToString(source *ast.SourceInfo) string { } func TestParse(t *testing.T) { - defaultParser := newTestParser(t) - for i, tst := range testCases { - name := fmt.Sprintf("%d %s", i, tst.I) - // Local variable required as the closure will reference the value for the last - // 'tst' value rather than the local 'tc' instance declared within the loop. - tc := tst + for _, pratt := range []bool{false, true} { + name := fmt.Sprintf("enablePrattParser=%t", pratt) t.Run(name, func(t *testing.T) { - // Runs the tests in parallel to ensure that there are no data races - // due to shared mutable state across tests. - t.Parallel() - p := defaultParser - if len(tc.Opts) > 0 { - p = newTestParser(t, tc.Opts...) - } - src := common.NewTextSource(tc.I) - parsed, errors := p.Parse(src) - if len(errors.GetErrors()) > 0 { - actualErr := errors.ToDisplayString() - if tc.E == "" { - t.Fatalf("Unexpected errors: %v", actualErr) - } else if !test.Compare(actualErr, tc.E) { - t.Fatal(test.DiffMessage("Error mismatch", actualErr, tc.E)) - } - return - } else if tc.E != "" { - t.Fatalf("Expected error not thrown: '%s'", tc.E) - } - failureDisplayMethod := fmt.Sprintf("Parse(\"%s\")", tc.I) - actualWithKind := debug.ToAdornedDebugString(parsed.Expr(), &kindAndIDAdorner{}) - if !test.Compare(actualWithKind, tc.P) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Structure - %s", failureDisplayMethod), actualWithKind, tc.P)) - } + defaultParser := newTestParser(t, EnablePrattParser(pratt)) + for i, tst := range testCases { + name := fmt.Sprintf("%d %s", i, tst.I) + // Local variable required as the closure will reference the value for the last + // 'tst' value rather than the local 'tc' instance declared within the loop. + tc := tst + t.Run(name, func(t *testing.T) { + // Runs the tests in parallel to ensure that there are no data races + // due to shared mutable state across tests. + t.Parallel() + p := defaultParser + if len(tc.Opts) > 0 { + p = newTestParser(t, append([]Option{EnablePrattParser(pratt)}, tc.Opts...)...) + } + src := common.NewTextSource(tc.I) + parsed, errors := p.Parse(src) + wantP := tc.P + wantE := tc.E + wantL := tc.L + wantM := tc.M + if pratt { + if tc.PrattP != "" { + wantP = tc.PrattP + } + if tc.PrattE == "-" { + wantE = "" + } else if tc.PrattE != "" { + wantE = tc.PrattE + } + if tc.PrattL != "" { + wantL = tc.PrattL + } + if tc.PrattM != "" { + wantM = tc.PrattM + } + } + if len(errors.GetErrors()) > 0 { + actualErr := errors.ToDisplayString() + if wantE == "" { + t.Fatalf("Unexpected errors: %v", actualErr) + } else if !test.Compare(actualErr, wantE) { + t.Fatal(test.DiffMessage("Error mismatch", actualErr, wantE)) + } + return + } else if wantE != "" { + t.Fatalf("Expected error not thrown: '%s'", wantE) + } + failureDisplayMethod := fmt.Sprintf("Parse(\"%s\")", tc.I) + if wantP != "" { + actualWithKind := debug.ToAdornedDebugString(parsed.Expr(), &kindAndIDAdorner{}) + if !test.Compare(actualWithKind, wantP) { + t.Fatal(test.DiffMessage(fmt.Sprintf("Structure - %s", failureDisplayMethod), actualWithKind, wantP)) + } + } - if tc.L != "" { - actualWithLocation := debug.ToAdornedDebugString(parsed.Expr(), &locationAdorner{parsed.SourceInfo()}) - if !test.Compare(actualWithLocation, tc.L) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Location - %s", failureDisplayMethod), actualWithLocation, tc.L)) - } - } + if !pratt && wantL != "" { + actualWithLocation := debug.ToAdornedDebugString(parsed.Expr(), &locationAdorner{parsed.SourceInfo()}) + if !test.Compare(actualWithLocation, wantL) { + t.Fatal(test.DiffMessage(fmt.Sprintf("Location - %s", failureDisplayMethod), actualWithLocation, wantL)) + } + } - if tc.M != "" { - actualAdornedMacroCalls := convertMacroCallsToString(parsed.SourceInfo()) - if !test.Compare(actualAdornedMacroCalls, tc.M) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Macro Calls - %s", failureDisplayMethod), actualAdornedMacroCalls, tc.M)) - } - } + if wantM != "" { + actualAdornedMacroCalls := convertMacroCallsToString(parsed.SourceInfo()) + if !test.Compare(actualAdornedMacroCalls, wantM) { + t.Fatal(test.DiffMessage(fmt.Sprintf("Macro Calls - %s", failureDisplayMethod), actualAdornedMacroCalls, wantM)) + } + } - // Verify there are no unused IDs in the source info. - astIDs := parsed.IDs() - unusedIDs := []int64{} - for id := range parsed.SourceInfo().OffsetRanges() { - if !astIDs[id] { - unusedIDs = append(unusedIDs, id) - } - } - if len(unusedIDs) > 0 { - t.Errorf("SourceInfo has offset range for IDs %v, but no such nodes exists in AST: %s", - unusedIDs, debug.ToDebugStringWithIDs(parsed.Expr())) - } + // Verify there are no unused IDs in the source info. + astIDs := parsed.IDs() + unusedIDs := []int64{} + for id := range parsed.SourceInfo().OffsetRanges() { + if !astIDs[id] { + unusedIDs = append(unusedIDs, id) + } + } + if len(unusedIDs) > 0 { + t.Errorf("SourceInfo has offset range for IDs %v, but no such nodes exists in AST: %s", + unusedIDs, debug.ToDebugStringWithIDs(parsed.Expr())) + } - // Verify that source info offset ranges are shifted when the source is prepended with whitespace. - padding := strings.Repeat(" \n", 10) - padSrc := &RelativeSource{ - Source: common.NewTextSource(padding + src.Content()), - localSrc: src, - absLoc: common.NewLocation(11, 0), - } - padded, padErrs := p.Parse(padSrc) - if len(padErrs.GetErrors()) > 0 { - t.Fatalf("Unexpected errors with padded source: %v", padErrs.ToDisplayString()) - } - for id, origRange := range parsed.SourceInfo().OffsetRanges() { - padRange, found := padded.SourceInfo().GetOffsetRange(id) - if !found { - t.Errorf("ID %d not found in padded source info", id) - continue - } - want := ast.OffsetRange{Start: origRange.Start + 100, Stop: origRange.Stop + 100} - if padRange != want { - t.Errorf("ID %d offset range mismatch: got %v, want %v", id, padRange, want) - } + // Verify that source info offset ranges are shifted when the source is prepended with whitespace. + padding := strings.Repeat(" \n", 10) + padSrc := &RelativeSource{ + Source: common.NewTextSource(padding + src.Content()), + localSrc: src, + absLoc: common.NewLocation(11, 0), + } + padded, padErrs := p.Parse(padSrc) + if len(padErrs.GetErrors()) > 0 { + t.Fatalf("Unexpected errors with padded source: %v", padErrs.ToDisplayString()) + } + for id, origRange := range parsed.SourceInfo().OffsetRanges() { + padRange, found := padded.SourceInfo().GetOffsetRange(id) + if !found { + t.Errorf("ID %d not found in padded source info", id) + continue + } + want := ast.OffsetRange{Start: origRange.Start + 100, Stop: origRange.Stop + 100} + if padRange != want { + t.Errorf("ID %d offset range mismatch: got %v, want %v", id, padRange, want) + } + } + }) } }) } } func TestExpressionSizeCodePointLimit(t *testing.T) { - p, err := NewParser(Macros(AllMacros...), ExpressionSizeCodePointLimit((2))) - if err != nil { - t.Fatal(err) - } - src := common.NewTextSource("foo") - _, errs := p.Parse(src) - if got, want := len(errs.GetErrors()), 1; got != want { - t.Fatalf("got %d errors, want %d errors: %s", got, want, errs.ToDisplayString()) - } - if got, want := errs.GetErrors()[0].Message, "expression code point size exceeds limit: size: 3, limit 2"; got != want { - t.Fatalf("got %q, want %q: %s", got, want, errs.GetErrors()[0].ToDisplayString(src)) + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), ExpressionSizeCodePointLimit(2), EnablePrattParser(pratt)) + if err != nil { + t.Fatal(err) + } + src := common.NewTextSource("foo") + _, errs := p.Parse(src) + if got, want := len(errs.GetErrors()), 1; got != want { + t.Fatalf("got %d errors, want %d errors: %s", got, want, errs.ToDisplayString()) + } + if got, want := errs.GetErrors()[0].Message, "expression code point size exceeds limit: size: 3, limit 2"; got != want { + t.Fatalf("got %q, want %q: %s", got, want, errs.GetErrors()[0].ToDisplayString(src)) + } + }) } } func TestMaxExpressionNodeCount(t *testing.T) { - p, err := NewParser(Macros(AllMacros...), MaxExpressionNodeCount(10)) - if err != nil { - t.Fatal(err) - } - src := common.NewTextSource("a.exists(x, x.exists(y, y == 1))") - _, errs := p.Parse(src) - if len(errs.GetErrors()) == 0 { - t.Fatalf("expected errors, got none: %s", errs.ToDisplayString()) - } - if !strings.Contains(errs.GetErrors()[0].Message, "expression count exceeds limit of 10 while expanding macro 'exists'") { - t.Fatalf("got %q, want substring matching limit error: %s", errs.GetErrors()[0].Message, errs.GetErrors()[0].ToDisplayString(src)) + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), MaxExpressionNodeCount(10), EnablePrattParser(pratt)) + if err != nil { + t.Fatal(err) + } + src := common.NewTextSource("a.exists(x, x.exists(y, y == 1))") + _, errs := p.Parse(src) + if len(errs.GetErrors()) == 0 { + t.Fatalf("expected errors, got none: %s", errs.ToDisplayString()) + } + if !strings.Contains(errs.GetErrors()[0].Message, "expression count exceeds limit of 10 while expanding macro 'exists'") { + t.Fatalf("got %q, want substring matching limit error: %s", errs.GetErrors()[0].Message, errs.GetErrors()[0].ToDisplayString(src)) + } + }) } } @@ -2354,6 +2597,245 @@ func TestParserOptionErrors(t *testing.T) { } } +func TestSourceInfoPositions(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("ASCII", func(t *testing.T) { + src := common.NewTextSource("a + b") + p, err := NewParser(EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(src) + if len(errs.GetErrors()) > 0 { + t.Fatalf("Parse() failed: %s", errs.ToDisplayString()) + } + sourceInfo := parsed.SourceInfo() + root := parsed.Expr() + if sourceInfo.GetStartLocation(root.ID()).Column() != 2 { + t.Errorf("expected root column 2, got %d", sourceInfo.GetStartLocation(root.ID()).Column()) + } + args := root.AsCall().Args() + if len(args) != 2 { + t.Fatalf("expected 2 args, got %d", len(args)) + } + if sourceInfo.GetStartLocation(args[0].ID()).Column() != 0 { + t.Errorf("expected arg[0] column 0, got %d", sourceInfo.GetStartLocation(args[0].ID()).Column()) + } + if sourceInfo.GetStartLocation(args[1].ID()).Column() != 4 { + t.Errorf("expected arg[1] column 4, got %d", sourceInfo.GetStartLocation(args[1].ID()).Column()) + } + }) + + t.Run("MixedUnicodeMultiByteAndMultiLine", func(t *testing.T) { + // Mix of 1-byte ASCII (a, b, +), 2-byte Unicode ("ñ"), 3-byte Unicode ("❤"), and 4-byte Unicode ("🚀") + expr := "a + \"ñ\" +\n\"🚀\" + \"❤\" + b" + src := common.NewTextSource(expr) + p, err := NewParser(EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(src) + if len(errs.GetErrors()) > 0 { + t.Fatalf("Parse() failed: %s", errs.ToDisplayString()) + } + sinfo := parsed.SourceInfo() + + // AST hierarchy: + // expr4: [expr3] + [b] (line 2, col 10) + // expr3: [expr2] + ["❤"] (line 2, col 4) + // expr2: [expr1] + ["🚀"] (line 1, col 8) + // expr1: [a] + ["ñ"] (line 1, col 2) + expr4 := parsed.Expr() + call4 := expr4.AsCall() + bExpr := call4.Args()[1] + + expr3 := call4.Args()[0] + call3 := expr3.AsCall() + heartExpr := call3.Args()[1] + + expr2 := call3.Args()[0] + call2 := expr2.AsCall() + rocketExpr := call2.Args()[1] + + expr1 := call2.Args()[0] + call1 := expr1.AsCall() + aExpr := call1.Args()[0] + enyeExpr := call1.Args()[1] + + assertLoc := func(name string, id int64, wantLine, wantCol int32) { + t.Helper() + loc := sinfo.GetStartLocation(id) + if int32(loc.Line()) != wantLine || int32(loc.Column()) != wantCol { + t.Errorf("%s location mismatch: got (%d, %d), want (%d, %d)", + name, loc.Line(), loc.Column(), wantLine, wantCol) + } + } + + assertLoc("call1 (+)", expr1.ID(), 1, 2) + assertLoc("a", aExpr.ID(), 1, 0) + assertLoc("\"ñ\" (2-byte)", enyeExpr.ID(), 1, 4) + assertLoc("call2 (+)", expr2.ID(), 1, 8) + assertLoc("\"🚀\" (4-byte)", rocketExpr.ID(), 2, 0) + assertLoc("call3 (+)", expr3.ID(), 2, 4) + assertLoc("\"❤\" (3-byte)", heartExpr.ID(), 2, 6) + assertLoc("call4 (+)", expr4.ID(), 2, 10) + assertLoc("b", bExpr.ID(), 2, 12) + }) + }) + } +} + +func TestPopulateMacroCalls(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("DisabledByDefault", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(false), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + if len(parsed.SourceInfo().MacroCalls()) != 0 { + t.Errorf("expected 0 macro calls, got %d", len(parsed.SourceInfo().MacroCalls())) + } + }) + + t.Run("GlobalMacroCallRecorded", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 1 { + t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) + } + }) + + t.Run("ReceiverMacroCallRecorded", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("[1, 2].exists(x, x > 0)")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 1 { + t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) + } + }) + + t.Run("NestedMacroCallsRecorded", func(t *testing.T) { + p, err := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + parsed, errs := p.Parse(common.NewTextSource("[1, 2].all(x, has(x.b))")) + if len(errs.GetErrors()) > 0 { + t.Fatalf("unexpected error: %s", errs.ToDisplayString()) + } + macroCalls := parsed.SourceInfo().MacroCalls() + if len(macroCalls) != 2 { + t.Fatalf("expected 2 macro calls, got %d", len(macroCalls)) + } + }) + }) + } +} + +func TestErrorRecoveryLimits(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("LimitZero", func(t *testing.T) { + p, err := NewParser(ErrorRecoveryLimit(0), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("......")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected error recovery limit error, got none") + } + }) + + t.Run("LimitOne", func(t *testing.T) { + p, err := NewParser(ErrorRecoveryLimit(1), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("......")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected error recovery limit error, got none") + } + }) + }) + } +} + +func TestRecursionLimit(t *testing.T) { + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + t.Run("DeeplyNestedBracketsLimitExceeded", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(5), EnablePrattParser(pratt)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("[[[[[[1]]]]]]")) + if len(errs.GetErrors()) == 0 { + t.Errorf("expected recursion limit error, got none") + } + }) + }) + } + + t.Run("PrattSequentialScopesDoNotAccumulateDepth", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(2), EnablePrattParser(true)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("[1] + [2] + [3]")) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on sequential scopes: %s", errs.ToDisplayString()) + } + }) + + t.Run("PrattIgnoreExtraParens", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(1), EnablePrattParser(true)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + _, errs := p.Parse(common.NewTextSource("((((1))))")) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error: %s", errs.ToDisplayString()) + } + }) + + t.Run("PrattDeeplyNestedParens1000", func(t *testing.T) { + p, err := NewParser(MaxRecursionDepth(1), EnablePrattParser(true)) + if err != nil { + t.Fatalf("NewParser() failed: %v", err) + } + expr1 := strings.Repeat("(", 1000) + "42" + strings.Repeat(")", 1000) + _, errs := p.Parse(common.NewTextSource(expr1)) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on 1000 parens literal: %s", errs.ToDisplayString()) + } + + expr2 := strings.Repeat("(", 1000) + "1 + 2" + strings.Repeat(")", 1000) + _, errs = p.Parse(common.NewTextSource(expr2)) + if len(errs.GetErrors()) > 0 { + t.Errorf("unexpected error on 1000 parens binary: %s", errs.ToDisplayString()) + } + }) +} + func BenchmarkParse(b *testing.B) { p, err := NewParser( Macros(AllMacros...), @@ -2551,19 +3033,28 @@ var benchCategories = []benchCategory{ // BenchmarkByCategory benchmarks parsing organized by workload categories. func BenchmarkByCategory(b *testing.B) { - p := newBenchmarkCategoryParser(b) - for _, cat := range benchCategories { - b.Run(cat.name, func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, tc := range cat.cases { - src := common.NewTextSource(tc.I) - _, errs := p.Parse(src) - hasErr := len(errs.GetErrors()) > 0 - if hasErr != tc.E { - b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + for _, pratt := range []bool{false, true} { + mode := "antlr" + if pratt { + mode = "pratt" + } + b.Run(mode, func(b *testing.B) { + p := newBenchmarkCategoryParser(b, EnablePrattParser(pratt)) + for _, cat := range benchCategories { + b.Run(cat.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, tc := range cat.cases { + src := common.NewTextSource(tc.I) + _, errs := p.Parse(src) + hasErr := len(errs.GetErrors()) > 0 + if hasErr != tc.E { + b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + } + } } - } + }) } }) } @@ -2571,22 +3062,31 @@ func BenchmarkByCategory(b *testing.B) { // BenchmarkParallelByCategory benchmarks parsing concurrently across goroutines by category. func BenchmarkParallelByCategory(b *testing.B) { - p := newBenchmarkCategoryParser(b) - for _, cat := range benchCategories { - b.Run(cat.name, func(b *testing.B) { - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - for _, tc := range cat.cases { - src := common.NewTextSource(tc.I) - _, errs := p.Parse(src) - hasErr := len(errs.GetErrors()) > 0 - if hasErr != tc.E { - b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + for _, pratt := range []bool{false, true} { + mode := "antlr" + if pratt { + mode = "pratt" + } + b.Run(mode, func(b *testing.B) { + p := newBenchmarkCategoryParser(b, EnablePrattParser(pratt)) + for _, cat := range benchCategories { + b.Run(cat.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + for _, tc := range cat.cases { + src := common.NewTextSource(tc.I) + _, errs := p.Parse(src) + hasErr := len(errs.GetErrors()) > 0 + if hasErr != tc.E { + b.Fatalf("p.Parse(%q) got error: %v, expected error: %v", tc.I, hasErr, tc.E) + } + } } - } - } - }) + }) + }) + } }) } } @@ -2622,14 +3122,15 @@ func optMapExpander(meh ExprHelper, target ast.Expr, args []ast.Expr) (ast.Expr, ), nil } -func newBenchmarkCategoryParser(tb testing.TB) *Parser { +func newBenchmarkCategoryParser(tb testing.TB, options ...Option) *Parser { tb.Helper() - p, err := NewParser( + opts := append([]Option{ Macros(append(AllMacros, optMapMacro)...), EnableOptionalSyntax(true), EnableIdentEscapeSyntax(true), MaxRecursionDepth(512), - ) + }, options...) + p, err := NewParser(opts...) if err != nil { tb.Fatalf("NewParser() failed: %v", err) } @@ -2637,18 +3138,22 @@ func newBenchmarkCategoryParser(tb testing.TB) *Parser { } func TestParseErrorData(t *testing.T) { - p := newTestParser(t) - src := common.NewTextSource(`a.?b`) - _, iss := p.Parse(src) - if len(iss.GetErrors()) != 1 { - t.Fatalf("Check() of a bad expression did produce a single error: %v", iss.ToDisplayString()) - } - celErr := iss.GetErrors()[0] - if celErr.ExprID != 2 { - t.Errorf("got exprID %v, wanted 2", celErr.ExprID) - } - if !strings.Contains(celErr.Message, "unsupported syntax") { - t.Errorf("got message %v, wanted unsupported syntax", celErr.Message) + for _, pratt := range []bool{false, true} { + t.Run(fmt.Sprintf("enablePrattParser=%t", pratt), func(t *testing.T) { + p := newTestParser(t, EnablePrattParser(pratt)) + src := common.NewTextSource(`a.?b`) + _, iss := p.Parse(src) + if len(iss.GetErrors()) != 1 { + t.Fatalf("Check() of a bad expression did produce a single error: %v", iss.ToDisplayString()) + } + celErr := iss.GetErrors()[0] + if celErr.ExprID != 2 { + t.Errorf("got exprID %v, wanted 2", celErr.ExprID) + } + if !strings.Contains(celErr.Message, "unsupported syntax") { + t.Errorf("got message %v, wanted unsupported syntax", celErr.Message) + } + }) } } diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go index c2eefd927..f9163322d 100644 --- a/parser/pratt_parser.go +++ b/parser/pratt_parser.go @@ -16,7 +16,6 @@ package parser import ( "fmt" - "math" "strconv" "strings" @@ -86,7 +85,7 @@ func getBinaryOpInfo(kind tokenKind) binaryOpInfo { } } -type prattParser struct { +type prattParserWorker struct { content runes.Buffer length int32 helper *parserHelper @@ -109,53 +108,13 @@ type prattParser struct { enableIdentEscapeSyntax bool } -// PrattParser encapsulates the context necessary to perform Pratt parsing for different expressions. -type PrattParser struct { +// prattParser encapsulates the context necessary to perform Pratt parsing for different expressions. +type prattParser struct { options } -// NewPrattParser builds and returns a new PrattParser using the provided options. -func NewPrattParser(opts ...Option) (*PrattParser, error) { - p := &PrattParser{} - p.enableHiddenAccumulatorName = true - p.enableIdentEscapeSyntax = true - for _, opt := range opts { - if err := opt(&p.options); err != nil { - return nil, err - } - } - if p.errorReportingLimit == 0 { - p.errorReportingLimit = 100 - } - if p.maxRecursionDepth == 0 { - p.maxRecursionDepth = 250 - } - if p.maxRecursionDepth == -1 { - p.maxRecursionDepth = math.MaxInt - } - if p.errorRecoveryLimit == 0 { - p.errorRecoveryLimit = 30 - } - if p.errorRecoveryLimit == -1 { - p.errorRecoveryLimit = math.MaxInt - } - if p.expressionSizeCodePointLimit == 0 { - p.expressionSizeCodePointLimit = 100_000 - } - if p.expressionSizeCodePointLimit == -1 { - p.expressionSizeCodePointLimit = math.MaxInt - } - if p.maxExpressionNodeCount == 0 { - p.maxExpressionNodeCount = 100_000 - } - if p.maxExpressionNodeCount == -1 { - p.maxExpressionNodeCount = math.MaxInt - } - return p, nil -} - // Parse parses the expression represented by source using the Pratt parser and returns the result. -func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { +func (p *prattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { errs := common.NewErrors(source) buf, ok := source.(runes.Buffer) if !ok { @@ -172,7 +131,7 @@ func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { accu = HiddenAccumulatorName } fac := ast.NewExprFactoryWithAccumulator(accu) - pratt := &prattParser{ + pratt := &prattParserWorker{ content: buf, length: int32(buf.Len()), helper: newParserHelper(source, fac), @@ -197,16 +156,16 @@ func (p *PrattParser) Parse(source common.Source) (*ast.AST, *common.Errors) { return ast.NewAST(out, pratt.helper.getSourceInfo()), errs } -func (p *prattParser) initTokenStream() { +func (p *prattParserWorker) initTokenStream() { p.currTok = token{kind: tokError, start: 0, end: 0} p.peekTok = p.nextSignificantToken(true) } -func (p *prattParser) isRecoveryLimitExceeded() bool { +func (p *prattParserWorker) isRecoveryLimitExceeded() bool { return p.errorCount > p.errorRecoveryLimit } -func (p *prattParser) nextSignificantToken(reportError bool) token { +func (p *prattParserWorker) nextSignificantToken(reportError bool) token { if p.isRecoveryLimitExceeded() { return token{kind: tokEnd, start: p.length, end: p.length} } @@ -225,7 +184,7 @@ func (p *prattParser) nextSignificantToken(reportError bool) token { } } -func (p *prattParser) nextToken() token { +func (p *prattParserWorker) nextToken() token { p.currTok = p.peekTok if p.isRecoveryLimitExceeded() { p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} @@ -237,18 +196,18 @@ func (p *prattParser) nextToken() token { return p.currTok } -func (p *prattParser) tokenText(tok token) string { +func (p *prattParserWorker) tokenText(tok token) string { if tok.start >= 0 && tok.end >= tok.start && tok.end <= p.length { return p.content.Slice(int(tok.start), int(tok.end)) } return "" } -func (p *prattParser) nextID(tok token) int64 { +func (p *prattParserWorker) nextID(tok token) int64 { return p.helper.idFromOffsets(tok.start, tok.end) } -func (p *prattParser) expect(kind tokenKind, msg string) bool { +func (p *prattParserWorker) expect(kind tokenKind, msg string) bool { if p.peekTok.kind == kind { p.nextToken() return true @@ -271,7 +230,7 @@ func (p *prattParser) expect(kind tokenKind, msg string) bool { return false } -func (p *prattParser) synchronizeOnDelimiter() { +func (p *prattParserWorker) synchronizeOnDelimiter() { if p.isRecoveryLimitExceeded() { p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} return @@ -287,8 +246,8 @@ func (p *prattParser) synchronizeOnDelimiter() { } } -func (p *prattParser) reportError(ctx any, format string, args ...any) ast.Expr { - if p.errorCount > p.errorRecoveryLimit { +func (p *prattParserWorker) reportError(ctx any, format string, args ...any) ast.Expr { + if p.isRecoveryLimitExceeded() { return p.helper.newExpr(common.NoLocation) } p.errorCount++ @@ -302,39 +261,37 @@ func (p *prattParser) reportError(ctx any, format string, args ...any) ast.Expr default: location = p.helper.getLocation(err.ID()) } - if p.errorCount == p.errorRecoveryLimit+1 { - p.errors.syntaxError(location, fmt.Sprintf("error recovery attempt limit exceeded: %d", p.errorRecoveryLimit)) - p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} - return err - } if p.errorCount <= p.errorReportingLimit { p.errors.reportErrorAtID(err.ID(), location, format, args...) } + if p.isRecoveryLimitExceeded() { + p.peekTok = token{kind: tokEnd, start: p.length, end: p.length} + } return err } -func (p *prattParser) newLogicManager(function string, term ast.Expr) *logicManager { +func (p *prattParserWorker) newLogicManager(function string, term ast.Expr) *logicManager { if p.enableVariadicOperatorASTs { return newVariadicLogicManager(p.exprFactory, function, term) } return newBalancingLogicManager(p.exprFactory, function, term) } -func (p *prattParser) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { +func (p *prattParserWorker) globalCallOrMacro(exprID int64, function string, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, nil, args...); found { return expr } return p.helper.newGlobalCall(exprID, function, args...) } -func (p *prattParser) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { +func (p *prattParserWorker) receiverCallOrMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) ast.Expr { if expr, found := p.expandMacro(exprID, function, target, args...); found { return expr } return p.helper.newReceiverCall(exprID, function, target, args...) } -func (p *prattParser) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { +func (p *prattParserWorker) expandMacro(exprID int64, function string, target ast.Expr, args ...ast.Expr) (ast.Expr, bool) { if len(p.macros) == 0 { return nil, false } @@ -378,7 +335,7 @@ func (p *prattParser) expandMacro(exprID int64, function string, target ast.Expr return expr, true } -func (p *prattParser) normalizeIdent(tok token, allowQuoted bool) string { +func (p *prattParserWorker) normalizeIdent(tok token, allowQuoted bool) string { text := p.tokenText(tok) if len(text) == 0 { return "" @@ -389,7 +346,7 @@ func (p *prattParser) normalizeIdent(tok token, allowQuoted bool) string { return "" } if !p.enableIdentEscapeSyntax { - p.reportError(tok, "unsupported syntax '`'") + p.reportError(tok, "unsupported syntax: '`'") } if len(text) < 2 || text[len(text)-1] != '`' { p.reportError(tok, "unterminated quoted identifier") @@ -413,18 +370,23 @@ func (p *prattParser) normalizeIdent(tok token, allowQuoted bool) string { return text } -func (p *prattParser) parse() ast.Expr { +func (p *prattParserWorker) parse() ast.Expr { expr := p.parseExpr() if p.recursionLimitExceeded || p.isRecoveryLimitExceeded() { return expr } - if p.peekTok.kind != tokEnd && p.peekTok.kind != tokError { - p.reportError(p.peekTok, "Syntax error: mismatched input '%s' expecting ", p.tokenText(p.peekTok)) + if p.peekTok.kind != tokEnd { + if p.peekTok.kind != tokError { + p.reportError(p.peekTok, "Syntax error: mismatched input '%s' expecting ", p.tokenText(p.peekTok)) + } + for p.peekTok.kind != tokEnd && !p.isRecoveryLimitExceeded() { + p.nextToken() + } } return expr } -func (p *prattParser) parseExpr() ast.Expr { +func (p *prattParserWorker) parseExpr() ast.Expr { if p.recursionLimitExceeded || p.isRecoveryLimitExceeded() { return p.helper.newExpr(common.NoLocation) } @@ -439,7 +401,7 @@ func (p *prattParser) parseExpr() ast.Expr { return expr } -func (p *prattParser) parseBinaryAndTernary(minPrec int) ast.Expr { +func (p *prattParserWorker) parseBinaryAndTernary(minPrec int) ast.Expr { lhs := p.parseSelectorChain() for { tok := p.peekTok.kind @@ -466,7 +428,7 @@ func (p *prattParser) parseBinaryAndTernary(minPrec int) ast.Expr { return lhs } -func (p *prattParser) parseTernary(lhs ast.Expr) ast.Expr { +func (p *prattParserWorker) parseTernary(lhs ast.Expr) ast.Expr { qTok := p.nextToken() opID := p.nextID(qTok) trueExpr := p.parseBinaryAndTernary(1) @@ -477,23 +439,23 @@ func (p *prattParser) parseTernary(lhs ast.Expr) ast.Expr { return p.helper.newGlobalCall(opID, operators.Conditional, lhs, trueExpr, falseExpr) } -func (p *prattParser) parseLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.Expr { +func (p *prattParserWorker) parseLogicalChain(lhs ast.Expr, opInfo binaryOpInfo) ast.Expr { l := p.newLogicManager(opInfo.name, lhs) for p.peekTok.kind == opInfo.kind { opTok := p.nextToken() - opID := p.nextID(opTok) rhs := p.parseBinaryAndTernary(opInfo.precedence + 1) + opID := p.nextID(opTok) l.addTerm(opID, rhs) } return l.toExpr() } -func (p *prattParser) parseSelectorChain() ast.Expr { +func (p *prattParserWorker) parseSelectorChain() ast.Expr { lhs := p.parseUnary() return p.parseSelectorChainTail(lhs) } -func (p *prattParser) parseSelectorChainTail(lhs ast.Expr) ast.Expr { +func (p *prattParserWorker) parseSelectorChainTail(lhs ast.Expr) ast.Expr { for { switch p.peekTok.kind { case tokDot: @@ -548,8 +510,13 @@ func (p *prattParser) parseSelectorChainTail(lhs ast.Expr) ast.Expr { } lhs = p.helper.newGlobalCall(opID, opName, lhs, index) case tokLeftBrace: - if structName, ok := p.extractStructName(lhs); ok { - lhs = p.parseStruct(lhs.ID(), structName) + if rng, found := p.helper.sourceInfo.GetOffsetRange(lhs.ID()); found { + if structName, ok := p.extractStructName(lhs); ok { + objID := p.helper.id(rng) + lhs = p.parseStruct(objID, structName) + } else { + return lhs + } } else { return lhs } @@ -559,7 +526,7 @@ func (p *prattParser) parseSelectorChainTail(lhs ast.Expr) ast.Expr { } } -func (p *prattParser) extractStructName(expr ast.Expr) (string, bool) { +func (p *prattParserWorker) extractStructName(expr ast.Expr) (string, bool) { if expr == nil || expr.Kind() == ast.LiteralKind { return "", false } @@ -583,7 +550,7 @@ func (p *prattParser) extractStructName(expr ast.Expr) (string, bool) { return "", false } -func (p *prattParser) parseStruct(objID int64, structName string) ast.Expr { +func (p *prattParserWorker) parseStruct(objID int64, structName string) ast.Expr { p.nextToken() // consumes { var fields []ast.EntryExpr for p.peekTok.kind != tokRightBrace && p.peekTok.kind != tokEnd { @@ -619,7 +586,7 @@ func (p *prattParser) parseStruct(objID int64, structName string) ast.Expr { return p.helper.newObject(objID, structName, fields...) } -func (p *prattParser) parseUnary() ast.Expr { +func (p *prattParserWorker) parseUnary() ast.Expr { tok := p.peekTok.kind if tok == tokExclamation || tok == tokMinus { return p.parseUnaryOps() @@ -627,7 +594,7 @@ func (p *prattParser) parseUnary() ast.Expr { return p.parsePrimary() } -func (p *prattParser) parseUnaryOps() ast.Expr { +func (p *prattParserWorker) parseUnaryOps() ast.Expr { op := p.nextToken() if p.peekTok.kind == tokExclamation || p.peekTok.kind == tokMinus { return p.parseUnaryOpsChain(op) @@ -651,7 +618,7 @@ func (p *prattParser) parseUnaryOps() ast.Expr { return p.globalCallOrMacro(opID, opName, operand) } -func (p *prattParser) parseUnaryOpsChain(firstOp token) ast.Expr { +func (p *prattParserWorker) parseUnaryOpsChain(firstOp token) ast.Expr { type unaryOpInfo struct { kind tokenKind id int64 @@ -691,7 +658,7 @@ func (p *prattParser) parseUnaryOpsChain(firstOp token) ast.Expr { return operand } -func (p *prattParser) countGroupingParentheses() int { +func (p *prattParserWorker) countGroupingParentheses() int { if p.peekTok.kind != tokLeftParen { return 0 } @@ -740,7 +707,7 @@ func (p *prattParser) countGroupingParentheses() int { return 1 } -func (p *prattParser) parsePrimary() ast.Expr { +func (p *prattParserWorker) parsePrimary() ast.Expr { switch p.peekTok.kind { case tokLeftParen: groupingCount := p.countGroupingParentheses() @@ -789,7 +756,7 @@ func (p *prattParser) parsePrimary() ast.Expr { } } -func (p *prattParser) parseList() ast.Expr { +func (p *prattParserWorker) parseList() ast.Expr { openTok := p.nextToken() listID := p.nextID(openTok) var elems []ast.Expr @@ -821,7 +788,7 @@ func (p *prattParser) parseList() ast.Expr { return p.helper.newList(listID, elems, optionals...) } -func (p *prattParser) parseMap() ast.Expr { +func (p *prattParserWorker) parseMap() ast.Expr { openTok := p.nextToken() mapID := p.nextID(openTok) var entries []ast.EntryExpr @@ -855,7 +822,7 @@ func (p *prattParser) parseMap() ast.Expr { return p.helper.newMap(mapID, entries...) } -func (p *prattParser) parseIdentOrCall() ast.Expr { +func (p *prattParserWorker) parseIdentOrCall() ast.Expr { leadingDot := false firstTok := p.peekTok if p.peekTok.kind == tokDot { @@ -888,7 +855,7 @@ func (p *prattParser) parseIdentOrCall() ast.Expr { return p.helper.newIdent(id, name) } -func (p *prattParser) parseArguments(closeTok tokenKind) []ast.Expr { +func (p *prattParserWorker) parseArguments(closeTok tokenKind) []ast.Expr { var args []ast.Expr if p.peekTok.kind != closeTok && p.peekTok.kind != tokEnd { for { @@ -907,7 +874,7 @@ func (p *prattParser) parseArguments(closeTok tokenKind) []ast.Expr { return args } -func (p *prattParser) parseIntLiteral() ast.Expr { +func (p *prattParserWorker) parseIntLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -923,7 +890,7 @@ func (p *prattParser) parseIntLiteral() ast.Expr { return p.helper.newLiteralInt(id, val) } -func (p *prattParser) parseNegativeIntLiteral(opID int64) ast.Expr { +func (p *prattParserWorker) parseNegativeIntLiteral(opID int64) ast.Expr { tok := p.nextToken() text := p.tokenText(tok) base := 10 @@ -938,7 +905,7 @@ func (p *prattParser) parseNegativeIntLiteral(opID int64) ast.Expr { return p.helper.newLiteralInt(opID, val) } -func (p *prattParser) parseUintLiteral() ast.Expr { +func (p *prattParserWorker) parseUintLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -955,7 +922,7 @@ func (p *prattParser) parseUintLiteral() ast.Expr { return p.helper.newLiteralUint(id, val) } -func (p *prattParser) parseDoubleLiteral() ast.Expr { +func (p *prattParserWorker) parseDoubleLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -966,7 +933,7 @@ func (p *prattParser) parseDoubleLiteral() ast.Expr { return p.helper.newLiteralDouble(id, val) } -func (p *prattParser) parseNegativeDoubleLiteral(opID int64) ast.Expr { +func (p *prattParserWorker) parseNegativeDoubleLiteral(opID int64) ast.Expr { tok := p.nextToken() text := p.tokenText(tok) val, err := strconv.ParseFloat(text, 64) @@ -976,7 +943,7 @@ func (p *prattParser) parseNegativeDoubleLiteral(opID int64) ast.Expr { return p.helper.newLiteralDouble(opID, -val) } -func (p *prattParser) parseStringLiteral() ast.Expr { +func (p *prattParserWorker) parseStringLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) @@ -987,7 +954,7 @@ func (p *prattParser) parseStringLiteral() ast.Expr { return p.helper.newLiteralString(id, unescaped) } -func (p *prattParser) parseBytesLiteral() ast.Expr { +func (p *prattParserWorker) parseBytesLiteral() ast.Expr { tok := p.nextToken() id := p.nextID(tok) text := p.tokenText(tok) diff --git a/parser/pratt_parser_test.go b/parser/pratt_parser_test.go deleted file mode 100644 index 17f450056..000000000 --- a/parser/pratt_parser_test.go +++ /dev/null @@ -1,1258 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package parser - -import ( - "fmt" - "strings" - "testing" - - "cel.dev/cel-go/common" - "cel.dev/cel-go/common/ast" - "cel.dev/cel-go/common/debug" - "cel.dev/cel-go/test" -) - -var prattTestCases = []testInfo{ - // Constants - { - I: `42`, - P: `42^#1:*expr.Constant_Int64Value#`, - }, - { - I: `0x2A`, - P: `42^#1:*expr.Constant_Int64Value#`, - }, - { - I: `42u`, - P: `42u^#1:*expr.Constant_Uint64Value#`, - }, - { - I: `0x2Au`, - P: `42u^#1:*expr.Constant_Uint64Value#`, - }, - { - I: `3.14`, - P: `3.14^#1:*expr.Constant_DoubleValue#`, - }, - { - I: `-42`, - P: `-42^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-3.14`, - P: `-3.14^#1:*expr.Constant_DoubleValue#`, - }, - { - I: `"hello world"`, - P: `"hello world"^#1:*expr.Constant_StringValue#`, - }, - { - I: `b"bytes"`, - P: `b"bytes"^#1:*expr.Constant_BytesValue#`, - }, - { - I: `true`, - P: `true^#1:*expr.Constant_BoolValue#`, - }, - { - I: `false`, - P: `false^#1:*expr.Constant_BoolValue#`, - }, - { - I: `null`, - P: `null^#1:*expr.Constant_NullValue#`, - }, - { - I: `9223372036854775807`, - P: `9223372036854775807^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-9223372036854775808`, - P: `-9223372036854775808^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-0x1A`, - P: `-26^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-0X1a`, - P: `-26^#1:*expr.Constant_Int64Value#`, - }, - { - I: `-0x8000000000000000`, - P: `-9223372036854775808^#1:*expr.Constant_Int64Value#`, - }, - { - I: `0u`, - P: `0u^#1:*expr.Constant_Uint64Value#`, - }, - { - I: `-5.5e-3`, - P: `-0.0055^#1:*expr.Constant_DoubleValue#`, - }, - { - I: "\"\u2764\"", - P: "\"❤\"^#1:*expr.Constant_StringValue#", - }, - { - I: "\"\\a\\b\\f\\n\\r\\t\\v'\\\"\\\\\\? Legal escapes\"", - P: `"\a\b\f\n\r\t\v'\"\\? Legal escapes"^#1:*expr.Constant_StringValue#`, - }, - - // Identifiers and Parentheses - { - I: `a`, - P: `a^#1:*expr.Expr_IdentExpr#`, - }, - { - I: `(a)`, - P: `a^#1:*expr.Expr_IdentExpr#`, - }, - { - I: `((a))`, - P: `a^#1:*expr.Expr_IdentExpr#`, - }, - - // Unary operators - { - I: `!a`, - P: `!_( - a^#2:*expr.Expr_IdentExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `!false`, - P: `!_( - false^#2:*expr.Constant_BoolValue# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `-x`, - P: `-_( - x^#2:*expr.Expr_IdentExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `---a`, - P: `-_( - -_( - -_( - a^#4:*expr.Expr_IdentExpr# - )^#3:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `- -1`, - P: `-_( - -1^#2:*expr.Constant_Int64Value# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `4--4`, - P: `_-_( - 4^#1:*expr.Constant_Int64Value#, - -4^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `!!a`, - P: `!_( - !_( - a^#3:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `-!true`, - P: `-_( - !_( - true^#3:*expr.Constant_BoolValue# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `-!-x`, - P: `-_( - !_( - -_( - x^#4:*expr.Expr_IdentExpr# - )^#3:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - - // Binary & Ternary operators - { - I: `1 + 2 * 3 - 4 / 2 % 3`, - P: `_-_( - _+_( - 1^#1:*expr.Constant_Int64Value#, - _*_( - 2^#3:*expr.Constant_Int64Value#, - 3^#5:*expr.Constant_Int64Value# - )^#4:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr#, - _%_( - _/_( - 4^#7:*expr.Constant_Int64Value#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr#, - 3^#11:*expr.Constant_Int64Value# - )^#10:*expr.Expr_CallExpr# - )^#6:*expr.Expr_CallExpr#`, - }, - { - I: `a < 10 && b <= 20 || c > 30 && d >= 40 || e == 50 && f != 60`, - P: `_||_( - _||_( - _&&_( - _<_( - a^#1:*expr.Expr_IdentExpr#, - 10^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - _<=_( - b^#5:*expr.Expr_IdentExpr#, - 20^#7:*expr.Constant_Int64Value# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#, - _&&_( - _>_( - c^#9:*expr.Expr_IdentExpr#, - 30^#11:*expr.Constant_Int64Value# - )^#10:*expr.Expr_CallExpr#, - _>=_( - d^#13:*expr.Expr_IdentExpr#, - 40^#15:*expr.Constant_Int64Value# - )^#14:*expr.Expr_CallExpr# - )^#12:*expr.Expr_CallExpr# - )^#8:*expr.Expr_CallExpr#, - _&&_( - _==_( - e^#17:*expr.Expr_IdentExpr#, - 50^#19:*expr.Constant_Int64Value# - )^#18:*expr.Expr_CallExpr#, - _!=_( - f^#21:*expr.Expr_IdentExpr#, - 60^#23:*expr.Constant_Int64Value# - )^#22:*expr.Expr_CallExpr# - )^#20:*expr.Expr_CallExpr# - )^#16:*expr.Expr_CallExpr#`, - }, - { - I: `a && b && c && d`, - P: `_&&_( - _&&_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#, - _&&_( - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `a && b && c && d`, - P: `_&&_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableVariadicOperatorASTs(true)}, - }, - { - I: `a || b || c || d`, - P: `_||_( - _||_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#, - _||_( - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `a || b || c || d`, - P: `_||_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#5:*expr.Expr_IdentExpr#, - d^#7:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableVariadicOperatorASTs(true)}, - }, - { - I: `10 - 3 - 2`, - P: `_-_( - _-_( - 10^#1:*expr.Constant_Int64Value#, - 3^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - 2^#5:*expr.Constant_Int64Value# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `(((10 - 3) - 2))`, - P: `_-_( - _-_( - 10^#1:*expr.Constant_Int64Value#, - 3^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - 2^#5:*expr.Constant_Int64Value# - )^#4:*expr.Expr_CallExpr#`, - }, - { - I: `x in [1, 2, 3]`, - P: `@in( - x^#1:*expr.Expr_IdentExpr#, - [ - 1^#4:*expr.Constant_Int64Value#, - 2^#5:*expr.Constant_Int64Value#, - 3^#6:*expr.Constant_Int64Value# - ]^#3:*expr.Expr_ListExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a ? b : c`, - P: `_?_:_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#4:*expr.Expr_IdentExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a ? b : c ? d : e`, - P: `_?_:_( - a^#1:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - _?_:_( - c^#4:*expr.Expr_IdentExpr#, - d^#6:*expr.Expr_IdentExpr#, - e^#7:*expr.Expr_IdentExpr# - )^#5:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `(((1 + 2))) * (3 + 4)`, - P: `_*_( - _+_( - 1^#1:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#, - _+_( - 3^#5:*expr.Constant_Int64Value#, - 4^#7:*expr.Constant_Int64Value# - )^#6:*expr.Expr_CallExpr# - )^#4:*expr.Expr_CallExpr#`, - }, - - // Members, Selects, Indexing, and Calls - { - I: `a.b()`, - P: `a^#1:*expr.Expr_IdentExpr#.b()^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a.b(1)`, - P: `a^#1:*expr.Expr_IdentExpr#.b( - 1^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a.b(1, 2)`, - P: `a^#1:*expr.Expr_IdentExpr#.b( - 1^#3:*expr.Constant_Int64Value#, - 2^#4:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `"foo".size()`, - P: `"foo"^#1:*expr.Constant_StringValue#.size()^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a.b.c`, - P: `a^#1:*expr.Expr_IdentExpr#.b^#2:*expr.Expr_SelectExpr#.c^#3:*expr.Expr_SelectExpr#`, - }, - { - I: `a[0]`, - P: `_[_]( - a^#1:*expr.Expr_IdentExpr#, - 0^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `[1, 3, 4][0]`, - P: `_[_]( - [ - 1^#2:*expr.Constant_Int64Value#, - 3^#3:*expr.Constant_Int64Value#, - 4^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - 0^#6:*expr.Constant_Int64Value# - )^#5:*expr.Expr_CallExpr#`, - }, - { - I: `a[b[c]]`, - P: `_[_]( - a^#1:*expr.Expr_IdentExpr#, - _[_]( - b^#3:*expr.Expr_IdentExpr#, - c^#5:*expr.Expr_IdentExpr# - )^#4:*expr.Expr_CallExpr# - )^#2:*expr.Expr_CallExpr#`, - }, - { - I: `a()`, - P: `a()^#1:*expr.Expr_CallExpr#`, - }, - { - I: `a(b)`, - P: `a( - b^#2:*expr.Expr_IdentExpr# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `func(1, 2, 3)`, - P: `func( - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - )^#1:*expr.Expr_CallExpr#`, - }, - { - I: `.func(1, 2)`, - P: `.func( - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value# - )^#1:*expr.Expr_CallExpr#`, - }, - - // Collections & Structs - { - I: `[1, 2, 3]`, - P: `[ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#`, - }, - { - I: `[1, 2, 3,]`, - P: `[ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#`, - }, - { - I: `[]`, - P: `[]^#1:*expr.Expr_ListExpr#`, - }, - { - I: `{"a": 1, "b": 2}`, - P: `{ - "a"^#2:*expr.Constant_StringValue#:1^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - "b"^#5:*expr.Constant_StringValue#:2^#7:*expr.Constant_Int64Value#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - }, - { - I: `{foo: 5, bar: "xyz"}`, - P: `{ - foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - }, - { - I: `{}`, - P: `{}^#1:*expr.Expr_StructExpr#`, - }, - { - I: `google.protobuf.Empty{}`, - P: `google.protobuf.Empty{}^#3:*expr.Expr_StructExpr#`, - }, - { - I: `foo{ a: b }`, - P: `foo{ - a:b^#2:*expr.Expr_IdentExpr#^#1:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - }, - { - I: `pkg.Msg{field1: "val", field2: 42}`, - P: `pkg.Msg{ - field1:"val"^#3:*expr.Constant_StringValue#^#2:*expr.Expr_CreateStruct_Entry#, - field2:42^#5:*expr.Constant_Int64Value#^#4:*expr.Expr_CreateStruct_Entry# - }^#2:*expr.Expr_StructExpr#`, - }, - { - I: `.pkg.Msg{field1: "val"}`, - P: `.pkg.Msg{ - field1:"val"^#3:*expr.Constant_StringValue#^#2:*expr.Expr_CreateStruct_Entry# - }^#2:*expr.Expr_StructExpr#`, - }, - - // Macros - { - I: `has(a.b)`, - P: `a^#2:*expr.Expr_IdentExpr#.b~test-only~^#4:has#`, - }, - { - I: `[1, 2, 3].all(x, x > 0)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - true^#10:*expr.Constant_BoolValue#, - // LoopCondition - @not_strictly_false( - @result^#11:*expr.Expr_IdentExpr# - )^#12:*expr.Expr_CallExpr#, - // LoopStep - _&&_( - @result^#13:*expr.Expr_IdentExpr#, - _>_( - x^#7:*expr.Expr_IdentExpr#, - 0^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr# - )^#14:*expr.Expr_CallExpr#, - // Result - @result^#15:*expr.Expr_IdentExpr#)^#16:all#`, - }, - { - I: `[1, 2, 3].exists(x, x == 2)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - false^#10:*expr.Constant_BoolValue#, - // LoopCondition - @not_strictly_false( - !_( - @result^#11:*expr.Expr_IdentExpr# - )^#12:*expr.Expr_CallExpr# - )^#13:*expr.Expr_CallExpr#, - // LoopStep - _||_( - @result^#14:*expr.Expr_IdentExpr#, - _==_( - x^#7:*expr.Expr_IdentExpr#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr# - )^#15:*expr.Expr_CallExpr#, - // Result - @result^#16:*expr.Expr_IdentExpr#)^#17:exists#`, - }, - { - I: `[1, 2, 3].exists_one(x, x == 2)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - 0^#10:*expr.Constant_Int64Value#, - // LoopCondition - true^#11:*expr.Constant_BoolValue#, - // LoopStep - _?_:_( - _==_( - x^#7:*expr.Expr_IdentExpr#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr#, - _+_( - @result^#12:*expr.Expr_IdentExpr#, - 1^#13:*expr.Constant_Int64Value# - )^#14:*expr.Expr_CallExpr#, - @result^#15:*expr.Expr_IdentExpr# - )^#16:*expr.Expr_CallExpr#, - // Result - _==_( - @result^#17:*expr.Expr_IdentExpr#, - 1^#18:*expr.Constant_Int64Value# - )^#19:*expr.Expr_CallExpr#)^#20:exists_one#`, - }, - { - I: `[1, 2, 3].map(x, x * 2)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - []^#10:*expr.Expr_ListExpr#, - // LoopCondition - true^#11:*expr.Constant_BoolValue#, - // LoopStep - _+_( - @result^#12:*expr.Expr_IdentExpr#, - [ - _*_( - x^#7:*expr.Expr_IdentExpr#, - 2^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr# - ]^#13:*expr.Expr_ListExpr# - )^#14:*expr.Expr_CallExpr#, - // Result - @result^#15:*expr.Expr_IdentExpr#)^#16:map#`, - }, - { - I: `[1, 2, 3].filter(x, x > 1)`, - P: `__comprehension__( - // Variable - x, - // Target - [ - 1^#2:*expr.Constant_Int64Value#, - 2^#3:*expr.Constant_Int64Value#, - 3^#4:*expr.Constant_Int64Value# - ]^#1:*expr.Expr_ListExpr#, - // Accumulator - @result, - // Init - []^#10:*expr.Expr_ListExpr#, - // LoopCondition - true^#11:*expr.Constant_BoolValue#, - // LoopStep - _?_:_( - _>_( - x^#7:*expr.Expr_IdentExpr#, - 1^#9:*expr.Constant_Int64Value# - )^#8:*expr.Expr_CallExpr#, - _+_( - @result^#12:*expr.Expr_IdentExpr#, - [ - x^#6:*expr.Expr_IdentExpr# - ]^#13:*expr.Expr_ListExpr# - )^#14:*expr.Expr_CallExpr#, - @result^#15:*expr.Expr_IdentExpr# - )^#16:*expr.Expr_CallExpr#, - // Result - @result^#17:*expr.Expr_IdentExpr#)^#18:filter#`, - }, - - // Optional Syntax - { - I: `a.?b`, - P: `_?._( - a^#1:*expr.Expr_IdentExpr#, - "b"^#3:*expr.Constant_StringValue# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `a[?0]`, - P: `_[?_]( - a^#1:*expr.Expr_IdentExpr#, - 0^#3:*expr.Constant_Int64Value# - )^#2:*expr.Expr_CallExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `[?a, b, ?c]`, - P: `[ - a^#2:*expr.Expr_IdentExpr#, - b^#3:*expr.Expr_IdentExpr#, - c^#4:*expr.Expr_IdentExpr# - ]^#1:*expr.Expr_ListExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `{?a: 1, b: 2}`, - P: `{ - ?a^#2:*expr.Expr_IdentExpr#:1^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - b^#5:*expr.Expr_IdentExpr#:2^#7:*expr.Constant_Int64Value#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - { - I: `pkg.Msg{?field: 42}`, - P: `pkg.Msg{ - ?field:42^#3:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry# - }^#2:*expr.Expr_StructExpr#`, - Opts: []Option{EnableOptionalSyntax(true)}, - }, - - // Escaped Identifier Syntax - { - I: "msg.`field-name`", - P: `msg^#1:*expr.Expr_IdentExpr#.field-name^#2:*expr.Expr_SelectExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "msg.`a.b.c`", - P: `msg^#1:*expr.Expr_IdentExpr#.a.b.c^#2:*expr.Expr_SelectExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "msg.`field name`", - P: `msg^#1:*expr.Expr_IdentExpr#.field name^#2:*expr.Expr_SelectExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "Msg{`field-1`: 42}", - P: `Msg{ - field-1:42^#2:*expr.Constant_Int64Value#^#1:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - - // Error Cases - { - I: "(1 + 2", - E: "ERROR: :1:7: expected ')'\n" + - " | (1 + 2\n" + - " | ......^", - }, - { - I: "[1, 2", - E: "ERROR: :1:6: expected ']'\n" + - " | [1, 2\n" + - " | .....^", - }, - { - I: "{1: 2", - E: "ERROR: :1:6: expected '}'\n" + - " | {1: 2\n" + - " | .....^", - }, - { - I: "a ? b", - E: "ERROR: :1:6: expected ':' in conditional expression\n" + - " | a ? b\n" + - " | .....^", - }, - { - I: "1 + 2 3", - E: "ERROR: :1:7: Syntax error: mismatched input '3' expecting \n" + - " | 1 + 2 3\n" + - " | ......^", - }, - { - I: "0xFFFFFFFFFFFFFFFFF", - E: "ERROR: :1:1: invalid int literal\n" + - " | 0xFFFFFFFFFFFFFFFFF\n" + - " | ^", - }, - { - I: "0xFFFFFFFFFFFFFFFFFu", - E: "ERROR: :1:1: invalid uint literal\n" + - " | 0xFFFFFFFFFFFFFFFFFu\n" + - " | ^", - }, - { - I: "1.99e90000009", - E: "ERROR: :1:1: invalid double literal\n" + - " | 1.99e90000009\n" + - " | ^", - }, - { - I: "as", - E: "ERROR: :1:1: reserved identifier: as\n" + - " | as\n" + - " | ^", - }, - { - I: "msg.`ident` + msg.`other`", - E: "ERROR: :1:5: unsupported syntax '`'\n" + - " | msg.`ident` + msg.`other`\n" + - " | ....^\n" + - "ERROR: :1:19: unsupported syntax '`'\n" + - " | msg.`ident` + msg.`other`\n" + - " | ..................^", - Opts: []Option{EnableIdentEscapeSyntax(false)}, - }, - { - I: "a.?b", - E: "ERROR: :1:2: unsupported syntax '.?'\n" + - " | a.?b\n" + - " | .^", - }, - { - I: "has(m)", - E: "ERROR: :1:5: invalid argument to has() macro\n" + - " | has(m)\n" + - " | ....^", - Opts: []Option{Macros(AllMacros...)}, - }, - { - I: "[1, 2].all(1 + 2, true)", - E: "ERROR: :1:14: argument must be a simple name\n" + - " | [1, 2].all(1 + 2, true)\n" + - " | .............^", - Opts: []Option{Macros(AllMacros...)}, - }, - { - I: "[1, 2].all(__result__, true)", - E: "ERROR: :1:12: iteration variable overwrites accumulator variable\n" + - " | [1, 2].all(__result__, true)\n" + - " | ...........^", - Opts: []Option{Macros(AllMacros...)}, - }, - { - I: "1{}", - E: "ERROR: :1:2: Syntax error: mismatched input '{' expecting \n" + - " | 1{}\n" + - " | .^", - }, - { - I: "a.", - E: "ERROR: :1:3: expected identifier after '.'\n" + - " | a.\n" + - " | ..^", - }, - { - I: ". *", - E: "ERROR: :1:3: expected identifier\n" + - " | . *\n" + - " | ..^", - }, - { - I: ".as", - E: "ERROR: :1:2: reserved identifier: as\n" + - " | .as\n" + - " | .^", - }, - { - I: "* 2", - E: "ERROR: :1:1: unexpected token\n" + - " | * 2\n" + - " | ^\n" + - "ERROR: :1:3: Syntax error: mismatched input '2' expecting \n" + - " | * 2\n" + - " | ..^", - }, - { - I: "{'k' 'v'}", - E: "ERROR: :1:6: expected ':' in map entry\n" + - " | {'k' 'v'}\n" + - " | .....^", - }, - { - I: "Msg{1: 2}", - E: "ERROR: :1:5: expected struct field name\n" + - " | Msg{1: 2}\n" + - " | ....^", - }, - { - I: "Msg{f 10}", - E: "ERROR: :1:7: expected ':' in struct field\n" + - " | Msg{f 10}\n" + - " | ......^", - }, - { - I: "Msg{f: 10", - E: "ERROR: :1:10: expected '}'\n" + - " | Msg{f: 10\n" + - " | .........^", - }, - { - I: "f(1, 2", - E: "ERROR: :1:7: Syntax error: mismatched input expecting ')'\n" + - " | f(1, 2\n" + - " | ......^", - }, - { - I: "1e", - E: "ERROR: :1:1: floating point literal missing digits after exponent separator\n" + - " | 1e\n" + - " | ^", - }, - { - I: "\"unterminated", - E: "ERROR: :1:1: unterminated string literal\n" + - " | \"unterminated\n" + - " | ^", - }, - { - I: "b\"unterminated", - E: "ERROR: :1:1: unterminated bytes literal\n" + - " | b\"unterminated\n" + - " | ^", - }, - { - I: "a.`foo`()", - E: "ERROR: :1:3: unexpected quoted identifier\n" + - " | a.`foo`()\n" + - " | ..^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "`foo`", - E: "ERROR: :1:1: unexpected quoted identifier\n" + - " | `foo`\n" + - " | ^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "a.`b@c`", - E: "ERROR: :1:3: unexpected quoted identifier\n" + - " | a.`b@c`\n" + - " | ..^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "a.``", - E: "ERROR: :1:3: unexpected quoted identifier\n" + - " | a.``\n" + - " | ..^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "`foo", - E: "ERROR: :1:1: unterminated quoted identifier\n" + - " | `foo\n" + - " | ^", - Opts: []Option{EnableIdentEscapeSyntax(true)}, - }, - { - I: "-0x8000000000000001", - E: "ERROR: :1:2: invalid int literal\n" + - " | -0x8000000000000001\n" + - " | .^", - }, - { - I: "-9223372036854775809", - E: "ERROR: :1:2: invalid int literal\n" + - " | -9223372036854775809\n" + - " | .^", - }, - { - I: "a[?0]", - E: "ERROR: :1:2: unsupported syntax '?'\n" + - " | a[?0]\n" + - " | .^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "[?1]", - E: "ERROR: :1:2: unsupported syntax '?'\n" + - " | [?1]\n" + - " | .^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "{?'k': 'v'}", - E: "ERROR: :1:2: unsupported syntax '?'\n" + - " | {?'k': 'v'}\n" + - " | .^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "Msg{?f: 1}", - E: "ERROR: :1:5: unsupported syntax '?'\n" + - " | Msg{?f: 1}\n" + - " | ....^", - Opts: []Option{EnableOptionalSyntax(false)}, - }, - { - I: "a.?`foo`", - E: "ERROR: :1:4: unsupported syntax '`'\n" + - " | a.?`foo`\n" + - " | ...^", - Opts: []Option{EnableOptionalSyntax(true), EnableIdentEscapeSyntax(false)}, - }, -} - -func parse(source common.Source, opts ...Option) (*ast.AST, *common.Errors) { - p, err := NewPrattParser(opts...) - if err != nil { - panic(err) - } - return p.Parse(source) -} - -func TestPrattParser(t *testing.T) { - for i, tst := range prattTestCases { - name := fmt.Sprintf("%d %s", i, tst.I) - // Local variable required as the closure will reference the value for the last - // 'tst' value rather than the local 'tc' instance declared within the loop. - tc := tst - t.Run(name, func(t *testing.T) { - t.Parallel() - opts := tc.Opts - if len(opts) == 0 { - opts = []Option{Macros(AllMacros...), PopulateMacroCalls(true)} - } - src := common.NewTextSource(tc.I) - parsed, errors := parse(src, opts...) - if len(errors.GetErrors()) > 0 { - actualErr := errors.ToDisplayString() - if tc.E == "" { - t.Fatalf("Unexpected errors: %v", actualErr) - } else if !test.Compare(actualErr, tc.E) { - t.Fatal(test.DiffMessage("Error mismatch", actualErr, tc.E)) - } - return - } else if tc.E != "" { - t.Fatalf("Expected error not thrown: '%s'", tc.E) - } - failureDisplayMethod := fmt.Sprintf("Parse(\"%s\")", tc.I) - actualWithKind := debug.ToAdornedDebugString(parsed.Expr(), &kindAndIDAdorner{parsed.SourceInfo()}) - if !test.Compare(actualWithKind, tc.P) { - t.Fatal(test.DiffMessage(fmt.Sprintf("Structure - %s", failureDisplayMethod), actualWithKind, tc.P)) - } - }) - } -} -func TestPrattParserSourceInfoPositions(t *testing.T) { - src := common.NewTextSource("a + b") - p, err := NewPrattParser() - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(src) - if len(errs.GetErrors()) > 0 { - t.Fatalf("Parse() failed: %s", errs.ToDisplayString()) - } - sourceInfo := parsed.SourceInfo() - root := parsed.Expr() - if sourceInfo.GetStartLocation(root.ID()).Column() != 2 { - t.Errorf("expected root column 2, got %d", sourceInfo.GetStartLocation(root.ID()).Column()) - } - args := root.AsCall().Args() - if len(args) != 2 { - t.Fatalf("expected 2 args, got %d", len(args)) - } - if sourceInfo.GetStartLocation(args[0].ID()).Column() != 0 { - t.Errorf("expected arg[0] column 0, got %d", sourceInfo.GetStartLocation(args[0].ID()).Column()) - } - if sourceInfo.GetStartLocation(args[1].ID()).Column() != 4 { - t.Errorf("expected arg[1] column 4, got %d", sourceInfo.GetStartLocation(args[1].ID()).Column()) - } -} - -func TestPrattParserRecursionDepth(t *testing.T) { - t.Run("DeeplyNestedBracketsLimitExceeded", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(5)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("[[[[[[1]]]]]]")) - if len(errs.GetErrors()) == 0 { - t.Errorf("expected recursion limit error, got none") - } - }) - - t.Run("IgnoreExtraParens", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(1)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("((((1))))")) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error: %s", errs.ToDisplayString()) - } - }) - - t.Run("DeeplyNestedParens1000", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(1)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - expr1 := strings.Repeat("(", 1000) + "42" + strings.Repeat(")", 1000) - _, errs := p.Parse(common.NewTextSource(expr1)) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error on 1000 parens literal: %s", errs.ToDisplayString()) - } - - expr2 := strings.Repeat("(", 1000) + "1 + 2" + strings.Repeat(")", 1000) - _, errs = p.Parse(common.NewTextSource(expr2)) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error on 1000 parens binary: %s", errs.ToDisplayString()) - } - }) - - t.Run("SequentialScopesDoNotAccumulateDepth", func(t *testing.T) { - p, err := NewPrattParser(MaxRecursionDepth(2)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("[1] + [2] + [3]")) - if len(errs.GetErrors()) > 0 { - t.Errorf("unexpected error on sequential scopes: %s", errs.ToDisplayString()) - } - }) -} - -func TestPrattParserMacroCalls(t *testing.T) { - t.Run("DisabledByDefault", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(false)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - if len(parsed.SourceInfo().MacroCalls()) != 0 { - t.Errorf("expected 0 macro calls, got %d", len(parsed.SourceInfo().MacroCalls())) - } - }) - - t.Run("GlobalMacroCallRecorded", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("has(a.b)")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - macroCalls := parsed.SourceInfo().MacroCalls() - if len(macroCalls) != 1 { - t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) - } - }) - - t.Run("ReceiverMacroCallRecorded", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("[1, 2].exists(x, x > 0)")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - macroCalls := parsed.SourceInfo().MacroCalls() - if len(macroCalls) != 1 { - t.Fatalf("expected 1 macro call, got %d", len(macroCalls)) - } - }) - - t.Run("NestedMacroCallsRecorded", func(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - parsed, errs := p.Parse(common.NewTextSource("[1, 2].all(x, has(x.b))")) - if len(errs.GetErrors()) > 0 { - t.Fatalf("unexpected error: %s", errs.ToDisplayString()) - } - macroCalls := parsed.SourceInfo().MacroCalls() - if len(macroCalls) != 2 { - t.Fatalf("expected 2 macro calls, got %d", len(macroCalls)) - } - }) -} - -func TestPrattParserErrorRecoveryLimits(t *testing.T) { - t.Run("LimitZero", func(t *testing.T) { - p, err := NewPrattParser(ErrorRecoveryLimit(0)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("......")) - if len(errs.GetErrors()) == 0 { - t.Errorf("expected error recovery limit error, got none") - } - }) - - t.Run("LimitOne", func(t *testing.T) { - p, err := NewPrattParser(ErrorRecoveryLimit(1)) - if err != nil { - t.Fatalf("NewPrattParser() failed: %v", err) - } - _, errs := p.Parse(common.NewTextSource("......")) - if len(errs.GetErrors()) == 0 { - t.Errorf("expected error recovery limit error, got none") - } - }) -} - -func TestPrattParserExpressionSizeCodePointLimit(t *testing.T) { - p, err := NewPrattParser(Macros(AllMacros...), ExpressionSizeCodePointLimit(2)) - if err != nil { - t.Fatal(err) - } - src := common.NewTextSource("foo") - _, errs := p.Parse(src) - if got, want := len(errs.GetErrors()), 1; got != want { - t.Fatalf("got %d errors, want %d errors: %s", got, want, errs.ToDisplayString()) - } - if got, want := errs.GetErrors()[0].Message, "expression code point size exceeds limit: size: 3, limit 2"; got != want { - t.Fatalf("got %q, want %q: %s", got, want, errs.GetErrors()[0].ToDisplayString(src)) - } -} - -func BenchmarkParsers(b *testing.B) { - exprs := []string{ - `42`, - `a > 5 && b < 10 || c == "xyz"`, - `[1, 2, 3].all(x, x > 0) && [4, 5, 6].exists(y, y == 5)`, - `pkg.Msg{field1: "value", field2: 123, list_field: [1, 2, 3], map_field: {"a": true, "b": false}}`, - `a.b.c.d.e.f(1, 2, [3, ?4], {?5: 6}) ? (x + y * z - w / v) : (!p && !q || r.s)`, - } - - antlrParser, _ := NewParser(Macros(AllMacros...), PopulateMacroCalls(true), EnableOptionalSyntax(true)) - prattParser, _ := NewPrattParser(Macros(AllMacros...), PopulateMacroCalls(true), EnableOptionalSyntax(true)) - - for _, expr := range exprs { - src := common.NewTextSource(expr) - - b.Run("ANTLR/"+expr[:min(len(expr), 20)], func(b *testing.B) { - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = antlrParser.Parse(src) - } - }) - - b.Run("Pratt/"+expr[:min(len(expr), 20)], func(b *testing.B) { - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = prattParser.Parse(src) - } - }) - } -} - -func min(a, b int) int { - if a < b { - return a - } - return b -}