Optimized compilation of string patterns

hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala

@@ -332,6 +332,7 @@ object Elaborator:
       loopEnd: ModuleOrObjectSymbol,
       tuple: ModuleOrObjectSymbol,
       str: ModuleOrObjectSymbol,
+      strPat: ModuleOrObjectSymbol,
       unreachable: TermSymbol,
       tupleGet: TermSymbol,
       tupleSlice: TermSymbol,
@@ -340,6 +341,9 @@ object Elaborator:
       strGet: TermSymbol,
       strTake: TermSymbol,
       strLeave: TermSymbol,
+      strPatMatchWhole: TermSymbol,
+      strPatParseWhole: TermSymbol,
+      strPatParsePrefix: TermSymbol,
       matchSuccessCls: ClassSymbol,
       matchSuccessTrm: TermSymbol,
       matchFailureCls: ClassSymbol,
@@ -375,11 +379,13 @@ object Elaborator:
 
       val tuple = modOrObj("Tuple")
       val str = modOrObj("Str")
+      val strPat = modOrObj("StrPat")
       RuntimeSymbols(
         unit = modOrObj("Unit"),
         loopEnd = modOrObj("LoopEnd"),
         tuple = tuple,
         str = str,
+        strPat = strPat,
         unreachable = term("unreachable"),
         tupleGet = moduleMember(tuple, "get"),
         tupleSlice = moduleMember(tuple, "slice"),
@@ -388,6 +394,9 @@ object Elaborator:
         strGet = moduleMember(str, "get"),
         strTake = moduleMember(str, "take"),
         strLeave = moduleMember(str, "leave"),
+        strPatMatchWhole = moduleMember(strPat, "matchWhole"),
+        strPatParseWhole = moduleMember(strPat, "parseWhole"),
+        strPatParsePrefix = moduleMember(strPat, "parsePrefix"),
         matchSuccessCls = cls("MatchSuccess"),
         matchSuccessTrm = term("MatchSuccess"),
         matchFailureCls = cls("MatchFailure"),
@@ -441,6 +450,10 @@ object Elaborator:
     def strGetSymbol: TermSymbol = runtimeSymbols.strGet
     def strTakeSymbol: TermSymbol = runtimeSymbols.strTake
     def strLeaveSymbol: TermSymbol = runtimeSymbols.strLeave
+    def strPatSymbol: ModuleOrObjectSymbol = runtimeSymbols.strPat
+    def strPatMatchWholeSymbol: TermSymbol = runtimeSymbols.strPatMatchWhole
+    def strPatParseWholeSymbol: TermSymbol = runtimeSymbols.strPatParseWhole
+    def strPatParsePrefixSymbol: TermSymbol = runtimeSymbols.strPatParsePrefix
     def matchSuccessClsSymbol: ClassSymbol = runtimeSymbols.matchSuccessCls
     def matchSuccessTrmSymbol: TermSymbol = runtimeSymbols.matchSuccessTrm
     def matchFailureClsSymbol: ClassSymbol = runtimeSymbols.matchFailureCls
@@ -2347,9 +2360,13 @@ extends Importer:
     /** String range bounds must be single characters. */
     def isInvalidStringBounds(lo: StrLit, hi: StrLit)(using Raise): Bool =
       val ds = collection.mutable.Buffer.empty[(Message, Option[Loc])]
-      if lo.value.length =/= 1 then
+      // A bound is a single character when it is one code point; astral
+      // characters (two UTF-16 code units) are accepted.
+      def isSingleCharacter(s: Str): Bool =
+        s.nonEmpty && s.codePointCount(0, s.length) == 1
+      if !isSingleCharacter(lo.value) then
         ds += msg"The lower bound of character ranges must be a single character." -> lo.toLoc
-      if hi.value.length =/= 1 then
+      if !isSingleCharacter(hi.value) then
         ds += msg"The upper bound of character ranges must be a single character." -> hi.toLoc
       if ds.nonEmpty then error(ds.toSeq*)
       ds.nonEmpty

hkmc2/shared/src/main/scala/hkmc2/semantics/ucs/TermSynthesizer.scala

@@ -63,6 +63,9 @@ trait TermSynthesizer(using State):
   protected lazy val stringGet = sel(sel(runtimeRef, "Str", State.strSymbol), "get", State.strGetSymbol)
   protected lazy val stringTake = sel(sel(runtimeRef, "Str", State.strSymbol), "take", State.strTakeSymbol)
   protected lazy val stringLeave = sel(sel(runtimeRef, "Str", State.strSymbol), "leave", State.strLeaveSymbol)
+  protected lazy val strPatMatchWhole = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "matchWhole", State.strPatMatchWholeSymbol)
+  protected lazy val strPatParseWhole = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "parseWhole", State.strPatParseWholeSymbol)
+  protected lazy val strPatParsePrefix = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "parsePrefix", State.strPatParsePrefixSymbol)
 
   /** Make a term that looks like `runtime.Tuple.get(t, i)`. */
   protected final def callTupleGet(t: Term, i: Int, label: Str): Term =
@@ -92,6 +95,17 @@ trait TermSynthesizer(using State):
   protected final def callStringDrop(t: Term.Ref, n: Int, label: Str) =
     app(stringLeave, tup(fld(t), fld(int(n))), label)
 
+  /** Aggregate the transform closures of a compiled string pattern into a
+    * tuple for the matching engine. The closures may originate from different
+    * source blocks, so the tuple's location must be pinned explicitly:
+    * deriving it from the children would mix origins and trip the location
+    * distinctness assertion in `AutoLocated`. */
+  protected final def actionsTuple(actions: Ls[Term], siteLoc: Opt[Loc]): Term =
+    val tuple = tup(actions)
+    siteLoc.orElse(actions.iterator.flatMap(_.toLoc.iterator).nextOption()) match
+      case loc @ S(_) => tuple.withLoc(loc)
+      case N => tuple // No sub-locations at all: nothing to mix.
+
   protected final def tempLet(dbgName: Str, term: Term)(inner: TempSymbol => Split): Split =
     val s = TempSymbol(N, dbgName)
     Split.Let(s, term, inner(s))

hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Compiler.scala

@@ -23,6 +23,12 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
   import Compiler.*, tl.*
   import Pattern.*
 
+  /** The location of the match site this compiler serves, used only to label
+    * `StringCompiler.TableStats` measurement records. Expanded patterns
+    * aggregate sub-trees from several blocks, so their own locations are not
+    * usable for this purpose. */
+  var statsSiteLoc: Opt[Loc] = N
+
   /** A previously-computed matcher result for one field of the current
     * multi-matcher. In full mode the value also carries the original field
     * input, which is needed when a successful field pattern preserves its
@@ -150,7 +156,20 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
       .mkString("{", ", ", "}")}"
   ):
     val expandedPatterns = patterns.map(p => (p.label, p.expand(Set.empty)))
-    val heads = expandedPatterns.flatMap((_, p) => p.heads).toList
+    // String-shaped patterns (sequences and character classes) cannot take
+    // part in head-based specialization: how a string scrutinee is split is a
+    // decision global to the whole sequence. Whenever one is present, all
+    // string-shaped patterns — including plain string literals, whose heads
+    // would overlap the `Str` class — are absorbed into a single `Str` head
+    // whose branch runs one compiled automaton per label, like a lexer
+    // jointly matching several token rules (see `StringCompiler`).
+    val absorbStrings = expandedPatterns.exists((_, p) => StringCompiler.containsStringNode(p))
+    val strSymbol = ctx.builtins.Str
+    val heads = expandedPatterns.flatMap((_, p) => p.heads).toList.filter: head =>
+      !absorbStrings || (head match
+        case _: StrLit => false
+        case symbol: ClassLikeSymbol if symbol is strSymbol => false
+        case _ => true)
     // This is the parameter of the current multi-matcher.
     val scrutinee = VarSymbol(Ident("input"))
     // Assemble branches for constructors and literals.
@@ -170,12 +189,16 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
         case _: (syntax.Literal | ModuleOrObjectSymbol) => empty
       val consequent = Split.Else(multiMatcherBranch(specialized, scrutinee, classFields))
       Branch(scrutinee.safeRef, head.toFlatPattern(classFieldArguments), consequent)
+    val stringBranch = if !absorbStrings then N else
+      val pattern = FlatPattern.ClassLike(strSymbol.safeRef, strSymbol, N, false)(Tree.Dummy)
+      val consequent = Split.Else(multiMatcherStringBranch(expandedPatterns, scrutinee))
+      S(Branch(scrutinee.safeRef, pattern, consequent))
     // Assemble the default branch.
     val default =
       val specialized = expandedPatterns.specializeSet(N)
       Split.Else(multiMatcherBranch(specialized, scrutinee, Map.empty))
     // Make a split that tries all branches in order.
-    val topmostSplit = branches.foldRight(default)(_ ~: _)
+    val topmostSplit = (branches ::: stringBranch.toList).foldRight(default)(_ ~: _)
     val bodyTerm = SynthIf(topmostSplit)
     log(s"Multi-matcher body:\n${topmostSplit.prettyPrint}")
     (paramList(param(scrutinee)), bodyTerm)
@@ -244,6 +267,98 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
     // and as a record otherwise.
     Blk(bindings ::: tests.reverse, resultTerm)
 
+  /** The branch body for the absorbed `Str` head: each label's string-shaped
+    * fragment is compiled to its own whole-match automaton (see the note in
+    * `buildMultiMatcherBody`). The per-label result terms follow the same
+    * protocol as `multiMatcherBranch`: a Boolean in match-only mode, and a
+    * `MatchSuccess`/`MatchFailure` value in full mode.
+    */
+  def multiMatcherStringBranch(
+      patterns: Set[(Label, ExPat)],
+      scrutinee: VarSymbol,
+  )(using ResultMode): Blk =
+    val z = (Nil: Ls[Statement], Nil: Ls[(Label, Term)])
+    val (tests, resultTerms) = patterns.iterator.foldLeft(z):
+      case ((stmts, results), (label, pattern)) =>
+        val fragment = StringCompiler.stringFragment(pattern).simplify
+        val resultTerm = fragment match
+          case Or(Nil) =>
+            // This label has no string-shaped alternative: it cannot match.
+            emptyMatchResult("not a string pattern")
+          // Note that each label needs its own compiler: a compiler instance
+          // accumulates the automaton states (and failure flag) of a single
+          // region.
+          case fragment => StringCompiler().compile(fragment, StringCompiler.Mode.Whole) match
+            case N => emptyMatchResult("rejected string pattern")
+            case S(compiled) =>
+              // Mirrors the branch chosen below: a match-only region still
+              // needs the parse table when it carries transforms or bindings.
+              val embedsMatchTable =
+                if isMatchOnly then compiled.recognitionSuffices(false) else compiled.pure
+              StringCompiler.TableStats.record(statsSiteLoc,
+                if embedsMatchTable then "mm-match" else "mm-parse",
+                if embedsMatchTable then compiled.matchTable else compiled.table)
+              val matchTableTerm = str(compiled.matchTable)
+              if isMatchOnly && compiled.recognitionSuffices(false) then
+                app(strPatMatchWhole, tup(fld(matchTableTerm), fld(scrutinee.safeRef)), "string match")
+              else if isMatchOnly then
+                // The region carries transforms (or bindings): even a
+                // condition-position match must run them, exactly once, on
+                // the committed parse. Only the success of the parse is
+                // observed.
+                val call = app(strPatParseWhole,
+                  tup(fld(str(compiled.table)), fld(actionsTuple(compiled.actions, N)), fld(scrutinee.safeRef)),
+                  "string parse")
+                val resultSymbol = TempSymbol(N, "parseResult")
+                SynthIf(Split.Let(resultSymbol, call,
+                  Branch(
+                    resultSymbol.safeRef,
+                    // The engine returns null on failure and an array on success.
+                    FlatPattern.Tuple(1, true),
+                    Split.Else(bool(true))
+                  ) ~: Split.Else(bool(false))))
+              else if compiled.pure then
+                // An operation-free whole match preserves the scrutinee.
+                val matchedSymbol = TempSymbol(N, "stringMatched")
+                val call = app(strPatMatchWhole, tup(fld(matchTableTerm), fld(scrutinee.safeRef)), "string match")
+                SynthIf(Split.Let(matchedSymbol, call,
+                  Branch(matchedSymbol.safeRef,
+                    Split.Else(makeMatchSuccess(scrutinee.safeRef))
+                  ) ~: Split.Else(emptyMatchResult("string mismatch"))))
+              else
+                // Note: expanded patterns may aggregate sub-patterns from
+                // several source blocks, so their auto-computed location is
+                // not usable here; the helper pins the first action's own
+                // location instead (the surrounding terms are location-free).
+                val call = app(strPatParseWhole,
+                  tup(fld(str(compiled.table)), fld(actionsTuple(compiled.actions, N)), fld(scrutinee.safeRef)),
+                  "string parse")
+                val resultSymbol = TempSymbol(N, "parseResult")
+                val outputSymbol = TempSymbol(N, "stringOutput")
+                val slotSymbols = compiled.visibleSlots.map: (symbol, slot) =>
+                  (symbol, slot, TempSymbol(N, s"${symbol.name}$$"))
+                val bindingsTerm = makeBindings(slotSymbols.map:
+                  (symbol, _, local) => RcdField(str(symbol.name), local.safeRef))
+                val success = slotSymbols.foldRight(
+                  Split.Else(makeMatchSuccess(outputSymbol.safeRef, bindingsTerm)): Split
+                ):
+                  case ((_, slot, local), inner) =>
+                    Split.Let(local, callTupleGet(resultSymbol.safeRef, 1 + slot, "string binding"), inner)
+                SynthIf(Split.Let(resultSymbol, call,
+                  Branch(
+                    resultSymbol.safeRef,
+                    // The engine returns null on failure and an array on success.
+                    FlatPattern.Tuple(1, true),
+                    Split.Let(outputSymbol, callTupleGet(resultSymbol.safeRef, 0, "string output"), success)
+                  ) ~: Split.Else(emptyMatchResult("string mismatch"))))
+        val symbol = TempSymbol(N, label.asFieldName + "$")
+        (DefineVar(symbol, resultTerm) :: LetDecl(symbol, Nil) :: stmts, (label, symbol.safeRef) :: results)
+    val resultTerm = resultTerms.reverse match
+      case (_, term) :: Nil => term
+      case terms => Rcd(false, terms.map: (label, term) =>
+        RcdField(str(label.asFieldName), term))
+    Blk(tests.reverse, resultTerm)
+
   import Pattern.*
 
   /** Represent things that can be used as expressions in consequents. */
@@ -525,8 +640,10 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
         val params = paramList(param(bindingsSymbol))
         // Because we pass the extracted values using recoreds. We need to bind
         // each property to its corresponding variable which is accessible from
-        // then `term`.
-        val letBindings = pattern.symbols.flatMap: symbol =>
+        // then `term`. Only the symbols the definition itself binds are
+        // mapped by `correspondence` (and referenced by `term`); symbols
+        // bound inside substituted pattern arguments are not.
+        val letBindings = pattern.symbols.filter(correspondence.contains).flatMap: symbol =>
           val termSymbol = correspondence(symbol)
           LetDecl(termSymbol, Nil) ::
           DefineVar(termSymbol, sel(bindingsSymbol.safeRef, termSymbol.name)) :: Nil

hkmc2/shared/src/main/scala/hkmc2/semantics/ups/FixedPointCompiler.scala

@@ -335,7 +335,8 @@ class FixedPointCompiler(using tl: TL)(using State, Ctx, Raise) extends TermSynt
     case Not(pattern) => mentions(pattern, target)
     case Rename(pattern, _) => mentions(pattern, target)
     case Extract(pattern, _, _) => mentions(pattern, target)
-    case Literal(_) => false
+    case Concat(patterns) => patterns.exists(mentions(_, target))
+    case Literal(_) | CharClass(_, _) => false
 
   /** Instantiate the pattern groups with a shared `Instantiator`,
     * monomorphizing higher-order patterns such as `Ctx(Redex)` into