Optimized compilation of string patterns

hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala

@@ -332,6 +332,7 @@ object Elaborator:
       loopEnd: ModuleOrObjectSymbol,
       tuple: ModuleOrObjectSymbol,
       str: ModuleOrObjectSymbol,
+      strPat: ModuleOrObjectSymbol,
       unreachable: TermSymbol,
       tupleGet: TermSymbol,
       tupleSlice: TermSymbol,
@@ -340,6 +341,9 @@ object Elaborator:
       strGet: TermSymbol,
       strTake: TermSymbol,
       strLeave: TermSymbol,
+      strPatMatchWhole: TermSymbol,
+      strPatParseWhole: TermSymbol,
+      strPatParsePrefix: TermSymbol,
       matchSuccessCls: ClassSymbol,
       matchSuccessTrm: TermSymbol,
       matchFailureCls: ClassSymbol,
@@ -375,11 +379,13 @@ object Elaborator:
 
       val tuple = modOrObj("Tuple")
       val str = modOrObj("Str")
+      val strPat = modOrObj("StrPat")
       RuntimeSymbols(
         unit = modOrObj("Unit"),
         loopEnd = modOrObj("LoopEnd"),
         tuple = tuple,
         str = str,
+        strPat = strPat,
         unreachable = term("unreachable"),
         tupleGet = moduleMember(tuple, "get"),
         tupleSlice = moduleMember(tuple, "slice"),
@@ -388,6 +394,9 @@ object Elaborator:
         strGet = moduleMember(str, "get"),
         strTake = moduleMember(str, "take"),
         strLeave = moduleMember(str, "leave"),
+        strPatMatchWhole = moduleMember(strPat, "matchWhole"),
+        strPatParseWhole = moduleMember(strPat, "parseWhole"),
+        strPatParsePrefix = moduleMember(strPat, "parsePrefix"),
         matchSuccessCls = cls("MatchSuccess"),
         matchSuccessTrm = term("MatchSuccess"),
         matchFailureCls = cls("MatchFailure"),
@@ -441,6 +450,10 @@ object Elaborator:
     def strGetSymbol: TermSymbol = runtimeSymbols.strGet
     def strTakeSymbol: TermSymbol = runtimeSymbols.strTake
     def strLeaveSymbol: TermSymbol = runtimeSymbols.strLeave
+    def strPatSymbol: ModuleOrObjectSymbol = runtimeSymbols.strPat
+    def strPatMatchWholeSymbol: TermSymbol = runtimeSymbols.strPatMatchWhole
+    def strPatParseWholeSymbol: TermSymbol = runtimeSymbols.strPatParseWhole
+    def strPatParsePrefixSymbol: TermSymbol = runtimeSymbols.strPatParsePrefix
     def matchSuccessClsSymbol: ClassSymbol = runtimeSymbols.matchSuccessCls
     def matchSuccessTrmSymbol: TermSymbol = runtimeSymbols.matchSuccessTrm
     def matchFailureClsSymbol: ClassSymbol = runtimeSymbols.matchFailureCls

hkmc2/shared/src/main/scala/hkmc2/semantics/ucs/TermSynthesizer.scala

@@ -63,6 +63,9 @@ trait TermSynthesizer(using State):
   protected lazy val stringGet = sel(sel(runtimeRef, "Str", State.strSymbol), "get", State.strGetSymbol)
   protected lazy val stringTake = sel(sel(runtimeRef, "Str", State.strSymbol), "take", State.strTakeSymbol)
   protected lazy val stringLeave = sel(sel(runtimeRef, "Str", State.strSymbol), "leave", State.strLeaveSymbol)
+  protected lazy val strPatMatchWhole = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "matchWhole", State.strPatMatchWholeSymbol)
+  protected lazy val strPatParseWhole = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "parseWhole", State.strPatParseWholeSymbol)
+  protected lazy val strPatParsePrefix = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "parsePrefix", State.strPatParsePrefixSymbol)
 
   /** Make a term that looks like `runtime.Tuple.get(t, i)`. */
   protected final def callTupleGet(t: Term, i: Int, label: Str): Term =
@@ -92,6 +95,17 @@ trait TermSynthesizer(using State):
   protected final def callStringDrop(t: Term.Ref, n: Int, label: Str) =
     app(stringLeave, tup(fld(t), fld(int(n))), label)
 
+  /** Aggregate the transform closures of a compiled string pattern into a
+    * tuple for the matching engine. The closures may originate from different
+    * source blocks, so the tuple's location must be pinned explicitly:
+    * deriving it from the children would mix origins and trip the location
+    * distinctness assertion in `AutoLocated`. */
+  protected final def actionsTuple(actions: Ls[Term], siteLoc: Opt[Loc]): Term =
+    val tuple = tup(actions)
+    siteLoc.orElse(actions.iterator.flatMap(_.toLoc.iterator).nextOption()) match
+      case loc @ S(_) => tuple.withLoc(loc)
+      case N => tuple // No sub-locations at all: nothing to mix.
+
   protected final def tempLet(dbgName: Str, term: Term)(inner: TempSymbol => Split): Split =
     val s = TempSymbol(N, dbgName)
     Split.Let(s, term, inner(s))

hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Compiler.scala

@@ -150,7 +150,20 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
       .mkString("{", ", ", "}")}"
   ):
     val expandedPatterns = patterns.map(p => (p.label, p.expand(Set.empty)))
-    val heads = expandedPatterns.flatMap((_, p) => p.heads).toList
+    // String-shaped patterns (sequences and character classes) cannot take
+    // part in head-based specialization: how a string scrutinee is split is a
+    // decision global to the whole sequence. Whenever one is present, all
+    // string-shaped patterns — including plain string literals, whose heads
+    // would overlap the `Str` class — are absorbed into a single `Str` head
+    // whose branch runs one compiled automaton per label, like a lexer
+    // jointly matching several token rules (see `StringCompiler`).
+    val absorbStrings = expandedPatterns.exists((_, p) => StringCompiler.containsStringNode(p))
+    val strSymbol = ctx.builtins.Str
+    val heads = expandedPatterns.flatMap((_, p) => p.heads).toList.filter: head =>
+      !absorbStrings || (head match
+        case _: StrLit => false
+        case symbol: ClassLikeSymbol if symbol is strSymbol => false
+        case _ => true)
     // This is the parameter of the current multi-matcher.
     val scrutinee = VarSymbol(Ident("input"))
     // Assemble branches for constructors and literals.
@@ -170,12 +183,16 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
         case _: (syntax.Literal | ModuleOrObjectSymbol) => empty
       val consequent = Split.Else(multiMatcherBranch(specialized, scrutinee, classFields))
       Branch(scrutinee.safeRef, head.toFlatPattern(classFieldArguments), consequent)
+    val stringBranch = if !absorbStrings then N else
+      val pattern = FlatPattern.ClassLike(strSymbol.safeRef, strSymbol, N, false)(Tree.Dummy)
+      val consequent = Split.Else(multiMatcherStringBranch(expandedPatterns, scrutinee))
+      S(Branch(scrutinee.safeRef, pattern, consequent))
     // Assemble the default branch.
     val default =
       val specialized = expandedPatterns.specializeSet(N)
       Split.Else(multiMatcherBranch(specialized, scrutinee, Map.empty))
     // Make a split that tries all branches in order.
-    val topmostSplit = branches.foldRight(default)(_ ~: _)
+    val topmostSplit = (branches ::: stringBranch.toList).foldRight(default)(_ ~: _)
     val bodyTerm = SynthIf(topmostSplit)
     log(s"Multi-matcher body:\n${topmostSplit.prettyPrint}")
     (paramList(param(scrutinee)), bodyTerm)
@@ -243,7 +260,76 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter
     // Lastly, we return the matcher result, directly for singleton matchers
     // and as a record otherwise.
     Blk(bindings ::: tests.reverse, resultTerm)
-
+
+  /** The branch body for the absorbed `Str` head: each label's string-shaped
+    * fragment is compiled to its own whole-match automaton (see the note in
+    * `buildMultiMatcherBody`). The per-label result terms follow the same
+    * protocol as `multiMatcherBranch`: a Boolean in match-only mode, and a
+    * `MatchSuccess`/`MatchFailure` value in full mode.
+    */
+  def multiMatcherStringBranch(
+      patterns: Set[(Label, ExPat)],
+      scrutinee: VarSymbol,
+  )(using ResultMode): Blk =
+    val z = (Nil: Ls[Statement], Nil: Ls[(Label, Term)])
+    val (tests, resultTerms) = patterns.iterator.foldLeft(z):
+      case ((stmts, results), (label, pattern)) =>
+        val fragment = StringCompiler.stringFragment(pattern).simplify
+        val resultTerm = fragment match
+          case And(Nil) =>
+            // This label has no string-shaped alternative: it cannot match.
+            emptyMatchResult("not a string pattern")
+          // Note that each label needs its own compiler: a compiler instance
+          // accumulates the automaton states (and failure flag) of a single
+          // region.
+          case fragment => StringCompiler().compile(fragment, StringCompiler.Mode.Whole) match
+            case N => emptyMatchResult("rejected string pattern")
+            case S(compiled) =>
+              val matchTableTerm = str(compiled.matchTable)
+              if isMatchOnly then
+                app(strPatMatchWhole, tup(fld(matchTableTerm), fld(scrutinee.safeRef)), "string match")
+              else if compiled.pure then
+                // An operation-free whole match preserves the scrutinee.
+                val matchedSymbol = TempSymbol(N, "stringMatched")
+                val call = app(strPatMatchWhole, tup(fld(matchTableTerm), fld(scrutinee.safeRef)), "string match")
+                SynthIf(Split.Let(matchedSymbol, call,
+                  Branch(matchedSymbol.safeRef,
+                    Split.Else(makeMatchSuccess(scrutinee.safeRef))
+                  ) ~: Split.Else(emptyMatchResult("string mismatch"))))
+              else
+                // Note: expanded patterns may aggregate sub-patterns from
+                // several source blocks, so their auto-computed location is
+                // not usable here; the helper pins the first action's own
+                // location instead (the surrounding terms are location-free).
+                val call = app(strPatParseWhole,
+                  tup(fld(str(compiled.table)), fld(actionsTuple(compiled.actions, N)), fld(scrutinee.safeRef)),
+                  "string parse")
+                val resultSymbol = TempSymbol(N, "parseResult")
+                val outputSymbol = TempSymbol(N, "stringOutput")
+                val slotSymbols = compiled.visibleSlots.map: (symbol, slot) =>
+                  (symbol, slot, TempSymbol(N, s"${symbol.name}$$"))
+                val bindingsTerm = makeBindings(slotSymbols.map:
+                  (symbol, _, local) => RcdField(str(symbol.name), local.safeRef))
+                val success = slotSymbols.foldRight(
+                  Split.Else(makeMatchSuccess(outputSymbol.safeRef, bindingsTerm)): Split
+                ):
+                  case ((_, slot, local), inner) =>
+                    Split.Let(local, callTupleGet(resultSymbol.safeRef, 1 + slot, "string binding"), inner)
+                SynthIf(Split.Let(resultSymbol, call,
+                  Branch(
+                    resultSymbol.safeRef,
+                    // The engine returns null on failure and an array on success.
+                    FlatPattern.Tuple(1, true),
+                    Split.Let(outputSymbol, callTupleGet(resultSymbol.safeRef, 0, "string output"), success)
+                  ) ~: Split.Else(emptyMatchResult("string mismatch"))))
+        val symbol = TempSymbol(N, label.asFieldName + "$")
+        (DefineVar(symbol, resultTerm) :: LetDecl(symbol, Nil) :: stmts, (label, symbol.safeRef) :: results)
+    val resultTerm = resultTerms.reverse match
+      case (_, term) :: Nil => term
+      case terms => Rcd(false, terms.map: (label, term) =>
+        RcdField(str(label.asFieldName), term))
+    Blk(tests.reverse, resultTerm)
+
   import Pattern.*
 
   /** Represent things that can be used as expressions in consequents. */

hkmc2/shared/src/main/scala/hkmc2/semantics/ups/FixedPointCompiler.scala

@@ -335,7 +335,8 @@ class FixedPointCompiler(using tl: TL)(using State, Ctx, Raise) extends TermSynt
     case Not(pattern) => mentions(pattern, target)
     case Rename(pattern, _) => mentions(pattern, target)
     case Extract(pattern, _, _) => mentions(pattern, target)
-    case Literal(_) => false
+    case Concat(patterns) => patterns.exists(mentions(_, target))
+    case Literal(_) | CharClass(_, _) => false
 
   /** Instantiate the pattern groups with a shared `Instantiator`,
     * monomorphizing higher-order patterns such as `Ctx(Redex)` into

hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Instantiator.scala

@@ -124,20 +124,28 @@ class Instantiator(using tl: TL)(using Ctx, State, Raise):
     case SP.Wildcard() => Wildcard
     case SP.Literal(literal) => Literal(literal)
     case SP.Range(lower, upper, rightInclusive) =>
-      // Currently, we expand the range pattern into a list of literals. After
-      // the `where` clause or chain patterns are implemented, we could directly
-      // expand the range pattern into a range test.
       (lower, upper) match
-        case (StrLit(lower), StrLit(upper)) =>
-          Or((lower.head to upper.head).map(c => Literal(StrLit(c.toString))).toList)
+        case (StrLit(lower), StrLit(upper)) if lower.nonEmpty && upper.nonEmpty =>
+          // String ranges compare the first UTF-16 code unit, mirroring the
+          // previous expansion `(lower.head to upper.head)`. Keeping the range
+          // symbolic lets the string pattern compiler emit compact
+          // character-class transitions instead of wide disjunctions.
+          CharClass(lower.head.toInt, upper.head.toInt).withLocOf(pattern)
         case (IntLit(lower), IntLit(upper)) =>
+          // Integer ranges are still expanded into a list of literals. After
+          // the `where` clause or chain patterns are implemented, we could
+          // directly expand the range pattern into a range test.
           Or((lower to upper).map(i => Literal(IntLit(i))).toList)
         case _ =>
           error(msg"Range patterns are not supported in pattern compilation." -> pattern.toLoc)
           Never
     case SP.Concatenation(left, right) =>
-      error(msg"String concatenation is not supported in pattern compilation." -> pattern.toLoc)
-      Never
+      // Flatten nested concatenations into one sequence so that the string
+      // pattern compiler sees the whole `~`-spine at once.
+      def parts(pattern: Pat): Ls[Pat] = pattern match
+        case Concat(patterns) => patterns
+        case pattern => pattern :: Nil
+      Concat(parts(instantiate(left)) ::: parts(instantiate(right))).withLocOf(pattern)
     case SP.Tuple(leading, spread) =>
       val instantiatedSpread = spread.map:
         case (spreadKind, middle, trailing) =>
@@ -171,4 +179,9 @@ class Instantiator(using tl: TL)(using Ctx, State, Raise):
             acc
           case N => true
       instantiate(pattern)
-    case _: SP.Guarded => TODO("instantiate for Guarded")
+    case _: SP.Guarded =>
+      // Guards may fail after consumption based on information the automaton
+      // cannot track, which would reintroduce backtracking; they are excluded
+      // from compiled patterns for now.
+      error(msg"Guarded patterns are not supported in pattern compilation." -> pattern.toLoc)
+      Never