diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala index f22f52fee6..b94534b284 100644 --- a/hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/Elaborator.scala @@ -332,6 +332,7 @@ object Elaborator: loopEnd: ModuleOrObjectSymbol, tuple: ModuleOrObjectSymbol, str: ModuleOrObjectSymbol, + strPat: ModuleOrObjectSymbol, unreachable: TermSymbol, tupleGet: TermSymbol, tupleSlice: TermSymbol, @@ -340,6 +341,9 @@ object Elaborator: strGet: TermSymbol, strTake: TermSymbol, strLeave: TermSymbol, + strPatMatchWhole: TermSymbol, + strPatParseWhole: TermSymbol, + strPatParsePrefix: TermSymbol, matchSuccessCls: ClassSymbol, matchSuccessTrm: TermSymbol, matchFailureCls: ClassSymbol, @@ -375,11 +379,13 @@ object Elaborator: val tuple = modOrObj("Tuple") val str = modOrObj("Str") + val strPat = modOrObj("StrPat") RuntimeSymbols( unit = modOrObj("Unit"), loopEnd = modOrObj("LoopEnd"), tuple = tuple, str = str, + strPat = strPat, unreachable = term("unreachable"), tupleGet = moduleMember(tuple, "get"), tupleSlice = moduleMember(tuple, "slice"), @@ -388,6 +394,9 @@ object Elaborator: strGet = moduleMember(str, "get"), strTake = moduleMember(str, "take"), strLeave = moduleMember(str, "leave"), + strPatMatchWhole = moduleMember(strPat, "matchWhole"), + strPatParseWhole = moduleMember(strPat, "parseWhole"), + strPatParsePrefix = moduleMember(strPat, "parsePrefix"), matchSuccessCls = cls("MatchSuccess"), matchSuccessTrm = term("MatchSuccess"), matchFailureCls = cls("MatchFailure"), @@ -441,6 +450,10 @@ object Elaborator: def strGetSymbol: TermSymbol = runtimeSymbols.strGet def strTakeSymbol: TermSymbol = runtimeSymbols.strTake def strLeaveSymbol: TermSymbol = runtimeSymbols.strLeave + def strPatSymbol: ModuleOrObjectSymbol = runtimeSymbols.strPat + def strPatMatchWholeSymbol: TermSymbol = runtimeSymbols.strPatMatchWhole + def strPatParseWholeSymbol: TermSymbol = runtimeSymbols.strPatParseWhole + def strPatParsePrefixSymbol: TermSymbol = runtimeSymbols.strPatParsePrefix def matchSuccessClsSymbol: ClassSymbol = runtimeSymbols.matchSuccessCls def matchSuccessTrmSymbol: TermSymbol = runtimeSymbols.matchSuccessTrm def matchFailureClsSymbol: ClassSymbol = runtimeSymbols.matchFailureCls diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/ucs/TermSynthesizer.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/ucs/TermSynthesizer.scala index 3018d79677..b201a02945 100644 --- a/hkmc2/shared/src/main/scala/hkmc2/semantics/ucs/TermSynthesizer.scala +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/ucs/TermSynthesizer.scala @@ -63,6 +63,9 @@ trait TermSynthesizer(using State): protected lazy val stringGet = sel(sel(runtimeRef, "Str", State.strSymbol), "get", State.strGetSymbol) protected lazy val stringTake = sel(sel(runtimeRef, "Str", State.strSymbol), "take", State.strTakeSymbol) protected lazy val stringLeave = sel(sel(runtimeRef, "Str", State.strSymbol), "leave", State.strLeaveSymbol) + protected lazy val strPatMatchWhole = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "matchWhole", State.strPatMatchWholeSymbol) + protected lazy val strPatParseWhole = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "parseWhole", State.strPatParseWholeSymbol) + protected lazy val strPatParsePrefix = sel(sel(runtimeRef, "StrPat", State.strPatSymbol), "parsePrefix", State.strPatParsePrefixSymbol) /** Make a term that looks like `runtime.Tuple.get(t, i)`. */ protected final def callTupleGet(t: Term, i: Int, label: Str): Term = @@ -92,6 +95,17 @@ trait TermSynthesizer(using State): protected final def callStringDrop(t: Term.Ref, n: Int, label: Str) = app(stringLeave, tup(fld(t), fld(int(n))), label) + /** Aggregate the transform closures of a compiled string pattern into a + * tuple for the matching engine. The closures may originate from different + * source blocks, so the tuple's location must be pinned explicitly: + * deriving it from the children would mix origins and trip the location + * distinctness assertion in `AutoLocated`. */ + protected final def actionsTuple(actions: Ls[Term], siteLoc: Opt[Loc]): Term = + val tuple = tup(actions) + siteLoc.orElse(actions.iterator.flatMap(_.toLoc.iterator).nextOption()) match + case loc @ S(_) => tuple.withLoc(loc) + case N => tuple // No sub-locations at all: nothing to mix. + protected final def tempLet(dbgName: Str, term: Term)(inner: TempSymbol => Split): Split = val s = TempSymbol(N, dbgName) Split.Let(s, term, inner(s)) diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Compiler.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Compiler.scala index 9e767fd48b..1bfb8833ef 100644 --- a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Compiler.scala +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Compiler.scala @@ -150,7 +150,20 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter .mkString("{", ", ", "}")}" ): val expandedPatterns = patterns.map(p => (p.label, p.expand(Set.empty))) - val heads = expandedPatterns.flatMap((_, p) => p.heads).toList + // String-shaped patterns (sequences and character classes) cannot take + // part in head-based specialization: how a string scrutinee is split is a + // decision global to the whole sequence. Whenever one is present, all + // string-shaped patterns — including plain string literals, whose heads + // would overlap the `Str` class — are absorbed into a single `Str` head + // whose branch runs one compiled automaton per label, like a lexer + // jointly matching several token rules (see `StringCompiler`). + val absorbStrings = expandedPatterns.exists((_, p) => StringCompiler.containsStringNode(p)) + val strSymbol = ctx.builtins.Str + val heads = expandedPatterns.flatMap((_, p) => p.heads).toList.filter: head => + !absorbStrings || (head match + case _: StrLit => false + case symbol: ClassLikeSymbol if symbol is strSymbol => false + case _ => true) // This is the parameter of the current multi-matcher. val scrutinee = VarSymbol(Ident("input")) // Assemble branches for constructors and literals. @@ -170,12 +183,16 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter case _: (syntax.Literal | ModuleOrObjectSymbol) => empty val consequent = Split.Else(multiMatcherBranch(specialized, scrutinee, classFields)) Branch(scrutinee.safeRef, head.toFlatPattern(classFieldArguments), consequent) + val stringBranch = if !absorbStrings then N else + val pattern = FlatPattern.ClassLike(strSymbol.safeRef, strSymbol, N, false)(Tree.Dummy) + val consequent = Split.Else(multiMatcherStringBranch(expandedPatterns, scrutinee)) + S(Branch(scrutinee.safeRef, pattern, consequent)) // Assemble the default branch. val default = val specialized = expandedPatterns.specializeSet(N) Split.Else(multiMatcherBranch(specialized, scrutinee, Map.empty)) // Make a split that tries all branches in order. - val topmostSplit = branches.foldRight(default)(_ ~: _) + val topmostSplit = (branches ::: stringBranch.toList).foldRight(default)(_ ~: _) val bodyTerm = SynthIf(topmostSplit) log(s"Multi-matcher body:\n${topmostSplit.prettyPrint}") (paramList(param(scrutinee)), bodyTerm) @@ -243,7 +260,76 @@ class Compiler(using Context)(using tl: TL)(using Ctx, State, Raise) extends Ter // Lastly, we return the matcher result, directly for singleton matchers // and as a record otherwise. Blk(bindings ::: tests.reverse, resultTerm) - + + /** The branch body for the absorbed `Str` head: each label's string-shaped + * fragment is compiled to its own whole-match automaton (see the note in + * `buildMultiMatcherBody`). The per-label result terms follow the same + * protocol as `multiMatcherBranch`: a Boolean in match-only mode, and a + * `MatchSuccess`/`MatchFailure` value in full mode. + */ + def multiMatcherStringBranch( + patterns: Set[(Label, ExPat)], + scrutinee: VarSymbol, + )(using ResultMode): Blk = + val z = (Nil: Ls[Statement], Nil: Ls[(Label, Term)]) + val (tests, resultTerms) = patterns.iterator.foldLeft(z): + case ((stmts, results), (label, pattern)) => + val fragment = StringCompiler.stringFragment(pattern).simplify + val resultTerm = fragment match + case And(Nil) => + // This label has no string-shaped alternative: it cannot match. + emptyMatchResult("not a string pattern") + // Note that each label needs its own compiler: a compiler instance + // accumulates the automaton states (and failure flag) of a single + // region. + case fragment => StringCompiler().compile(fragment, StringCompiler.Mode.Whole) match + case N => emptyMatchResult("rejected string pattern") + case S(compiled) => + val matchTableTerm = str(compiled.matchTable) + if isMatchOnly then + app(strPatMatchWhole, tup(fld(matchTableTerm), fld(scrutinee.safeRef)), "string match") + else if compiled.pure then + // An operation-free whole match preserves the scrutinee. + val matchedSymbol = TempSymbol(N, "stringMatched") + val call = app(strPatMatchWhole, tup(fld(matchTableTerm), fld(scrutinee.safeRef)), "string match") + SynthIf(Split.Let(matchedSymbol, call, + Branch(matchedSymbol.safeRef, + Split.Else(makeMatchSuccess(scrutinee.safeRef)) + ) ~: Split.Else(emptyMatchResult("string mismatch")))) + else + // Note: expanded patterns may aggregate sub-patterns from + // several source blocks, so their auto-computed location is + // not usable here; the helper pins the first action's own + // location instead (the surrounding terms are location-free). + val call = app(strPatParseWhole, + tup(fld(str(compiled.table)), fld(actionsTuple(compiled.actions, N)), fld(scrutinee.safeRef)), + "string parse") + val resultSymbol = TempSymbol(N, "parseResult") + val outputSymbol = TempSymbol(N, "stringOutput") + val slotSymbols = compiled.visibleSlots.map: (symbol, slot) => + (symbol, slot, TempSymbol(N, s"${symbol.name}$$")) + val bindingsTerm = makeBindings(slotSymbols.map: + (symbol, _, local) => RcdField(str(symbol.name), local.safeRef)) + val success = slotSymbols.foldRight( + Split.Else(makeMatchSuccess(outputSymbol.safeRef, bindingsTerm)): Split + ): + case ((_, slot, local), inner) => + Split.Let(local, callTupleGet(resultSymbol.safeRef, 1 + slot, "string binding"), inner) + SynthIf(Split.Let(resultSymbol, call, + Branch( + resultSymbol.safeRef, + // The engine returns null on failure and an array on success. + FlatPattern.Tuple(1, true), + Split.Let(outputSymbol, callTupleGet(resultSymbol.safeRef, 0, "string output"), success) + ) ~: Split.Else(emptyMatchResult("string mismatch")))) + val symbol = TempSymbol(N, label.asFieldName + "$") + (DefineVar(symbol, resultTerm) :: LetDecl(symbol, Nil) :: stmts, (label, symbol.safeRef) :: results) + val resultTerm = resultTerms.reverse match + case (_, term) :: Nil => term + case terms => Rcd(false, terms.map: (label, term) => + RcdField(str(label.asFieldName), term)) + Blk(tests.reverse, resultTerm) + import Pattern.* /** Represent things that can be used as expressions in consequents. */ diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/FixedPointCompiler.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/FixedPointCompiler.scala index 1776a410f2..f539dcecdd 100644 --- a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/FixedPointCompiler.scala +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/FixedPointCompiler.scala @@ -335,7 +335,8 @@ class FixedPointCompiler(using tl: TL)(using State, Ctx, Raise) extends TermSynt case Not(pattern) => mentions(pattern, target) case Rename(pattern, _) => mentions(pattern, target) case Extract(pattern, _, _) => mentions(pattern, target) - case Literal(_) => false + case Concat(patterns) => patterns.exists(mentions(_, target)) + case Literal(_) | CharClass(_, _) => false /** Instantiate the pattern groups with a shared `Instantiator`, * monomorphizing higher-order patterns such as `Ctx(Redex)` into diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Instantiator.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Instantiator.scala index 51dd0ec354..d6ca734c1e 100644 --- a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Instantiator.scala +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Instantiator.scala @@ -124,20 +124,28 @@ class Instantiator(using tl: TL)(using Ctx, State, Raise): case SP.Wildcard() => Wildcard case SP.Literal(literal) => Literal(literal) case SP.Range(lower, upper, rightInclusive) => - // Currently, we expand the range pattern into a list of literals. After - // the `where` clause or chain patterns are implemented, we could directly - // expand the range pattern into a range test. (lower, upper) match - case (StrLit(lower), StrLit(upper)) => - Or((lower.head to upper.head).map(c => Literal(StrLit(c.toString))).toList) + case (StrLit(lower), StrLit(upper)) if lower.nonEmpty && upper.nonEmpty => + // String ranges compare the first UTF-16 code unit, mirroring the + // previous expansion `(lower.head to upper.head)`. Keeping the range + // symbolic lets the string pattern compiler emit compact + // character-class transitions instead of wide disjunctions. + CharClass(lower.head.toInt, upper.head.toInt).withLocOf(pattern) case (IntLit(lower), IntLit(upper)) => + // Integer ranges are still expanded into a list of literals. After + // the `where` clause or chain patterns are implemented, we could + // directly expand the range pattern into a range test. Or((lower to upper).map(i => Literal(IntLit(i))).toList) case _ => error(msg"Range patterns are not supported in pattern compilation." -> pattern.toLoc) Never case SP.Concatenation(left, right) => - error(msg"String concatenation is not supported in pattern compilation." -> pattern.toLoc) - Never + // Flatten nested concatenations into one sequence so that the string + // pattern compiler sees the whole `~`-spine at once. + def parts(pattern: Pat): Ls[Pat] = pattern match + case Concat(patterns) => patterns + case pattern => pattern :: Nil + Concat(parts(instantiate(left)) ::: parts(instantiate(right))).withLocOf(pattern) case SP.Tuple(leading, spread) => val instantiatedSpread = spread.map: case (spreadKind, middle, trailing) => @@ -171,4 +179,9 @@ class Instantiator(using tl: TL)(using Ctx, State, Raise): acc case N => true instantiate(pattern) - case _: SP.Guarded => TODO("instantiate for Guarded") + case _: SP.Guarded => + // Guards may fail after consumption based on information the automaton + // cannot track, which would reintroduce backtracking; they are excluded + // from compiled patterns for now. + error(msg"Guarded patterns are not supported in pattern compilation." -> pattern.toLoc) + Never diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Pattern.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Pattern.scala index 6a61154888..875f18f787 100644 --- a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Pattern.scala +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/Pattern.scala @@ -56,6 +56,8 @@ sealed abstract class Pattern[+K <: Kind.Complete] extends AutoLocated: case Record(entries) => entries.values.toVector case Tuple(leading, spread) => leading.toVector ++ spread.fold(Vector.empty): case (_, middle, trailing) => middle +: trailing.toVector + case Concat(patterns) => patterns.toVector + case CharClass(_, _) => Vector.empty case And(patterns) => patterns.toVector case Or(patterns) => patterns.toVector case Not(pattern) => Vector.single(pattern) @@ -71,6 +73,8 @@ sealed abstract class Pattern[+K <: Kind.Complete] extends AutoLocated: case Record(entries) => entries.values.flatMap(_.symbols).toList case Tuple(leading, spread) => leading.flatMap(_.symbols) ::: spread.fold(Nil): case (_, middle, trailing) => middle.symbols ::: trailing.flatMap(_.symbols) + case Concat(patterns) => patterns.flatMap(_.symbols) + case CharClass(_, _) => Nil case Synonym(_) => Nil case And(patterns) => patterns.flatMap(_.symbols) case Or(patterns) => @@ -101,6 +105,11 @@ sealed abstract class Pattern[+K <: Kind.Complete] extends AutoLocated: leading.forall(loop(_, visiting)) && spread.forall: (_, middle, trailing) => loop(middle, visiting) && trailing.forall(loop(_, visiting)) + // A whole-string match against a transform-free concatenation produces + // the concatenation of the consumed parts, which is the scrutinee itself. + case Concat(patterns) => + patterns.forall(loop(_, visiting)) + case CharClass(_, _) => true case And(patterns) => patterns.forall(loop(_, visiting)) case Or(patterns) => @@ -191,6 +200,18 @@ sealed abstract class Pattern[+K <: Kind.Complete] extends AutoLocated: if leading2.contains(Never) || middle2 === Never || trailing2.contains(Never) then Never else Tuple(leading2, S((spreadKind, middle2, trailing2))) + case Concat(patterns) => + val simplified = patterns.map(_.simplify) + if simplified contains Never then Never else Concat: + // Empty string literals consume nothing and contribute nothing to the + // output, so they can be dropped as long as one element remains. + simplified.filter: + case Literal(StrLit("")) => false + case _ => true + match + case Nil => Literal(StrLit("")) :: Nil + case simplified => simplified + case CharClass(_, _) => this case And(patterns) => // TODO: Complete the simplification logic here. val simplified = patterns.foldRight(Nil: Ls[Pattern[K]]): @@ -242,6 +263,11 @@ sealed abstract class Pattern[+K <: Kind.Complete] extends AutoLocated: case pattern: Record => pattern case pattern: Tuple => pattern case pattern: Literal => pattern + // Note that synonyms nested in `Concat` are deliberately kept: recursive + // references are handled by the string pattern compiler, which treats them + // as grammar nonterminals rather than expanding them (which would diverge). + case pattern: Concat => pattern + case pattern: CharClass => pattern case _: MatchedClassLike => lastWords("MatchedClassLike encountered during expansion") /** Unwrap `Rename` patterns until we reach a non-`Rename` pattern. Collect @@ -273,6 +299,10 @@ sealed abstract class Pattern[+K <: Kind.Complete] extends AutoLocated: val spreadItem = Iterator.single(spreadKind.str + spread.showDbg) val trailingItems = trailing.iterator.map(_.showDbg) (leadingItems ++ spreadItem ++ trailingItems).mkString("[", ", ", "]") + case Concat(patterns) => + patterns.iterator.map(_.showDbg).mkString("(", " ~ ", ")") + case CharClass(lo, hi) => + s"[${StrLit(lo.toChar.toString).idStr}..${StrLit(hi.toChar.toString).idStr}]" case And(Nil) => "⊤" case And(pattern :: Nil) => pattern.showDbg case And(patterns) => @@ -344,6 +374,24 @@ object Pattern: leading: List[Pat], spread: Opt[(SpreadKind, Pat, List[Pat])], ) extends NonCompositional[Kind.Specialized] + + /** A sequential composition of string patterns: the scrutinee is split into + * consecutive segments, one per element. Unlike the other composite nodes, + * how the scrutinee is split cannot be decided locally, so this node (and + * everything nested in it, including recursive `Synonym` references) is + * compiled as a whole to a finite automaton by `StringCompiler`. It is kept + * opaque (non-compositional) during expansion and specialization: expanding + * synonyms nested under it would diverge on recursive string patterns. + */ + final case class Concat(patterns: Ls[Pat]) extends NonCompositional[Kind.Expanded] + + /** Matches a single-character string whose sole UTF-16 code unit lies in the + * inclusive range `lo` to `hi`. This is the compiled form of string range + * patterns such as `"a" ..= "z"`; keeping the range symbolic (instead of + * expanding it to a disjunction of character literals) lets the string + * pattern compiler build compact character-class transitions. + */ + final case class CharClass(lo: Int, hi: Int) extends NonCompositional[Kind.Expanded] /** Represents a pattern synonym. * @param sym the pattern symbol corresponding @@ -396,15 +444,23 @@ import Pattern.* extension (pattern: ExPat) /** Modifies the pattern under the assumption that the scrutinee matches the * given literal. We decide to not care about literals with fields. (For - * example, wrapped primitives with properties in JavaScript.) */ + * example, wrapped primitives with properties in JavaScript.) + * + * Note on string-shaped nodes (`Concat` and `CharClass`): whenever they are + * present in a multi-matcher, all string-shaped patterns (including string + * literals) are absorbed into a single `Str` class head and handled by the + * string pattern compiler, so this function is never called with a `StrLit` + * head in their presence. Under any other head, they cannot match. */ def specialize(lit: syntax.Literal): SpPat = pattern.map: case Literal(`lit`) => Wildcard - case _: (Literal | ClassLike) => Never + case _: (Literal | ClassLike | Concat | CharClass) => Never case pattern: (Record | Tuple) => pattern case _: (MatchedClassLike | Synonym) => lastWords("unexpected specialized/complete node in specialize(lit)") - + /** Modifies the pattern under the assumption that the scrutinee matches the - * given class. */ + * given class. String-shaped patterns are `Never` here because the `Str` + * head branch extracts them via `StringCompiler.stringFragment` instead of + * relying on head specialization. */ def specialize(symbol: ClassLikeSymbol): SpPat = pattern.map: case Literal(_) => Never /** Note that `ClassLike` corresponds the syntax sugar of class patterns @@ -413,14 +469,15 @@ extension (pattern: ExPat) case ClassLike(`symbol`, arguments) => arguments.fold(Wildcard)(MatchedClassLike(symbol, _)) case ClassLike(_, _) => Never + case _: (Concat | CharClass) => Never case pattern: (MatchedClassLike | Record | Tuple) => pattern - + /** Modifies the pattern under the assumption that the scrutinee matches the * given literal or class. */ def specialize(head: Option[Head]): SpPat = head match case Some(h: syntax.Literal) => pattern.specialize(h) case Some(h: ClassLikeSymbol) => pattern.specialize(h) case None => pattern.map: - case _: (Literal | ClassLike) => Never + case _: (Literal | ClassLike | Concat | CharClass) => Never case pattern: (Record | Tuple) => pattern case _: (MatchedClassLike | Synonym) => lastWords("unexpected specialized/complete node in specialize(None)") diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/SplitCompiler.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/SplitCompiler.scala index 0e5465894f..1aec2d8069 100644 --- a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/SplitCompiler.scala +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/SplitCompiler.scala @@ -657,7 +657,10 @@ class SplitCompiler(using tl: TL)(using State, Ctx, Raise) extends TermSynthesiz case S(objectSymbol: ModuleOrObjectSymbol) => makeMatchObjectSplit(pattern.toLoc, scrutinee, target, objectSymbol, arguments) case S(patternSymbol: PatternSymbol) => - makeMatchPatternSplit(scrutinee, target, patternSymbol, arguments) + if isParametricStringSite(patternSymbol, arguments) then + makeStringRegionSplit(scrutinee, pattern, outputNeeded) + else + makeMatchPatternSplit(scrutinee, target, patternSymbol, arguments) case N => error(msg"Cannot use this ${target.describe} as a pattern." -> target.toLoc) RejectSplit @@ -699,27 +702,42 @@ class SplitCompiler(using tl: TL)(using State, Ctx, Raise) extends TermSynthesiz Branch(scrutinee(), FlatPattern.Lit(literal), makeConsequent(scrutinee, SeqMap.empty)) ~: alternative case Range(lower, upper, rightInclusive) => (makeConsequent, alternative) => makeRangeTest(scrutinee, lower, upper, rightInclusive, makeConsequent(scrutinee, SeqMap.empty)) ~~: alternative - case Concatenation(left, right) => (makeConsequent, alternative) => - makeStringPrefixMatchSplit(scrutinee, left)( - (consumedOutput, remainingOutput, bindingsFromConsumed) => - makeMatchSplit(remainingOutput, right, outputNeeded)( - (postfixOutput, bindingsFromRemaining) => - if outputNeeded then - val combinedOutput = new LazyScrut(S("concatenatedOutput")) - val combinedTerm = app( - add.ref(), - tup(fld(consumedOutput()), fld(postfixOutput())), - "concatenated string output" - ) - combinedOutput.toLet( - combinedTerm, - makeConsequent(combinedOutput, bindingsFromConsumed ++ bindingsFromRemaining) - ) ~~: alternative - else - makeConsequent(scrutinee, bindingsFromConsumed ++ bindingsFromRemaining) ~~: alternative, - alternative), - alternative - ) + case Concatenation(left, right) => + if !regionSupported(pattern, Set.empty) then + // Either this sequence occurs in the body of a parametric pattern + // definition — its pattern parameters are only known at use sites, + // which compile their own automata (see `isParametricStringSite`) — + // or it contains constructs the automaton does not compile yet + // (guards, chains, extraction arguments). Such sequences remain on + // the legacy greedy prefix composition. + (makeConsequent, alternative) => + makeStringPrefixMatchSplit(scrutinee, left)( + (consumedOutput, remainingOutput, bindingsFromConsumed) => + makeMatchSplit(remainingOutput, right, outputNeeded)( + (postfixOutput, bindingsFromRemaining) => + if outputNeeded then + val combinedOutput = new LazyScrut(S("concatenatedOutput")) + val combinedTerm = app( + add.ref(), + tup(fld(consumedOutput()), fld(postfixOutput())), + "concatenated string output" + ) + combinedOutput.toLet( + combinedTerm, + makeConsequent(combinedOutput, bindingsFromConsumed ++ bindingsFromRemaining) + ) ~~: alternative + else + makeConsequent(scrutinee, bindingsFromConsumed ++ bindingsFromRemaining) ~~: alternative, + alternative), + alternative + ) + else + // String sequences are compiled as a whole — together with + // everything nested in them, including recursive pattern references + // — into a finite automaton. Splitting decisions are global to the + // sequence, so no prefix-protocol composition (with its greedy + // misbehavior) takes place anymore. + makeStringRegionSplit(scrutinee, pattern, outputNeeded) case Tuple(elements, N) => (makeConsequent, alternative) => // Fixed-length tuple patterns are similar to constructor patterns. val (subScrutinees, makeChainedConsequent) = @@ -1118,6 +1136,167 @@ class SplitCompiler(using tl: TL)(using State, Ctx, Raise) extends TermSynthesiz msg"String patterns are not yet supported by efficient compilation." -> pattern.toLoc makeStringPrefixMatchSplit(scrutinee, pattern) + /** Whether the pattern transitively contains a string sequencing construct + * (`~`), looking through the definitions of referenced pattern symbols. + * Such patterns are compiled to finite automata (see `StringCompiler`). + */ + private def containsStringSeq(pattern: SP): Bool = + val visited = collection.mutable.Set.empty[PatternSymbol] + def loop(pattern: SP): Bool = pattern match + case Concatenation(_, _) => true + case Constructor(target, arguments) => + arguments.exists(_.exists(loop)) || + target.resolvedSym.flatMap(_.asPat).exists: patternSymbol => + visited.add(patternSymbol) && + patternSymbol.defn.exists(defn => loop(defn.pattern)) + case Composition(_, left, right) => loop(left) || loop(right) + case Negation(pattern) => loop(pattern) + case Wildcard() | Literal(_) | Range(_, _, _) => false + case Tuple(leading, spread) => leading.exists(loop) || spread.exists: + case (_, middle, trailing) => loop(middle) || trailing.exists(loop) + case Record(fields) => fields.exists((_, pattern) => loop(pattern)) + case Chain(first, second) => loop(first) || loop(second) + case Alias(pattern, _) => loop(pattern) + case Transform(pattern, _, _) => loop(pattern) + case Annotated(pattern, _) => loop(pattern) + case Guarded(pattern, _) => loop(pattern) + loop(pattern) + + /** Whether the pattern (and every pattern definition it references) only + * uses constructs that the string automaton compiles faithfully. Patterns + * failing this check keep the legacy backtracking translation: + * + * - Guards and chains may fail after consumption based on information the + * automaton does not track; + * - references to pattern symbols with extraction arguments (including + * the output-matching shorthand) bind values that cannot cross the + * automaton boundary; + * - pattern parameter references are only meaningful once bound: they are + * accepted when `boundParams` says an instantiation will substitute + * them (at use sites of parametric patterns), and rejected inside the + * ahead-of-time compilation of the parametric definition itself. + */ + private def regionSupported(pattern: SP, boundParams: Set[VarSymbol]): Bool = + val visited = collection.mutable.Set.empty[PatternSymbol] + def loop(pattern: SP, bound: Set[VarSymbol]): Bool = pattern match + case Guarded(_, _) | Chain(_, _) => false + case Constructor(target, arguments) => target.resolvedSym match + case S(symbol: VarSymbol) => + bound.contains(symbol) && arguments.isEmpty + case symbolOption => symbolOption.flatMap(_.asPat) match + case S(patternSymbol) => patternSymbol.defn match + case N => false + case S(defn) => + arguments.fold(0)(_.length) == defn.patternParams.length && + arguments.getOrElse(Nil).forall(loop(_, bound)) && + (!visited.add(patternSymbol) || + loop(defn.pattern, defn.patternParams.iterator.map(_.sym).toSet)) + // Class and object patterns cannot match a string, so they are + // harmless dead alternatives within a region; still check their + // sub-patterns, whose transforms would otherwise be miscompiled. + case N => arguments.forall(_.forall(loop(_, bound))) + case Composition(_, left, right) => loop(left, bound) && loop(right, bound) + case Negation(pattern) => loop(pattern, bound) + case Wildcard() | Literal(_) | Range(_, _, _) => true + case Concatenation(left, right) => loop(left, bound) && loop(right, bound) + case Tuple(leading, spread) => leading.forall(loop(_, bound)) && spread.forall: + case (_, middle, trailing) => loop(middle, bound) && trailing.forall(loop(_, bound)) + case Record(fields) => fields.forall((_, pattern) => loop(pattern, bound)) + case Alias(pattern, _) => loop(pattern, bound) + case Transform(pattern, _, _) => loop(pattern, bound) + case Annotated(pattern, _) => loop(pattern, bound) + loop(pattern, boundParams) + + /** Parametric pattern definitions cannot precompile their matcher methods — + * the pattern arguments are only known at the use site — so use sites of + * parametric string patterns compile the fully instantiated pattern into an + * automaton in place. (Non-parametric definitions keep calling their + * automaton-backed `unapply` instead, avoiding per-site code duplication.) + * This is the case exactly when every argument is a pattern argument and + * a string sequence occurs in the definition or the arguments. + */ + private def isParametricStringSite(patternSymbol: PatternSymbol, arguments: Opt[Ls[SP]]): Bool = + patternSymbol.defn.exists: defn => + defn.patternParams.nonEmpty && + arguments.fold(0)(_.length) == defn.patternParams.length && + (containsStringSeq(defn.pattern) || arguments.getOrElse(Nil).exists(containsStringSeq)) && + // The definition body is checked with its own parameters considered + // bound (instantiation will substitute them); the arguments themselves + // must be closed, which also rules out references like `Rep(S)` inside + // the body of another parametric definition. + regionSupported(defn.pattern, defn.patternParams.iterator.map(_.sym).toSet) && + arguments.getOrElse(Nil).forall(regionSupported(_, Set.empty)) + + /** Compile a whole-match string region rooted at `pattern` into a finite + * automaton and emit the split that runs it (see `StringCompiler`). + * + * When the region carries no transforms and neither output nor bindings + * are demanded, recognition alone suffices: a single backward scan with no + * allocation. Transforms, however, always force the parsing entry point — + * they run exactly once on the committed parse, even when the match is + * only used as a condition. Then the result array `[output, bindings...]` + * is destructured. + */ + private def makeStringRegionSplit(scrutinee: Scrut, pattern: SP, outputNeeded: Bool): MakeSplit = + val instantiator = new Instantiator + val (instantiated, context) = instantiator(pattern) + val compiler = new StringCompiler(using context) + compiler.compile(instantiated, StringCompiler.Mode.Whole) match + case N => RejectSplit // Errors have been reported; compile nothing. + case S(compiled) => + if compiled.pure || + (!outputNeeded && compiled.visibleSlots.isEmpty && compiled.actions.isEmpty) then + (makeConsequent, alternative) => + val callTerm = app(strPatMatchWhole, + tup(fld(str(compiled.matchTable)), fld(scrutinee())), "whole string match") + tempLet("stringMatched", callTerm): resultSymbol => + Branch(resultSymbol.safeRef, makeConsequent(scrutinee, SeqMap.empty)) ~: alternative + else (makeConsequent, alternative) => + val callTerm = app(strPatParseWhole, + tup(fld(str(compiled.table)), fld(actionsTuple(compiled.actions, pattern.toLoc)), fld(scrutinee())), + "whole string parse") + tempLet("parseResult", callTerm): resultSymbol => + val outputSymbol = TempSymbol(N, "stringOutput") + val slotSymbols = compiled.visibleSlots.map: (symbol, slot) => + (symbol, slot, TempSymbol(N, s"${symbol.name}$$")) + val bindings: BindingMap = SeqMap.from(slotSymbols.map: + (symbol, _, local) => symbol -> local.toScrut) + val consequent = slotSymbols.foldRight(makeConsequent(outputSymbol.toScrut, bindings)): + case ((_, slot, local), inner) => + Split.Let(local, callTupleGet(resultSymbol.safeRef, 1 + slot, "string binding"), inner) + Branch( + resultSymbol.safeRef, + // The engine returns null on failure and an array on success. + FlatPattern.Tuple(1, true), + Split.Let(outputSymbol, callTupleGet(resultSymbol.safeRef, 0, "string output"), consequent) + ) ~: alternative + + /** Compile the body of a pattern definition into a prefix-matching + * automaton for its `unapplyStringPrefix` method. The result follows the + * established protocol: `MatchSuccess([consumed, remaining], null)`. + */ + private def makeStringPrefixAutomatonSplit(inputSymbol: VarSymbol, pattern: SP)(using Raise): Split = + val instantiator = new Instantiator + val (instantiated, context) = instantiator(pattern) + val compiler = new StringCompiler(using context) + compiler.compile(instantiated, StringCompiler.Mode.Prefix) match + case N => failure + case S(compiled) => + val callTerm = app(strPatParsePrefix, + tup(fld(str(compiled.table)), fld(actionsTuple(compiled.actions, pattern.toLoc)), fld(inputSymbol.safeRef)), + "string prefix parse") + tempLet("prefixResult", callTerm): resultSymbol => + val consumedSymbol = TempSymbol(N, "consumed") + val remainingSymbol = TempSymbol(N, "remaining") + Branch( + resultSymbol.safeRef, + FlatPattern.Tuple(2, true), + Split.Let(consumedSymbol, callTupleGet(resultSymbol.safeRef, 0, "consumed prefix"), + Split.Let(remainingSymbol, callTupleGet(resultSymbol.safeRef, 1, "remaining input"), + Split.Else(makeMatchSuccess( + tup(fld(consumedSymbol.safeRef), fld(remainingSymbol.safeRef)))))) + ) ~: failure + def compilePattern(scrutinee: Scrut, pattern: SP): MakeSplit = compilePattern(scrutinee, pattern, true) @@ -1307,9 +1486,21 @@ class SplitCompiler(using tl: TL)(using State, Ctx, Raise) extends TermSynthesiz // the translation of `unapply` function. given Raise = Function.const(()) val inputSymbol = VarSymbol(Ident("input")) - val topmost = makeStringPrefixMatchSplit(inputSymbol.toScrut, pd.pattern) - ((consumedOutput, remainingOutput, bindings) => Split.Else: - makeMatchSuccess(tup(fld(consumedOutput()), fld(remainingOutput()))), failure) + val topmost = + if pd.patternParams.isEmpty && containsStringSeq(pd.pattern) + && regionSupported(pd.pattern, Set.empty) then + // The whole body — including alternatives without `~` — is compiled + // into one prefix automaton, so alternation and splitting decisions + // are resolved jointly rather than by greedy composition. + makeStringPrefixAutomatonSplit(inputSymbol, pd.pattern) + else + // Bodies without string sequences have no splitting ambiguity, and + // parametric definitions cannot be compiled ahead of the use site + // (their use sites compile in-place automata instead; this legacy + // method remains only for direct calls). + makeStringPrefixMatchSplit(inputSymbol.toScrut, pd.pattern) + ((consumedOutput, remainingOutput, bindings) => Split.Else: + makeMatchSuccess(tup(fld(consumedOutput()), fld(remainingOutput()))), failure) log(s"Translated `unapplyStringPrefix`: ${topmost.prettyPrint}") makeMethod("unapplyStringPrefix", pd.patternParams, inputSymbol, topmost) unapply :: unapplyStringPrefix :: Nil diff --git a/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/StringCompiler.scala b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/StringCompiler.scala new file mode 100644 index 0000000000..99b4fb26d0 --- /dev/null +++ b/hkmc2/shared/src/main/scala/hkmc2/semantics/ups/StringCompiler.scala @@ -0,0 +1,927 @@ +package hkmc2 +package semantics +package ups + +import hkmc2.utils.*, shorthands.* + +import syntax.Tree, Tree.StrLit +import Elaborator.{Ctx, State, ctx}, utils.TL +import Message.MessageContext, ucs.error +import Pattern.* + +import collection.mutable.{Buffer, Map as MutMap, LinkedHashMap, Set as MutSet} + +/** Compiles the string fragment of patterns — sequential compositions (`~`), + * character classes, string literals, alternations, bindings, transforms, and + * (mutually) tail-recursive pattern references — into a finite automaton that + * is executed by the runtime engine `Runtime.StrPat`. + * + * ## Semantics implemented + * + * Recognition is relational: a concatenation matches if *some* split of the + * scrutinee matches, regardless of how greedy sub-patterns are. Among all + * successful parses, the committed one (which determines bindings, transform + * applications, and outputs) is the parse that a recursive-descent matcher + * with unlimited backtracking would find first, trying alternatives from left + * to right ("leftmost-greedy by alternation order"). Transforms run exactly + * once, and only on the committed parse. + * + * ## Execution scheme + * + * We follow the two-pass scheme of Frisch and Cardelli ("Greedy Regular + * Expression Matching", ICALP 2004): + * + * 1. A *backward* pass runs the determinized reverse automaton from the end + * of the input, recording for each position the set of NFA states that + * can still reach acceptance by consuming the remaining suffix. + * 2. A *forward* pass walks the prioritized NFA deterministically: at every + * choice point it takes the first branch that the backward pass proved + * viable. Value operations (slices, bindings, transform calls) execute on + * this committed walk only. + * + * Recognition alone (`ResultMode.MatchOnly`-style call sites) only needs the + * backward automaton, run in a rolling fashion with no allocation. + * + * ## Recursion + * + * After instantiation, pattern definitions form a grammar whose nonterminals + * are `Pattern.Instantiation`s. A strongly connected component of that + * grammar can be compiled to a finite automaton iff every reference to a + * member of the same SCC sits in *tail position* of its defining body (the + * grammar is right-linear modulo non-recursive content, cf. Mohri and + * Nederhof's "strongly regular" grammars). Such references become plain + * ε-transitions (tail calls compile to gotos). Self-embedding references are + * rejected with a compile-time error and nothing is compiled. + * + * Value operations that syntactically follow a tail call (e.g. the binding + * and transform in `((L as h) ~ (Tail as t)) => [h, ...t]`) cannot run on the + * transition itself — the automaton never "returns". They are recorded as + * deferred frames (`Op.Defer`) carrying the values they capture, and executed + * innermost-first when the recursive component is finally exited + * (`Op.Enter`/`Op.Exit` bracket the component). + */ +object StringCompiler: + + /** An inclusive range of UTF-16 code units. Matching is code-unit-based, + * mirroring the previous runtime helpers (`Str.get`/`startsWith`). */ + type CharRange = (Int, Int) + + val MaxUnit: Int = 0xFFFF + + val AnyChar: Ls[CharRange] = (0, MaxUnit) :: Nil + + /** Value operations attached to ε-transitions. Executed only on the + * committed forward walk. The value stack discipline is: evaluating a + * pattern with `needValue` pushes exactly one value (its output). + */ + enum Op: + /** Push the current input position onto the mark stack. */ + case Mark + /** Pop a mark; push the input slice from it to the current position. */ + case Slice + /** Pop two values; push their string concatenation. */ + case Add + /** Pop one value. */ + case Drop + /** Store the top of the value stack (without popping) into a binding slot. */ + case Bind(slot: Int) + /** Push the result of calling an action closure on the given slots. */ + case Call(actionId: Int, argSlots: Ls[Int]) + /** Push a sentinel delimiting a recursive component's deferred frames. */ + case Enter + /** Pop and execute deferred frames down to the sentinel, innermost first. */ + case Exit + /** Record the ops to run at `Exit` time, capturing the listed slots now. */ + case Defer(ops: Ls[Op], captured: Ls[Int]) + /** Record the current position as the start of the remaining string. */ + case Rem + + /** Whether the automaton must consume the entire scrutinee or only a + * prefix of it (for `unapplyStringPrefix`). */ + enum Mode: + case Whole, Prefix + + /** The result of compiling one string region. + * + * @param table the encoded automaton program (see `encode` for the layout) + * @param matchTable the recognition-only program for `matchWhole`: just + * the reverse scan, omitting the NFA, the operation pool, and the + * viability matrix — call sites that only ask *whether* the + * scrutinee matches should embed this much smaller table + * @param actions transform closures, in `actionId` order + * @param visibleSlots the root-visible bindings and their slot indices, in + * the order the caller should destructure them + * @param pure true when the region carries no value operations at all, so + * recognition suffices and a whole-match output is the scrutinee + */ + final case class Compiled( + table: Str, + matchTable: Str, + actions: Ls[Term], + visibleSlots: Ls[(VarSymbol, Int)], + pure: Bool, + ): + /** Elements of the array returned by the parse entry points, before the + * binding slots: `[output, slots...]` or `[output, remaining, slots...]`. */ + def resultPrefixSize(mode: Mode): Int = mode match + case Mode.Whole => 1 + case Mode.Prefix => 2 + + /** Extract the string-shaped fragment of an expanded pattern: everything a + * `Str`-headed multi-matcher branch should try to match. Non-string leaves + * become `Never` (they cannot match a string scrutinee). + */ + def stringFragment(pattern: ExPat)(using Ctx): Pat = pattern.map: + case pattern: (Concat | CharClass) => pattern + case pattern @ Literal(_: StrLit) => pattern + case Literal(_) => Never + case pattern @ ClassLike(sym, _) => + if sym is ctx.builtins.Str then pattern else Never + case _: (Record | Tuple) => Never + case _: MatchedClassLike => lastWords("MatchedClassLike encountered in stringFragment") + + /** Whether an expanded pattern contains a string-sequencing node at a + * position visible to head specialization. Used by the multi-matcher to + * decide whether string-shaped patterns should be absorbed into a single + * `Str` head backed by an automaton. + */ + def containsStringNode(pattern: ExPat): Bool = pattern.reduce[Bool](_.exists(identity)): + case _: (Concat | CharClass) => true + +class StringCompiler(using context: Context)(using tl: TL)(using Ctx, State, Raise): + import StringCompiler.*, tl.* + + // ------------------------------------------------------------------------ + // Grammar analysis + // ------------------------------------------------------------------------ + + /** Bodies of all instantiations reachable from the region root. */ + private val bodies = LinkedHashMap.empty[Instantiation, Pat] + + /** Walk all sub-patterns in string position. Arguments of dead (non-string) + * nodes are not traversed: they can never match within a string region. */ + private def stringPositions(pattern: Pat)(f: Pat => Unit): Unit = + f(pattern) + pattern match + case Concat(ps) => ps.foreach(stringPositions(_)(f)) + case Or(ps) => ps.foreach(stringPositions(_)(f)) + case And(ps) => ps.foreach(stringPositions(_)(f)) + case Not(p) => stringPositions(p)(f) + case Rename(p, _) => stringPositions(p)(f) + case Extract(p, _, _) => stringPositions(p)(f) + case _: (Literal | CharClass | ClassLike | MatchedClassLike | Record | Tuple | Synonym) => () + + private def collectBodies(pattern: Pat): Unit = + stringPositions(pattern): + case Synonym(inst) => + if !bodies.contains(inst) then + bodies += inst -> context.get(inst) + collectBodies(bodies(inst)) + case _ => () + + private def references(pattern: Pat): Ls[Instantiation] = + val buffer = Buffer.empty[Instantiation] + stringPositions(pattern): + case Synonym(inst) => buffer += inst + case _ => () + buffer.toList + + /** Strongly connected components of the nonterminal reference graph, + * computed with Tarjan's algorithm. `sccOf` maps each instantiation to its + * component id; `recursiveSccs` contains components that are actually + * recursive (more than one member, or a self-loop). */ + private val sccOf = MutMap.empty[Instantiation, Int] + private val recursiveSccs = MutSet.empty[Int] + + private def computeSccs(): Unit = + val indices = MutMap.empty[Instantiation, Int] + val lowLinks = MutMap.empty[Instantiation, Int] + val onStack = MutSet.empty[Instantiation] + val stack = Buffer.empty[Instantiation] + var nextIndex = 0 + var nextScc = 0 + def strongConnect(v: Instantiation): Unit = + indices(v) = nextIndex + lowLinks(v) = nextIndex + nextIndex += 1 + stack += v + onStack += v + references(bodies(v)).foreach: w => + if !indices.contains(w) then + strongConnect(w) + lowLinks(v) = lowLinks(v) min lowLinks(w) + else if onStack contains w then + lowLinks(v) = lowLinks(v) min indices(w) + if lowLinks(v) == indices(v) then + val sccId = nextScc + nextScc += 1 + var done = false + var size = 0 + while !done do + val w = stack.remove(stack.size - 1) + onStack -= w + sccOf(w) = sccId + size += 1 + if w == v then done = true + // A single-member component is recursive only if it references itself. + if size > 1 || references(bodies(v)).contains(v) then + recursiveSccs += sccId + bodies.keysIterator.foreach: inst => + if !indices.contains(inst) then strongConnect(inst) + + /** Check that every reference to a member of the same (recursive) SCC is in + * tail position of the body it occurs in: nothing may be consumed after it. + * References may still sit under alternations and under binding/transform + * wrappers (whose pending operations are deferred at run time). + * + * On failure, the offending reference is pinned in an error message and + * compilation of the whole region is abandoned. + */ + private def checkTailPositions(): Unit = + def check(owner: Instantiation, pattern: Pat, tail: Bool): Unit = pattern match + case Synonym(inst) => + if sccOf.get(inst) == sccOf.get(owner) && !tail then + failed = true + error( + msg"This recursive use of pattern `${inst.symbol.nme}` is not in tail position." -> inst.toLoc, + msg"Only self references at the end of a string sequence can be compiled to a finite string matcher." -> N, + msg"The recursion involves this pattern." -> owner.toLoc) + case Concat(ps) => ps match + case Nil => () + case _ => + ps.init.foreach(check(owner, _, false)) + check(owner, ps.last, tail) + case Or(ps) => ps.foreach(check(owner, _, tail)) + case Rename(p, _) => check(owner, p, tail) + case Extract(p, _, _) => check(owner, p, tail) + case And(ps) => ps.foreach(check(owner, _, false)) + case Not(p) => check(owner, p, false) + case _: (Literal | CharClass | ClassLike | MatchedClassLike | Record | Tuple) => () + bodies.foreach: (inst, body) => + if sccOf.get(inst).exists(recursiveSccs.contains) then + check(inst, body, true) + + /** A pattern is pure when matching it involves no value operations: no + * bindings, no transforms, transitively through pattern references. The + * output of a pure whole-match is the scrutinee itself, and the consumed + * part of a pure sub-match is a single slice of the input. Cycles are + * treated as pure so the traversal stays well-founded; a cycle only ends + * up pure if every body on it is operation-free. */ + private val pureMemo = MutMap.empty[Instantiation, Bool] + + private def isPureDeep(pattern: Pat): Bool = + def loop(pattern: Pat, visiting: Set[Instantiation]): Bool = pattern match + case _: (Rename[?] | Extract[?]) => false + case Concat(ps) => ps.forall(loop(_, visiting)) + case Or(ps) => ps.forall(loop(_, visiting)) + case And(ps) => ps.forall(loop(_, visiting)) + case Not(p) => loop(p, visiting) + case Synonym(inst) => + if visiting contains inst then true + else pureMemo.getOrElseUpdate(inst, loop(context.get(inst), visiting + inst)) + case _: (Literal | CharClass | ClassLike | MatchedClassLike | Record | Tuple) => true + loop(pattern, Set.empty) + + // ------------------------------------------------------------------------ + // NFA construction + // ------------------------------------------------------------------------ + + private enum Edge: + case Chr(ranges: Ls[CharRange], target: Int) + case Eps(target: Int, ops: Ls[Op]) + + private val states = Buffer.empty[Buffer[Edge]] + + private def newState(): Int = + states += Buffer.empty + states.size - 1 + + private def addChr(from: Int, ranges: Ls[CharRange], target: Int): Unit = + states(from) += Edge.Chr(ranges, target) + + private def addEps(from: Int, target: Int, ops: Ls[Op]): Unit = + if ops.nonEmpty then anyOps = true + states(from) += Edge.Eps(target, ops) + + /** Binding slots, keyed by the bound symbol. */ + private val slots = LinkedHashMap.empty[VarSymbol, Int] + + private def slotOf(symbol: VarSymbol): Int = + slots.getOrElseUpdate(symbol, slots.size) + + /** Transform closures, in `actionId` order. */ + private val actions = Buffer.empty[Term] + + /** The transform terms backing `actions`, for interning by identity. */ + private val actionSources = Buffer.empty[Term] + + private var failed = false + private var anyOps = false + + private def fail(messages: (Message, Opt[Loc])*): Unit = + failed = true + error(messages*) + + /** The slots that deferred operations read before writing them locally: + * their values must be captured when the deferred frame is created, not + * when it is executed (later iterations overwrite the global slots). */ + private def capturedSlots(ops: Ls[Op]): Ls[Int] = + val written = MutSet.empty[Int] + val captured = Buffer.empty[Int] + ops.foreach: + case Op.Bind(slot) => written += slot + case Op.Call(_, argSlots) => + argSlots.foreach: slot => + if !written.contains(slot) && !captured.contains(slot) then captured += slot + case Op.Defer(_, _) => lastWords("nested deferred operations") + case _ => () + captured.toList + + /** When building the body of a recursive SCC copy, references to members of + * the same SCC resolve to the copy's entry states. */ + private final case class SccContext(sccId: Int, entries: Map[Instantiation, Int], pure: Bool) + + /** Build the NFA fragment for `pattern`. + * + * Completion protocol: every path through the fragment either reaches + * `cont` via ε-edges that carry the pattern's value operations (pushing + * exactly one value if `needValue`) followed by `exitOps`, or jumps to a + * same-SCC entry with those pending operations packaged in a deferred + * frame. + */ + private def build(pattern: Pat, cont: Int, needValue: Bool, exitOps: Ls[Op], scc: Opt[SccContext]): Int = + // Pure subtrees need no internal operations even when their value is + // demanded: the consumed part is a single slice of the input. + if (needValue || exitOps.nonEmpty) && isPureDeep(pattern) then + val sliceOps = if needValue then Op.Slice :: exitOps else exitOps + val exit = newState() + addEps(exit, cont, sliceOps) + val inner = build(pattern, exit, false, Nil, scc) + if needValue then + val entry = newState() + addEps(entry, inner, Op.Mark :: Nil) + entry + else inner + else pattern match + case Literal(StrLit(value)) => + // The value ops (if any) were handled by the pure-subtree shortcut. + softAssert(!needValue && exitOps.isEmpty, "literal with pending value operations") + value.foldRight(cont): (unit, next) => + val entry = newState() + addChr(entry, (unit.toInt, unit.toInt) :: Nil, next) + entry + case CharClass(lo, hi) => + softAssert(!needValue && exitOps.isEmpty, "character class with pending value operations") + val entry = newState() + addChr(entry, (lo, hi) :: Nil, cont) + entry + case Or(Nil) => + // The wildcard in string position matches any string, preferring to + // consume as much as possible (the consuming edge comes first). + softAssert(!needValue && exitOps.isEmpty, "wildcard with pending value operations") + val entry = newState() + addChr(entry, AnyChar, entry) + addEps(entry, cont, Nil) + entry + case ClassLike(sym, arguments) if sym is ctx.builtins.Str => + arguments match + case S(_) => + fail(msg"`${sym.nme}` cannot have arguments in a string pattern." -> pattern.toLoc) + newState() + case N => + // `Str` literally means all strings, like a wildcard. (The naive + // translation used to consume exactly one character here, which + // made `Str ~ "!"` unmatchable against "ab!".) + softAssert(!needValue && exitOps.isEmpty, "Str with pending value operations") + val entry = newState() + addChr(entry, AnyChar, entry) + addEps(entry, cont, Nil) + entry + case Or(patterns) => + val entry = newState() + patterns.foreach: p => + addEps(entry, build(p, cont, needValue, exitOps, scc), Nil) + entry + case Concat(patterns) => patterns match + case Nil => + // An empty sequence matches the empty string. The pure-subtree + // shortcut has already handled any pending value operations. + val entry = newState() + addEps(entry, cont, Nil) + entry + case _ => + if needValue then + // Each element contributes its value; concatenate left to right. + // Elements after the first are followed by an `Add`, and the last + // one additionally carries the pending exit operations. + def go(patterns: Ls[Pat], first: Bool): Int = patterns match + case last :: Nil => + val ops = if first then exitOps else Op.Add :: exitOps + build(last, cont, true, ops, scc) + case p :: rest => + val next = go(rest, false) + build(p, next, true, if first then Nil else Op.Add :: Nil, scc) + case Nil => lastWords("unreachable: empty concatenation") + go(patterns, true) + else + def go(patterns: Ls[Pat]): Int = patterns match + case last :: Nil => build(last, cont, false, exitOps, scc) + case p :: rest => build(p, go(rest), false, Nil, scc) + case Nil => lastWords("unreachable: empty concatenation") + go(patterns) + case And(Nil) => newState() // `Never` matches nothing: a dead state. + case And(_) => + fail(msg"Conjunctions are not supported within string patterns yet." -> pattern.toLoc) + newState() + case Not(_) => + fail(msg"Negations are not supported within string patterns yet." -> pattern.toLoc) + newState() + case Rename(p, symbol) => + val ops = Op.Bind(slotOf(symbol)) :: (if needValue then exitOps else Op.Drop :: exitOps) + build(p, cont, true, ops, scc) + case Extract(p, correspondence, term) => + // The same transform can be reached several times within one region: + // expansion shares definition bodies, and references to definitions + // outside the current recursive component are inlined per reference. + // Each transform must nevertheless yield exactly one closure — its + // parameters are the correspondence symbols shared by all copies of + // the pattern, so duplicating the lambda would define those symbols + // twice in the lowered block (which `SymbolRefresher` asserts + // against). Closures are therefore interned by the identity of the + // transform term. + // + // TODO: Interning is per-region, so two regions in one block that + // reach the same definition's transform still produce two closures + // sharing parameter symbols. This only trips the refresher when a + // simplifier pass duplicates a subtree containing both. The principled + // fix is to host each definition's transforms as methods on the + // pattern object (compiled once, next to `unapply`) and reference + // them from regions by selection. + val actionId = actionSources.indexWhere(_ eq term) match + case -1 => + val params = p.symbols.map: symbol => + Param(FldFlags.empty, correspondence(symbol), N, Modulefulness.none) + actionSources += term + actions += Term.Lam(PlainParamList(params), term.mkClone) + actions.size - 1 + case index => index + val argSlots = p.symbols.map(slotOf) + val ops = Op.Call(actionId, argSlots) :: (if needValue then exitOps else Op.Drop :: exitOps) + build(p, cont, false, ops, scc) + case Synonym(inst) => scc match + case S(sccContext) if sccOf.get(inst).contains(sccContext.sccId) => + // A tail call within the current SCC copy compiles to a goto. Any + // pending operations run when the component is exited. + val entry = sccContext.entries.getOrElse(inst, lastWords("missing SCC entry")) + val valueAdjust = + // Impure SCC copies canonically produce a value; drop it if this + // reference does not want one. + if !sccContext.pure && !needValue then Op.Drop :: Nil else Nil + softAssert(!(sccContext.pure && (needValue || exitOps.nonEmpty)), + "value operations inside a pure recursive component") + val pending = valueAdjust ::: exitOps + val gotoState = newState() + if pending.isEmpty then addEps(gotoState, entry, Nil) + else addEps(gotoState, entry, Op.Defer(pending, capturedSlots(pending)) :: Nil) + gotoState + case _ => buildReference(inst, cont, needValue, exitOps) + case _: (Literal | ClassLike | MatchedClassLike | Record | Tuple) => + // Non-string patterns can never match within a string region. This + // mirrors the naive translation, which silently rejected them. + newState() + + /** Build a reference to an instantiation from outside its SCC. */ + private def buildReference(inst: Instantiation, cont: Int, needValue: Bool, exitOps: Ls[Op]): Int = + val sccId = sccOf.getOrElse(inst, lastWords("missing SCC id")) + if !recursiveSccs.contains(sccId) then + // Non-recursive definitions are inlined at the reference. + build(bodies(inst), cont, needValue, exitOps, N) + else if isPureDeep(Synonym(inst)) then + // Pure recursive components carry no operations; their bodies are + // recognition-only and tail calls are plain gotos. + softAssert(!needValue && exitOps.isEmpty, + "the pure-subtree shortcut should have handled value operations") + buildSccCopy(inst, sccId, cont, pure = true) + else + // Impure recursive components: bracket the copy with Enter/Exit so that + // frames deferred at internal tail calls unwind exactly here. The + // pending operations of this reference itself must be deferred too, as + // the outermost frame of the activation: they may read slots bound + // before the recursion (e.g. `head` in + // `(S as head) ~ "," ~ (CommaSep(S) as tail) => head :: tail` + // when this occurrence sits outside the recursive component), and the + // recursion overwrites those slots — so their values are captured now + // and the operations run after all inner frames have unwound. + val exitState = newState() + addEps(exitState, cont, Op.Exit :: Nil) + val copyEntry = buildSccCopy(inst, sccId, exitState, pure = false) + val entry = newState() + val pending = (if needValue then Nil else Op.Drop :: Nil) ::: exitOps + val entryOps = Op.Enter :: + (if pending.isEmpty then Nil else Op.Defer(pending, capturedSlots(pending)) :: Nil) + addEps(entry, copyEntry, entryOps) + entry + + /** Materialize one copy of a recursive SCC, returning the entry state of + * `inst`. All members share the continuation: since internal references + * are tail calls, completing any member's body completes the component. */ + private def buildSccCopy(inst: Instantiation, sccId: Int, cont: Int, pure: Bool): Int = + val members = bodies.keysIterator.filter(sccOf.get(_).contains(sccId)).toList + val entries = members.map(_ -> newState()).toMap + val sccContext = SccContext(sccId, entries, pure) + members.foreach: member => + val bodyEntry = build(bodies(member), cont, needValue = !pure, Nil, S(sccContext)) + addEps(entries(member), bodyEntry, Nil) + entries(inst) + + // ------------------------------------------------------------------------ + // NFA reduction + // ------------------------------------------------------------------------ + + private def edgeTarget(edge: Edge): Int = edge match + case Edge.Chr(_, target) => target + case Edge.Eps(target, _) => target + + private def retarget(edge: Edge, target: Int): Edge = edge match + case Edge.Chr(ranges, _) => Edge.Chr(ranges, target) + case Edge.Eps(_, ops) => Edge.Eps(target, ops) + + /** Rewrite the target of every edge through `f`; returns whether any + * edge changed. */ + private def retargetAll(edges: Buffer[Edge])(f: Int => Int): Bool = + var changed = false + var i = 0 + while i < edges.size do + val edge = edges(i) + val target = edgeTarget(edge) + val mapped = f(target) + if mapped != target then + edges(i) = retarget(edge, mapped) + changed = true + i += 1 + changed + + /** Shrink the NFA in place before encoding; returns the renumbered + * (start, accept) pair. Three semantics-preserving reductions run to a + * fixed point, then unreachable states are pruned: + * + * 1. Edges into states that cannot reach `accept` are dropped. No path + * through them completes for any input, so neither recognition nor + * the committed parse changes (the forward walk would refuse them + * via the viability oracle anyway, at run-time cost). + * 2. States whose entire content is a single operation-free ε-edge are + * contracted: in-edges are redirected to the ε-target. No choice + * point and no operation is involved, so priorities are unaffected. + * 3. States with identical ordered edge lists are merged: they have + * identical futures, including alternation priorities. The accept + * state is never merged away (acceptance is not visible in edges). + * + * The construction above is deliberately generous with ε-plumbing, and + * cross-SCC references are inlined per reference, so this typically + * removes a quarter of the states. Size matters twice: the encoded NFA + * section shrinks linearly and the viability matrix — one bit per + * (reverse state, NFA state) pair — shrinks with the state count. + */ + private def reduceStates(start: Int, accept: Int): (Int, Int) = + var entry = start + def dropDeadEdges(): Bool = + val live = new Array[Bool](states.size) + val preds = Array.fill(states.size)(Buffer.empty[Int]) + states.iterator.zipWithIndex.foreach: (edges, source) => + edges.foreach(edge => preds(edgeTarget(edge)) += source) + val worklist = Buffer(accept) + live(accept) = true + while worklist.nonEmpty do + val state = worklist.remove(worklist.size - 1) + preds(state).foreach: source => + if !live(source) then + live(source) = true + worklist += source + var changed = false + states.foreach: edges => + val kept = edges.filter(edge => live(edgeTarget(edge))) + if kept.size != edges.size then + changed = true + edges.clear() + edges ++= kept + changed + def contractTrivial(): Bool = + val next = Array.fill(states.size)(-1) + states.iterator.zipWithIndex.foreach: (edges, source) => + if source != accept && edges.size == 1 then edges(0) match + case Edge.Eps(target, Nil) if target != source => next(source) = target + case _ => () + // Follow chains of trivial states; the path set guards against cycles + // (an all-trivial ε-cycle is dead and gets dismantled by the other + // reductions, but resolution must not hang on it meanwhile). + def resolve(state: Int): Int = + val path = MutSet.empty[Int] + var cur = state + while next(cur) != -1 && path.add(cur) do cur = next(cur) + cur + var changed = false + states.foreach: edges => + changed |= retargetAll(edges)(resolve) + val resolvedEntry = resolve(entry) + if resolvedEntry != entry then + entry = resolvedEntry + changed = true + changed + def mergeIdentical(): Bool = + val representative = MutMap.empty[(Bool, Ls[Edge]), Int] + val rep = Array.tabulate(states.size)(identity) + states.iterator.zipWithIndex.foreach: (edges, state) => + val key = (state == accept, edges.toList) + representative.get(key) match + case S(canonical) => rep(state) = canonical + case N => representative(key) = state + var changed = false + states.foreach: edges => + changed |= retargetAll(edges)(rep(_)) + if rep(entry) != entry then + entry = rep(entry) + changed = true + changed + var changed = true + while changed do + changed = false + changed |= dropDeadEdges() + changed |= contractTrivial() + changed |= mergeIdentical() + // Prune states unreachable from the entry and renumber the survivors. + // The accept state is kept even if unreachable (a never-matching region): + // the encoding refers to it. + val keep = new Array[Bool](states.size) + keep(entry) = true + val worklist = Buffer(entry) + while worklist.nonEmpty do + val state = worklist.remove(worklist.size - 1) + states(state).foreach: edge => + val target = edgeTarget(edge) + if !keep(target) then + keep(target) = true + worklist += target + keep(accept) = true + val renumber = new Array[Int](states.size) + var nextId = 0 + states.indices.foreach: state => + if keep(state) then + renumber(state) = nextId + nextId += 1 + val compacted = Buffer.empty[Buffer[Edge]] + states.iterator.zipWithIndex.foreach: (edges, state) => + if keep(state) then + retargetAll(edges)(renumber(_)) + compacted += edges + states.clear() + states ++= compacted + (renumber(entry), renumber(accept)) + + // ------------------------------------------------------------------------ + // Reverse determinization (the viability oracle) + // ------------------------------------------------------------------------ + + /** Split the alphabet into equivalence classes: within a class, all + * character edges behave identically. Returns the ordered upper boundaries; + * `classOf(u)` is the number of boundaries that are ≤ u. */ + private def computeBounds(): Ls[Int] = + val points = MutSet.empty[Int] + states.foreach: edges => + edges.foreach: + case Edge.Chr(ranges, _) => + ranges.foreach: (lo, hi) => + points += lo + if hi < MaxUnit then points += hi + 1 + case _ => () + points.toList.sorted + + private def classCount(bounds: Ls[Int]): Int = bounds.size + 1 + + /** A representative code unit for each class. */ + private def classRepresentative(bounds: Ls[Int], classId: Int): Int = + if classId == 0 then 0 else bounds(classId - 1) + + private def rangesContain(ranges: Ls[CharRange], unit: Int): Bool = + ranges.exists((lo, hi) => lo <= unit && unit <= hi) + + /** Subset-construct the determinized *reverse* automaton. State `R_i` of a + * run over positions n..0 is the set of NFA states that can reach the + * accept state by consuming `input[i..n)`. The forward walk then only + * follows transitions into states proved viable by this oracle. + * + * @return (transition table by [revState * nClasses + class], the subset + * of NFA states denoted by each reverse state in id order, seed + * state id) + */ + private def reverseDeterminize(accept: Int, bounds: Ls[Int]): (Buffer[Int], Ls[Set[Int]], Int) = + val stateCount = states.size + val classes = classCount(bounds) + // Reverse ε-adjacency: epsPre(t) lists the sources of ε-edges into t. + val epsPre = Array.fill(stateCount)(Buffer.empty[Int]) + // Reverse char-adjacency per class: chrPre(c)(t) lists sources of + // class-c character edges into t. + val chrPre = Array.fill(classes)(MutMap.empty[Int, Buffer[Int]]) + states.iterator.zipWithIndex.foreach: (edges, source) => + edges.foreach: + case Edge.Eps(target, _) => epsPre(target) += source + case Edge.Chr(ranges, target) => + var classId = 0 + while classId < classes do + if rangesContain(ranges, classRepresentative(bounds, classId)) then + chrPre(classId).getOrElseUpdate(target, Buffer.empty) += source + classId += 1 + def close(seed: Set[Int]): Set[Int] = + val worklist = Buffer.from(seed) + val result = MutSet.from(seed) + while worklist.nonEmpty do + val state = worklist.remove(worklist.size - 1) + epsPre(state).foreach: source => + if result.add(source) then worklist += source + result.toSet + val ids = LinkedHashMap.empty[Set[Int], Int] + val worklist = Buffer.empty[Set[Int]] + def idOf(set: Set[Int]): Int = ids.getOrElseUpdate(set, { + worklist += set + ids.size + }) + val seed = close(Set(accept)) + val seedId = idOf(seed) + val transitions = Buffer.empty[Int] + while worklist.nonEmpty do + val set = worklist.remove(0) + val id = ids(set) + // Rows may be discovered out of order; grow the table as needed. + while transitions.size < (id + 1) * classes do transitions += 0 + var classId = 0 + while classId < classes do + val pre = chrPre(classId) + val next = close(set.iterator.flatMap(t => pre.get(t).iterator.flatMap(_.iterator)).toSet) + transitions(id * classes + classId) = idOf(next) + classId += 1 + (transitions, ids.keysIterator.toList, seedId) + + // ------------------------------------------------------------------------ + // Encoding + // ------------------------------------------------------------------------ + + private def encodeOps(ops: Ls[Op], pool: Buffer[Ls[Int]]): Int = + val encoded = ops.flatMap: + case Op.Mark => 0 :: Nil + case Op.Slice => 1 :: Nil + case Op.Add => 2 :: Nil + case Op.Drop => 3 :: Nil + case Op.Bind(slot) => 4 :: slot :: Nil + case Op.Call(actionId, argSlots) => 5 :: actionId :: argSlots.size :: argSlots + case Op.Enter => 6 :: Nil + case Op.Exit => 7 :: Nil + case Op.Rem => 8 :: Nil + case Op.Defer(deferred, captured) => + 9 :: encodeOps(deferred, pool) :: captured.size :: captured + pool.indexOf(encoded) match + case -1 => + pool += encoded + pool.size - 1 + case index => index + + /** The alphabet for bit-packed sections: six bits per character, highest + * bit first, using the standard Base64 characters — they need no escaping + * in string literals and clash with neither the `;` section separator nor + * the `,` integer separator. */ + private val packAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + + private def packBits(bits: Iterator[Bool]): Str = + val packed = new StringBuilder + var value = 0 + var count = 0 + bits.foreach: bit => + value = value * 2 + (if bit then 1 else 0) + count += 1 + if count == 6 then + packed += packAlphabet.charAt(value) + value = 0 + count = 0 + if count > 0 then + while count < 6 do + value = value * 2 + count += 1 + packed += packAlphabet.charAt(value) + packed.result() + + /** Encode the program as semicolon-separated sections of comma-separated + * integers (except the last section). The full parsing table: + * + * 0. header: stateCount, start, accept, slotCount, classCount, + * revStateCount, seedRevState + * 1. class boundaries (classCount - 1 integers) + * 2. NFA: per state: edgeCount, then per edge either + * 0, target, rangeCount, lo, hi, ... (character edge) or + * 1, target, opsId (ε-edge, -1 when it carries no operations) + * 3. operation pool: entryCount, then per entry: length, integers + * 4. reverse transitions (revStateCount * classCount integers) + * 5. viability: one bit per (revState, state) pair, row-major, + * bit-packed six per character (see `packAlphabet`) + * + * Recognition alone runs only the reverse scan, so `matchWhole` call + * sites embed a much smaller table instead: + * + * 0. header: classCount, seedRevState + * 1. class boundaries + * 2. reverse transitions + * 3. one '0'/'1' character per reverse state: whether its subset + * contains the start state (the whole-match acceptance test) + * + * String literals are interned by the JavaScript engine, so the runtime + * caches the decoded program keyed by this very string: the table is built + * once per program, not once per call. + * + * @return (the full parsing table, the recognition-only table) + */ + private def encode(start: Int, accept: Int): (Str, Str) = + val bounds = computeBounds() + val (revTransitions, revSets, seedId) = reverseDeterminize(accept, bounds) + val classes = classCount(bounds) + val opsPool = Buffer.empty[Ls[Int]] + val nfa = Buffer.empty[Int] + states.foreach: edges => + nfa += edges.size + edges.foreach: + case Edge.Chr(ranges, target) => + nfa += 0 + nfa += target + nfa += ranges.size + ranges.foreach: (lo, hi) => + nfa += lo + nfa += hi + case Edge.Eps(target, ops) => + nfa += 1 + nfa += target + nfa += (if ops.isEmpty then -1 else encodeOps(ops, opsPool)) + val opsSection = Buffer.empty[Int] + opsSection += opsPool.size + opsPool.foreach: entry => + opsSection += entry.size + opsSection ++= entry + val stateCount = states.size + val viability = packBits: + revSets.iterator.flatMap: set => + (0 until stateCount).iterator.map(set.contains) + val header = stateCount :: start :: accept :: slots.size :: classes :: + revSets.size :: seedId :: Nil + val table = Iterator( + header.mkString(","), + bounds.mkString(","), + nfa.mkString(","), + opsSection.mkString(","), + revTransitions.mkString(","), + viability, + ).mkString(";") + val starts = revSets.iterator.map(set => if set contains start then '1' else '0').mkString + val matchTable = Iterator( + s"$classes,$seedId", + bounds.mkString(","), + revTransitions.mkString(","), + starts, + ).mkString(";") + (table, matchTable) + + // ------------------------------------------------------------------------ + // Entry point + // ------------------------------------------------------------------------ + + /** Compile a string region rooted at `root`. Returns `N` when the region + * cannot be compiled (an error has been reported and the caller should + * compile nothing). */ + def compile(root: Pat, mode: Mode): Opt[Compiled] = scoped("ucs:string-compiler"): + collectBodies(root) + computeSccs() + checkTailPositions() + if failed then N else + val rootIsPure = isPureDeep(root) + // Pre-allocate slots for the root-visible bindings so the caller can + // rely on their presence even if some binding branches are dead. + val visible = root.symbols.map(symbol => symbol -> slotOf(symbol)) + val accept = newState() + val entry = mode match + case Mode.Whole => + build(root, accept, needValue = !rootIsPure, Nil, N) + case Mode.Prefix => + // A prefix match is a whole match of `root ~ lazy-Σ*`: the trailing + // star prefers not to consume, so the split point is exactly the + // committed greedy parse of `root` alone. The boundary is recorded + // as the start of the remaining string. + val star = newState() + addEps(star, accept, Nil) + addChr(star, AnyChar, star) + val boundary = newState() + addEps(boundary, star, Op.Rem :: Nil) + build(root, boundary, needValue = true, Nil, N) + if failed then N else + val builtCount = states.size + val (reducedEntry, reducedAccept) = reduceStates(entry, accept) + log(s"String region: ${states.size} states (built $builtCount), " + + s"${slots.size} slots, ${actions.size} actions") + val (table, matchTable) = encode(reducedEntry, reducedAccept) + log(s"Encoded table (${table.length} characters): $table") + log(s"Encoded match table (${matchTable.length} characters): $matchTable") + S(Compiled(table, matchTable, actions.toList, visible, + pure = !anyOps && slots.isEmpty && actions.isEmpty)) diff --git a/hkmc2/shared/src/test/mlscript-compile/Runtime.mjs b/hkmc2/shared/src/test/mlscript-compile/Runtime.mjs index eb9f1fbaf2..bb08e5eef4 100644 --- a/hkmc2/shared/src/test/mlscript-compile/Runtime.mjs +++ b/hkmc2/shared/src/test/mlscript-compile/Runtime.mjs @@ -5,55 +5,55 @@ import RuntimeJS from "./RuntimeJS.mjs"; import Rendering from "./Rendering.mjs"; import LazyArray from "./LazyArray.mjs"; import Iter from "./Iter.mjs"; -let Runtime1, lambda, lambda1, lambda2, lambda$, lambda$1, Capture$scope301, lambda$2, Capture$scope321, lambda$3; -(class Capture$scope32 { +let Runtime1, lambda, lambda1, lambda2, lambda3, lambda4, lambda5, lambda6, lambda7, lambda8, lambda9, lambda$, Capture$scope201, lambda$1, lambda$2, lambda$3, lambda$4, lambda$5, Capture$scope691, lambda$6, Capture$scope711, lambda$7; +(class Capture$scope71 { static { - Capture$scope321 = this + Capture$scope711 = this } constructor(result$0) { this.result$0 = result$0; } toString() { return runtime.render(this); } - static [definitionMetadata] = ["class", "Capture$scope32"]; + static [definitionMetadata] = ["class", "Capture$scope71"]; }); -lambda$3 = (undefined, function (scope32$cap, cont) { +lambda$7 = (undefined, function (scope71$cap, cont) { return (m, marker) => { - return lambda2(scope32$cap, cont, m, marker) + return lambda2(scope71$cap, cont, m, marker) } }); -lambda2 = (undefined, function (scope32$cap, cont, m, marker) { +lambda2 = (undefined, function (scope71$cap, cont, m, marker) { let scrut, tmp, tmp1; scrut = runtime.safeCall(m.has(cont)); if (scrut === true) { tmp = ", " + marker; - tmp1 = scope32$cap.result$0 + tmp; - scope32$cap.result$0 = tmp1; + tmp1 = scope71$cap.result$0 + tmp; + scope71$cap.result$0 = tmp1; return runtime.Unit } return runtime.Unit; }); -(class Capture$scope30 { +(class Capture$scope69 { static { - Capture$scope301 = this + Capture$scope691 = this } constructor(result$0) { this.result$0 = result$0; } toString() { return runtime.render(this); } - static [definitionMetadata] = ["class", "Capture$scope30"]; + static [definitionMetadata] = ["class", "Capture$scope69"]; }); -lambda$2 = (undefined, function (scope30$cap, cont) { +lambda$6 = (undefined, function (scope69$cap, cont) { return (m, marker) => { - return lambda1(scope30$cap, cont, m, marker) + return lambda1(scope69$cap, cont, m, marker) } }); -lambda1 = (undefined, function (scope30$cap, cont, m, marker) { +lambda1 = (undefined, function (scope69$cap, cont, m, marker) { let scrut, tmp, tmp1; scrut = runtime.safeCall(m.has(cont)); if (scrut === true) { tmp = ", " + marker; - tmp1 = scope30$cap.result$0 + tmp; - scope30$cap.result$0 = tmp1; + tmp1 = scope69$cap.result$0 + tmp; + scope69$cap.result$0 = tmp1; return runtime.Unit } return runtime.Unit; @@ -64,12 +64,257 @@ lambda = (undefined, function (l) { tmp1 = runtime.safeCall(Rendering.render(l.value)); return tmp + tmp1 }); -lambda$1 = (undefined, function (Runtime2) { +lambda$5 = (undefined, function (Runtime2) { return (k) => { Runtime2.stackResume = k; return runtime.Unit } }); +(class Capture$scope20 { + static { + Capture$scope201 = this + } + constructor(cur$0, pos$1, remStart$2) { + this.remStart$2 = remStart$2; + this.pos$1 = pos$1; + this.cur$0 = cur$0; + } + toString() { return runtime.render(this); } + static [definitionMetadata] = ["class", "Capture$scope20"]; +}); +lambda$4 = (undefined, function (bindings) { + return (slot, overlay) => { + return lambda4(bindings, slot, overlay) + } +}); +lambda$3 = (undefined, function (scope20$cap, actions, input, valStack, markStack, bindings, readSlot) { + return (ops, k, overlay) => { + return lambda5(scope20$cap, actions, input, valStack, markStack, bindings, readSlot, ops, k, overlay) + } +}); +lambda$2 = (undefined, function (execValueOp) { + return (ops, overlay) => { + return lambda6(execValueOp, ops, overlay) + } +}); +lambda$1 = (undefined, function (prog, frames, bindings, execValueOp, runFrameOps) { + return (opsId) => { + return lambda7(prog, frames, bindings, execValueOp, runFrameOps, opsId) + } +}); +lambda4 = (undefined, function (bindings, slot, overlay) { + let tmp, tmp1; + tmp = overlay !== null; + if (tmp === true) { + tmp1 = runtime.safeCall(overlay.has(slot)); + if (tmp1 === true) { + return runtime.safeCall(overlay.get(slot)) + } + return runtime.safeCall(bindings.get(slot)); + } + return runtime.safeCall(bindings.get(slot)); +}); +lambda5 = (undefined, function (scope20$cap, actions, input, valStack, markStack, bindings, readSlot, ops, k, overlay) { + let op, b, a, actionId, argCount, args, j, tmp, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, tmp9; + op = ops.at(k); + switch (op) { + case 0: + runtime.safeCall(markStack.push(scope20$cap.pos$1)); + return k + 1; + case 1: + tmp = runtime.safeCall(markStack.pop()); + tmp1 = runtime.safeCall(input.slice(tmp, scope20$cap.pos$1)); + runtime.safeCall(valStack.push(tmp1)); + return k + 1; + case 2: + b = runtime.safeCall(valStack.pop()); + a = runtime.safeCall(valStack.pop()); + tmp2 = a + b; + runtime.safeCall(valStack.push(tmp2)); + return k + 1; + case 3: + runtime.safeCall(valStack.pop()); + return k + 1; + case 4: + tmp3 = k + 1; + tmp4 = valStack.length - 1; + runtime.safeCall(bindings.set(ops.at(tmp3), valStack.at(tmp4))); + return k + 2; + case 5: + tmp5 = k + 1; + actionId = ops.at(tmp5); + tmp6 = k + 2; + argCount = ops.at(tmp6); + args = []; + j = 0; + lbl: while (true) { + let scrut, tmp10, tmp11, tmp12, tmp13; + scrut = j < argCount; + if (scrut === true) { + tmp10 = k + 3; + tmp11 = tmp10 + j; + tmp12 = runtime.safeCall(readSlot(ops.at(tmp11), overlay)); + runtime.safeCall(args.push(tmp12)); + tmp13 = j + 1; + j = tmp13; + continue lbl + } + break; + } + tmp7 = runtime.safeCall(actions.at(actionId).apply(null, args)); + runtime.safeCall(valStack.push(tmp7)); + tmp8 = k + 3; + return tmp8 + argCount; + case 8: + scope20$cap.remStart$2 = scope20$cap.pos$1; + return k + 1; + } + tmp9 = "StrPat: malformed operation " + op; + throw runtime.safeCall(globalThis.Error(tmp9)) +}); +lambda6 = (undefined, function (execValueOp, ops, overlay) { + let k; + k = 0; + lbl: while (true) { + let scrut, tmp; + scrut = k < ops.length; + if (scrut === true) { + tmp = runtime.safeCall(execValueOp(ops, k, overlay)); + k = tmp; + continue lbl + } + break; + } + return runtime.Unit +}); +lambda7 = (undefined, function (prog, frames, bindings, execValueOp, runFrameOps, opsId) { + let ops, k; + ops = prog.opsPool.at(opsId); + k = 0; + lbl: while (true) { + let scrut, op, frame, deferredOps, capCount, captures, j, tmp, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + scrut = k < ops.length; + if (scrut === true) { + op = ops.at(k); + switch (op) { + case 6: + runtime.safeCall(frames.push(null)); + tmp = k + 1; + k = tmp; + continue lbl; + case 7: + frame = runtime.safeCall(frames.pop()); + lbl1: while (true) { + let scrut1, tmp8; + scrut1 = frame !== null; + if (scrut1 === true) { + runtime.safeCall(runFrameOps(prog.opsPool.at(frame.at(0)), frame.at(1))); + tmp8 = runtime.safeCall(frames.pop()); + frame = tmp8; + continue lbl1 + } + break; + } + tmp1 = k + 1; + k = tmp1; + continue lbl; + case 9: + tmp2 = k + 1; + deferredOps = ops.at(tmp2); + tmp3 = k + 2; + capCount = ops.at(tmp3); + captures = globalThis.Object.freeze(new globalThis.Map()); + j = 0; + lbl2: while (true) { + let scrut1, slot, tmp8, tmp9, tmp10, tmp11; + scrut1 = j < capCount; + if (scrut1 === true) { + tmp8 = k + 3; + tmp9 = tmp8 + j; + slot = ops.at(tmp9); + tmp10 = runtime.safeCall(bindings.get(slot)); + runtime.safeCall(captures.set(slot, tmp10)); + tmp11 = j + 1; + j = tmp11; + continue lbl2 + } + break; + } + tmp4 = globalThis.Object.freeze([ + deferredOps, + captures + ]); + runtime.safeCall(frames.push(tmp4)); + tmp5 = 3 + capCount; + tmp6 = k + tmp5; + k = tmp6; + continue lbl; + } + tmp7 = runtime.safeCall(execValueOp(ops, k, null)); + k = tmp7; + continue lbl + } + break; + } + return runtime.Unit +}); +lambda8 = (undefined, function (scope20$cap, parentState, parentOps, runOps, state) { + let chain, w, idx; + chain = []; + w = state; + lbl: while (true) { + let scrut, opsId, scrut1, tmp; + scrut = w !== scope20$cap.cur$0; + if (scrut === true) { + opsId = runtime.safeCall(parentOps.get(w)); + scrut1 = opsId !== -1; + if (scrut1 === true) { + runtime.safeCall(chain.push(opsId)); + } + tmp = runtime.safeCall(parentState.get(w)); + w = tmp; + continue lbl + } + break; + } + idx = chain.length - 1; + lbl1: while (true) { + let scrut, tmp; + scrut = idx >= 0; + if (scrut === true) { + runtime.safeCall(runOps(chain.at(idx))); + tmp = idx - 1; + idx = tmp; + continue lbl1 + } + break; + } + return runtime.Unit +}); +lambda9 = (undefined, function (prog, stack, state) { + let edges, i; + edges = prog.states.at(state); + i = edges.length - 1; + lbl: while (true) { + let scrut, tmp, tmp1; + scrut = i >= 0; + if (scrut === true) { + tmp = globalThis.Object.freeze([ + state, + edges.at(i) + ]); + runtime.safeCall(stack.push(tmp)); + tmp1 = i - 1; + i = tmp1; + continue lbl + } + break; + } + return runtime.Unit +}); +lambda3 = (undefined, function (x) { + return runtime.safeCall(globalThis.parseInt(x, 10)) +}); lambda$ = (undefined, function (Runtime2, EffectHandle1, value) { return () => { return Runtime2.resume(EffectHandle1.reified.contTrace)(value) @@ -273,6 +518,491 @@ lambda$ = (undefined, function (Runtime2, EffectHandle1, value) { toString() { return runtime.render(this); } static [definitionMetadata] = ["class", "Str"]; }); + (class StrPat { + static { + Runtime.StrPat = this + } + static { + let tmp, tmp1; + StrPat.Program = function Program(stateCount, start, accept, slotCount, classCount, seedRev, bounds, states, opsPool, revTrans, viability) { + return globalThis.Object.freeze(new Program.class(stateCount, start, accept, slotCount, classCount, seedRev, bounds, states, opsPool, revTrans, viability)); + }; + (class Program { + static { + StrPat.Program.class = this + } + constructor(stateCount, start, accept, slotCount, classCount, seedRev, bounds, states, opsPool, revTrans, viability) { + this.stateCount = stateCount; + this.start = start; + this.accept = accept; + this.slotCount = slotCount; + this.classCount = classCount; + this.seedRev = seedRev; + this.bounds = bounds; + this.states = states; + this.opsPool = opsPool; + this.revTrans = revTrans; + this.viability = viability; + } + toString() { return runtime.render(this); } + static [definitionMetadata] = ["class", "Program", ["stateCount", "start", "accept", "slotCount", "classCount", "seedRev", "bounds", "states", "opsPool", "revTrans", "viability"]]; + }); + StrPat.Matcher = function Matcher(classCount, seedRev, bounds, revTrans, starts) { + return globalThis.Object.freeze(new Matcher.class(classCount, seedRev, bounds, revTrans, starts)); + }; + (class Matcher { + static { + StrPat.Matcher.class = this + } + constructor(classCount, seedRev, bounds, revTrans, starts) { + this.classCount = classCount; + this.seedRev = seedRev; + this.bounds = bounds; + this.revTrans = revTrans; + this.starts = starts; + } + toString() { return runtime.render(this); } + static [definitionMetadata] = ["class", "Matcher", ["classCount", "seedRev", "bounds", "revTrans", "starts"]]; + }); + tmp = globalThis.Object.freeze(new globalThis.Map()); + StrPat.programs = tmp; + tmp1 = globalThis.Object.freeze(new globalThis.Map()); + StrPat.matchers = tmp1; + } + static decodeInts(s) { + let scrut, tmp; + scrut = s.length === 0; + if (scrut === true) { + return globalThis.Object.freeze([]) + } + tmp = runtime.safeCall(s.split(",")); + return runtime.safeCall(tmp.map(lambda3)); + } + static sextet(code) { + let scrut, scrut1, scrut2, scrut3; + scrut = code >= 97; + if (scrut === true) { + return code - 71 + } + scrut1 = code >= 65; + if (scrut1 === true) { + return code - 65 + } + scrut2 = code >= 48; + if (scrut2 === true) { + return code + 4 + } + scrut3 = code === 43; + if (scrut3 === true) { + return 62 + } + return 63; + } + static decodeBits(packed) { + let thresholds, bits, i; + thresholds = globalThis.Object.freeze([ + 32, + 16, + 8, + 4, + 2, + 1 + ]); + bits = []; + i = 0; + lbl: while (true) { + let scrut, v, j, tmp, tmp1; + scrut = i < packed.length; + if (scrut === true) { + tmp = runtime.safeCall(packed.charCodeAt(i)); + v = StrPat.sextet(tmp); + j = 0; + lbl1: while (true) { + let scrut1, scrut2, tmp2, tmp3; + scrut1 = j < 6; + if (scrut1 === true) { + scrut2 = v >= thresholds.at(j); + if (scrut2 === true) { + runtime.safeCall(bits.push(true)); + tmp2 = v - thresholds.at(j); + v = tmp2; + } else { + runtime.safeCall(bits.push(false)); + } + tmp3 = j + 1; + j = tmp3; + continue lbl1 + } + break; + } + tmp1 = i + 1; + i = tmp1; + continue lbl + } + break; + } + return bits + } + static decodeProgram(table) { + let sections, header, stateCount, nfaInts, states, cursor, opsInts, opsPool, opsCount, tmp, tmp1, tmp2; + sections = runtime.safeCall(table.split(";")); + header = StrPat.decodeInts(sections.at(0)); + stateCount = header.at(0); + nfaInts = StrPat.decodeInts(sections.at(2)); + states = []; + cursor = 0; + lbl: while (true) { + let scrut, edgeCount, edges, tmp3; + scrut = states.length < stateCount; + if (scrut === true) { + edgeCount = nfaInts.at(cursor); + edges = []; + tmp3 = cursor + 1; + cursor = tmp3; + lbl1: while (true) { + let scrut1, kind, target, scrut2, rangeCount, tmp4, tmp5, tmp6, tmp7, tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15, tmp16, tmp17; + scrut1 = edges.length < edgeCount; + if (scrut1 === true) { + kind = nfaInts.at(cursor); + tmp4 = cursor + 1; + target = nfaInts.at(tmp4); + scrut2 = kind === 0; + if (scrut2 === true) { + tmp5 = cursor + 2; + rangeCount = nfaInts.at(tmp5); + tmp6 = cursor + 3; + tmp7 = cursor + 3; + tmp8 = rangeCount * 2; + tmp9 = tmp7 + tmp8; + tmp10 = runtime.safeCall(nfaInts.slice(tmp6, tmp9)); + tmp11 = globalThis.Object.freeze([ + 0, + target, + tmp10 + ]); + runtime.safeCall(edges.push(tmp11)); + tmp12 = rangeCount * 2; + tmp13 = 3 + tmp12; + tmp14 = cursor + tmp13; + cursor = tmp14; + continue lbl1 + } + tmp15 = cursor + 2; + tmp16 = globalThis.Object.freeze([ + 1, + target, + nfaInts.at(tmp15) + ]); + runtime.safeCall(edges.push(tmp16)); + tmp17 = cursor + 3; + cursor = tmp17; + continue lbl1; + } + break; + } + runtime.safeCall(states.push(edges)); + continue lbl + } + break; + } + opsInts = StrPat.decodeInts(sections.at(3)); + opsPool = []; + opsCount = opsInts.at(0); + cursor = 1; + lbl1: while (true) { + let scrut, len, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + scrut = opsPool.length < opsCount; + if (scrut === true) { + len = opsInts.at(cursor); + tmp3 = cursor + 1; + tmp4 = cursor + 1; + tmp5 = tmp4 + len; + tmp6 = runtime.safeCall(opsInts.slice(tmp3, tmp5)); + runtime.safeCall(opsPool.push(tmp6)); + tmp7 = 1 + len; + tmp8 = cursor + tmp7; + cursor = tmp8; + continue lbl1 + } + break; + } + tmp = StrPat.decodeInts(sections.at(1)); + tmp1 = StrPat.decodeInts(sections.at(4)); + tmp2 = StrPat.decodeBits(sections.at(5)); + return StrPat.Program(stateCount, header.at(1), header.at(2), header.at(3), header.at(4), header.at(6), tmp, states, opsPool, tmp1, tmp2) + } + static getProgram(table) { + let scrut, prog; + scrut = runtime.safeCall(StrPat.programs.has(table)); + if (scrut === true) { + return runtime.safeCall(StrPat.programs.get(table)) + } + prog = StrPat.decodeProgram(table); + runtime.safeCall(StrPat.programs.set(table, prog)); + return prog; + } + static decodeMatcher(table) { + let sections, header, tmp, tmp1; + sections = runtime.safeCall(table.split(";")); + header = StrPat.decodeInts(sections.at(0)); + tmp = StrPat.decodeInts(sections.at(1)); + tmp1 = StrPat.decodeInts(sections.at(2)); + return StrPat.Matcher(header.at(0), header.at(1), tmp, tmp1, sections.at(3)) + } + static getMatcher(table) { + let scrut, matcher; + scrut = runtime.safeCall(StrPat.matchers.has(table)); + if (scrut === true) { + return runtime.safeCall(StrPat.matchers.get(table)) + } + matcher = StrPat.decodeMatcher(table); + runtime.safeCall(StrPat.matchers.set(table, matcher)); + return matcher; + } + static classOf(prog, unit) { + let bounds, i; + bounds = prog.bounds; + i = 0; + lbl: while (true) { + let tmp, tmp1, tmp2; + tmp = i < bounds.length; + if (tmp === true) { + tmp1 = bounds.at(i) <= unit; + if (tmp1 === true) { + tmp2 = i + 1; + i = tmp2; + continue lbl + } + } + break; + } + return i + } + static viable(prog, revState, state) { + let tmp, tmp1; + tmp = revState * prog.stateCount; + tmp1 = tmp + state; + return prog.viability.at(tmp1) + } + static unitInRanges(ranges, unit) { + let i; + i = 0; + lbl: while (true) { + let scrut, tmp, tmp1, tmp2, tmp3; + scrut = i < ranges.length; + if (scrut === true) { + tmp = ranges.at(i) <= unit; + if (tmp === true) { + tmp2 = i + 1; + tmp1 = unit <= ranges.at(tmp2); + if (tmp1 === true) { + return true + } + tmp3 = i + 2; + i = tmp3; + continue lbl; + } + tmp3 = i + 2; + i = tmp3; + continue lbl; + } + break; + } + return false + } + static matchWhole(table, input) { + let matcher, rev, i, tmp; + matcher = StrPat.getMatcher(table); + rev = matcher.seedRev; + i = input.length - 1; + lbl: while (true) { + let scrut, tmp1, tmp2, tmp3, tmp4, tmp5; + scrut = i >= 0; + if (scrut === true) { + tmp1 = rev * matcher.classCount; + tmp2 = runtime.safeCall(input.charCodeAt(i)); + tmp3 = StrPat.classOf(matcher, tmp2); + tmp4 = tmp1 + tmp3; + rev = matcher.revTrans.at(tmp4); + tmp5 = i - 1; + i = tmp5; + continue lbl + } + break; + } + tmp = runtime.safeCall(matcher.starts.charCodeAt(rev)); + return tmp === 49 + } + static parseRun(table, actions, input, prefix) { + let prog, n, revArr, rev, i, scrut, gen, visited, parentState, parentOps, valStack, markStack, frames, bindings, readSlot, execValueOp, runFrameOps, scrut1, result, s, tmp, tmp1, scope20$cap; + scope20$cap = new Capture$scope201(undefined, undefined, undefined); + prog = StrPat.getProgram(table); + n = input.length; + revArr = []; + rev = prog.seedRev; + runtime.safeCall(revArr.push(rev)); + i = n - 1; + lbl: while (true) { + let scrut2, tmp2, tmp3, tmp4, tmp5, tmp6; + scrut2 = i >= 0; + if (scrut2 === true) { + tmp2 = rev * prog.classCount; + tmp3 = runtime.safeCall(input.charCodeAt(i)); + tmp4 = StrPat.classOf(prog, tmp3); + tmp5 = tmp2 + tmp4; + rev = prog.revTrans.at(tmp5); + runtime.safeCall(revArr.push(rev)); + tmp6 = i - 1; + i = tmp6; + continue lbl + } + break; + } + scrut = StrPat.viable(prog, revArr.at(n), prog.start); + if (scrut === false) { + return null + } + scope20$cap.cur$0 = prog.start; + scope20$cap.pos$1 = 0; + gen = 0; + visited = globalThis.Object.freeze(new globalThis.Map()); + parentState = globalThis.Object.freeze(new globalThis.Map()); + parentOps = globalThis.Object.freeze(new globalThis.Map()); + valStack = []; + markStack = []; + frames = []; + bindings = globalThis.Object.freeze(new globalThis.Map()); + scope20$cap.remStart$2 = n; + readSlot = lambda$4(bindings); + execValueOp = lambda$3(scope20$cap, actions, input, valStack, markStack, bindings, readSlot); + runFrameOps = lambda$2(execValueOp); + lbl1: while (true) { + let stack, committed, tmp2, tmp3, tmp4; + tmp2 = scope20$cap.cur$0 === prog.accept; + if (tmp2 === true) { + tmp3 = scope20$cap.pos$1 === n; + } else { + tmp3 = false; + } + if (tmp3 === false) { + let state; + tmp4 = gen + 1; + gen = tmp4; + runtime.safeCall(visited.set(scope20$cap.cur$0, tmp4)); + stack = []; + state = scope20$cap.cur$0; + lambda9(prog, stack, state); + committed = false; + lbl2: while (true) { + let scrut2, item, source, edge, scrut3, target, scrut4, scrut5, tmp5, tmp6, tmp7, tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + if (committed === false) { + scrut2 = stack.length === 0; + if (scrut2 === true) { + throw runtime.safeCall(globalThis.Error("StrPat: no viable transition (this is a compiler bug)")) + } + item = runtime.safeCall(stack.pop()); + source = item.at(0); + edge = item.at(1); + scrut3 = edge.at(0) === 0; + if (scrut3 === true) { + tmp5 = scope20$cap.pos$1 < n; + if (tmp5 === true) { + tmp7 = runtime.safeCall(input.charCodeAt(scope20$cap.pos$1)); + tmp6 = StrPat.unitInRanges(edge.at(2), tmp7); + if (tmp6 === true) { + tmp9 = scope20$cap.pos$1 + 1; + tmp10 = n - tmp9; + tmp8 = StrPat.viable(prog, revArr.at(tmp10), edge.at(1)); + if (tmp8 === true) { + let runOps; + runOps = lambda$1(prog, frames, bindings, execValueOp, runFrameOps); + lambda8(scope20$cap, parentState, parentOps, runOps, source); + scope20$cap.cur$0 = edge.at(1); + tmp11 = scope20$cap.pos$1 + 1; + scope20$cap.pos$1 = tmp11; + committed = true; + continue lbl2 + } + continue lbl2; + } + continue lbl2; + } + continue lbl2; + } + target = edge.at(1); + scrut4 = target === prog.accept; + if (scrut4 === true) { + scrut5 = scope20$cap.pos$1 === n; + if (scrut5 === true) { + let runOps; + runtime.safeCall(parentState.set(target, source)); + runtime.safeCall(parentOps.set(target, edge.at(2))); + runOps = lambda$1(prog, frames, bindings, execValueOp, runFrameOps); + lambda8(scope20$cap, parentState, parentOps, runOps, target); + scope20$cap.cur$0 = target; + committed = true; + continue lbl2 + } + continue lbl2; + } + tmp12 = runtime.safeCall(visited.get(target)); + tmp13 = tmp12 !== tmp4; + if (tmp13 === true) { + tmp15 = n - scope20$cap.pos$1; + tmp14 = StrPat.viable(prog, revArr.at(tmp15), target); + if (tmp14 === true) { + runtime.safeCall(visited.set(target, tmp4)); + runtime.safeCall(parentState.set(target, source)); + runtime.safeCall(parentOps.set(target, edge.at(2))); + lambda9(prog, stack, target); + continue lbl2 + } + continue lbl2; + } + continue lbl2; + } + break; + } + continue lbl1 + } + break; + } + scrut1 = valStack.length > 0; + if (scrut1 === true) { + tmp = runtime.safeCall(valStack.pop()); + } else { + tmp = input; + } + result = []; + runtime.safeCall(result.push(tmp)); + if (prefix === true) { + tmp1 = runtime.safeCall(input.slice(scope20$cap.remStart$2)); + runtime.safeCall(result.push(tmp1)); + } + s = 0; + lbl2: while (true) { + let scrut2, tmp2, tmp3; + scrut2 = s < prog.slotCount; + if (scrut2 === true) { + tmp2 = runtime.safeCall(bindings.get(s)); + runtime.safeCall(result.push(tmp2)); + tmp3 = s + 1; + s = tmp3; + continue lbl2 + } + break; + } + return result; + } + static parseWhole(table, actions, input) { + return StrPat.parseRun(table, actions, input, false) + } + static parsePrefix(table, actions, input) { + return StrPat.parseRun(table, actions, input, true) + } + toString() { return runtime.render(this); } + static [definitionMetadata] = ["class", "StrPat"]; + }); Runtime.render = Rendering.render; (class TraceLogger { static { @@ -578,7 +1308,7 @@ lambda$ = (undefined, function (Runtime2, EffectHandle1, value) { } delay() { let lambda$here; - lambda$here = lambda$1(Runtime); + lambda$here = lambda$5(Runtime); return Runtime.mkEffect(this, lambda$here) } toString() { return runtime.render(this); } @@ -916,12 +1646,12 @@ lambda$ = (undefined, function (Runtime2, EffectHandle1, value) { return header; } static showFunctionContChain(cont, hl, vis, reps) { - let scrut, scrut1, scrut2, tmp, tmp1, tmp2, tmp3, tmp4, scope30$cap, lambda$here; - scope30$cap = new Capture$scope301(undefined); + let scrut, scrut1, scrut2, tmp, tmp1, tmp2, tmp3, tmp4, scope69$cap, lambda$here; + scope69$cap = new Capture$scope691(undefined); if (cont instanceof Runtime.FunctionContFrame.class) { tmp = cont.constructor.name + "(pc="; - scope30$cap.result$0 = tmp + cont.saved.at(1); - lambda$here = lambda$2(scope30$cap, cont); + scope69$cap.result$0 = tmp + cont.saved.at(1); + lambda$here = lambda$6(scope69$cap, cont); runtime.safeCall(hl.forEach(lambda$here)); scrut = runtime.safeCall(vis.has(cont)); if (scrut === true) { @@ -931,12 +1661,12 @@ lambda$ = (undefined, function (Runtime2, EffectHandle1, value) { if (scrut1 === true) { throw runtime.safeCall(globalThis.Error("10 repeated continuation frame (loop?)")) } - tmp2 = scope30$cap.result$0 + ", REPEAT"; - scope30$cap.result$0 = tmp2; + tmp2 = scope69$cap.result$0 + ", REPEAT"; + scope69$cap.result$0 = tmp2; } else { runtime.safeCall(vis.add(cont)); } - tmp3 = scope30$cap.result$0 + ") -> "; + tmp3 = scope69$cap.result$0 + ") -> "; tmp4 = Runtime.showFunctionContChain(cont.next, hl, vis, reps); return tmp3 + tmp4 } @@ -947,11 +1677,11 @@ lambda$ = (undefined, function (Runtime2, EffectHandle1, value) { return "(NOT CONT)"; } static showHandlerContChain(cont, hl, vis, reps) { - let scrut, scrut1, scrut2, tmp, tmp1, tmp2, tmp3, scope32$cap, lambda$here; - scope32$cap = new Capture$scope321(undefined); + let scrut, scrut1, scrut2, tmp, tmp1, tmp2, tmp3, scope71$cap, lambda$here; + scope71$cap = new Capture$scope711(undefined); if (cont instanceof Runtime.HandlerContFrame.class) { - scope32$cap.result$0 = cont.handler.constructor.name; - lambda$here = lambda$3(scope32$cap, cont); + scope71$cap.result$0 = cont.handler.constructor.name; + lambda$here = lambda$7(scope71$cap, cont); runtime.safeCall(hl.forEach(lambda$here)); scrut = runtime.safeCall(vis.has(cont)); if (scrut === true) { @@ -961,12 +1691,12 @@ lambda$ = (undefined, function (Runtime2, EffectHandle1, value) { if (scrut1 === true) { throw runtime.safeCall(globalThis.Error("10 repeated continuation frame (loop?)")) } - tmp1 = scope32$cap.result$0 + ", REPEAT"; - scope32$cap.result$0 = tmp1; + tmp1 = scope71$cap.result$0 + ", REPEAT"; + scope71$cap.result$0 = tmp1; } else { runtime.safeCall(vis.add(cont)); } - tmp2 = scope32$cap.result$0 + " -> "; + tmp2 = scope71$cap.result$0 + " -> "; tmp3 = Runtime.showFunctionContChain(cont.next, hl, vis, reps); return tmp2 + tmp3 } diff --git a/hkmc2/shared/src/test/mlscript-compile/Runtime.mls b/hkmc2/shared/src/test/mlscript-compile/Runtime.mls index 7ca96f1e33..f4040462b7 100644 --- a/hkmc2/shared/src/test/mlscript-compile/Runtime.mls +++ b/hkmc2/shared/src/test/mlscript-compile/Runtime.mls @@ -111,22 +111,355 @@ module Tuple with val split = LazyArray.__split module Str with - + @mayNotRaiseEffects fun startsWith(string, prefix) = string.startsWith(prefix) - + @mayNotRaiseEffects fun get(string, i) = if i >= string.length then throw RangeError("Str.get: index out of bounds") else string.at(i) - + @mayNotRaiseEffects fun take(string, n) = string.slice(0, n) - + @mayNotRaiseEffects fun leave(string, n) = string.slice(n) +// The engine for compiled string patterns. The compiler (`ups.StringCompiler`) +// encodes a prioritized NFA together with its determinized reverse automaton +// as a string of integers; this module decodes and runs such programs. Refer +// to `StringCompiler.scala` for the table layouts and the algorithm. In short: +// matching follows the two-pass scheme of Frisch and Cardelli. A backward scan +// of the reverse automaton computes, for every position, the set of NFA states +// that can still reach acceptance ("viability"). Recognition needs only that +// scan. To produce outputs and bindings, a forward walk then follows, at every +// choice point, the first viable alternative — reproducing the parse an +// unlimited-backtracking matcher would commit to, in linear time — while +// executing the value operations recorded on the taken ε-transitions. +// +// There are two table formats: full parsing tables (decoded to `Program`, +// run by `parseWhole`/`parsePrefix`) and recognition-only tables that keep +// just the reverse scan (decoded to `Matcher`, run by `matchWhole`). +module StrPat with + + data class Program( + stateCount, start, accept, slotCount, classCount, seedRev, + bounds, states, opsPool, revTrans, viability) + + // A recognition-only program: the reverse scan needs neither the NFA, nor + // the operation pool, nor the viability matrix. `starts` records, for each + // reverse state, whether the subset it denotes contains the NFA start + // state — the whole-match acceptance test. + data class Matcher(classCount, seedRev, bounds, revTrans, starts) + + // Decoded programs are cached keyed by the (interned) table string, so each + // table is decoded once per program, not once per call. + val programs = new Map() + val matchers = new Map() + + fun decodeInts(s) = + if s.length === 0 then [] else s.split(",").map(x => parseInt(x, 10)) + + // The value of a character of the Base64 alphabet used for bit-packed + // sections (six bits per character, highest bit first). + fun sextet(code) = + if code >= 97 then code - 71 // 'a'..'z' denote 26..51 + else if code >= 65 then code - 65 // 'A'..'Z' denote 0..25 + else if code >= 48 then code + 4 // '0'..'9' denote 52..61 + else if code === 43 then 62 // '+' + else 63 // '/' + + // Unpack a bit-packed section into an array of booleans (possibly with up + // to five padding entries at the end, which are never indexed). + fun decodeBits(packed) = + let thresholds = [32, 16, 8, 4, 2, 1] + let bits = mut [] + let i = 0 + while i < packed.length do + let v = sextet(packed.charCodeAt(i)) + let j = 0 + while j < 6 do + if v >= thresholds.[j] then + bits.push(true) + set v -= thresholds.[j] + else + bits.push(false) + set j += 1 + set i += 1 + bits + + fun decodeProgram(table) = + let sections = table.split(";") + let header = decodeInts(sections.[0]) + let stateCount = header.[0] + let nfaInts = decodeInts(sections.[2]) + let states = mut [] + let cursor = 0 + while states.length < stateCount do + let edgeCount = nfaInts.[cursor] + let edges = mut [] + set cursor += 1 + while edges.length < edgeCount do + let kind = nfaInts.[cursor] + let target = nfaInts.[cursor + 1] + if kind === 0 then + let rangeCount = nfaInts.[cursor + 2] + edges.push([0, target, nfaInts.slice(cursor + 3, cursor + 3 + rangeCount * 2)]) + set cursor += 3 + rangeCount * 2 + else + edges.push([1, target, nfaInts.[cursor + 2]]) + set cursor += 3 + states.push(edges) + let opsInts = decodeInts(sections.[3]) + let opsPool = mut [] + let opsCount = opsInts.[0] + set cursor = 1 + while opsPool.length < opsCount do + let len = opsInts.[cursor] + opsPool.push(opsInts.slice(cursor + 1, cursor + 1 + len)) + set cursor += 1 + len + Program( + stateCount, header.[1], header.[2], header.[3], header.[4], header.[6], + decodeInts(sections.[1]), states, opsPool, decodeInts(sections.[4]), + decodeBits(sections.[5])) + + fun getProgram(table) = + if programs.has(table) then programs.get(table) + else + let prog = decodeProgram(table) + programs.set(table, prog) + prog + + fun decodeMatcher(table) = + let sections = table.split(";") + let header = decodeInts(sections.[0]) + Matcher(header.[0], header.[1], + decodeInts(sections.[1]), decodeInts(sections.[2]), sections.[3]) + + fun getMatcher(table) = + if matchers.has(table) then matchers.get(table) + else + let matcher = decodeMatcher(table) + matchers.set(table, matcher) + matcher + + // The index of the character class of the given code unit. Works on both + // programs and matchers (it only reads their `bounds`). + fun classOf(prog, unit) = + let bounds = prog.bounds + let i = 0 + while i < bounds.length && bounds.[i] <= unit do + set i += 1 + i + + // Is `state` in the co-reachable set denoted by `revState`? + fun viable(prog, revState, state) = + prog.viability.[revState * prog.stateCount + state] + + fun unitInRanges(ranges, unit) = + let i = 0 + while i < ranges.length do + if ranges.[i] <= unit && unit <= ranges.[i + 1] do return true + set i += 2 + false + + // Whether the program matches the whole input: a single right-to-left scan + // of the determinized reverse automaton, with no allocation. Takes a + // recognition-only table, not a full parsing table. + @mayNotRaiseEffects + fun matchWhole(table, input) = + let matcher = getMatcher(table) + let rev = matcher.seedRev + let i = input.length - 1 + while i >= 0 do + set rev = matcher.revTrans.[rev * matcher.classCount + classOf(matcher, input.charCodeAt(i))] + set i -= 1 + matcher.starts.charCodeAt(rev) === 49 + + // The committed forward walk. Returns null when the input does not match; + // otherwise returns [output, slot values...] (or, in prefix mode, + // [output, remaining, slot values...]). + fun parseRun(table, actions, input, prefix) = + let prog = getProgram(table) + let n = input.length + // revArr.[k] is the reverse-automaton state after consuming the last k + // characters; it encodes the viable NFA states at position n - k. + let revArr = mut [] + let rev = prog.seedRev + revArr.push(rev) + let i = n - 1 + while i >= 0 do + set rev = prog.revTrans.[rev * prog.classCount + classOf(prog, input.charCodeAt(i))] + revArr.push(rev) + set i -= 1 + if viable(prog, revArr.[n], prog.start) is false do return null + let cur = prog.start + let pos = 0 + let gen = 0 + // Per-round visit marks and the parent links of the current search, used + // to reconstruct the ε-path to the chosen transition. + let visited = new Map() + let parentState = new Map() + let parentOps = new Map() + let valStack = mut [] + let markStack = mut [] + let frames = mut [] + let bindings = new Map() + let remStart = n + let readSlot = (slot, overlay) => + if overlay !== null && overlay.has(slot) then overlay.get(slot) else bindings.get(slot) + // Execute the single value operation at offset `k` of `ops` and return + // the offset of the next one. When executing a deferred frame, `overlay` + // holds the slot values captured when the frame was created, so that each + // recursion level reads its own bindings. Writes always go to the global + // store: they are correctly sequenced by the execution order (frames + // unwind innermost-first, and each frame reads a slot it wrote before any + // other frame overwrites it), and bindings visible to the caller — such + // as one on a region root ending in a recursive pattern — must land in + // the global store. + let execValueOp = (ops, k, overlay) => + let op = ops.[k] + if op is + 0 then // Mark: remember the current position + markStack.push(pos) + k + 1 + 1 then // Slice: push the input since the latest mark + valStack.push(input.slice(markStack.pop(), pos)) + k + 1 + 2 then // Add: concatenate the two topmost values + let b = valStack.pop() + let a = valStack.pop() + valStack.push(a + b) + k + 1 + 3 then // Drop + valStack.pop() + k + 1 + 4 then // Bind: store the topmost value into a slot + bindings.set(ops.[k + 1], valStack.[valStack.length - 1]) + k + 2 + 5 then // Call: apply a transform to the values in its argument slots + let actionId = ops.[k + 1] + let argCount = ops.[k + 2] + let args = mut [] + let j = 0 + while j < argCount do + args.push(readSlot(ops.[k + 3 + j], overlay)) + set j += 1 + valStack.push(actions.[actionId].apply(null, args)) + k + 3 + argCount + 8 then // Rem: the remaining input starts here + set remStart = pos + k + 1 + else + throw Error("StrPat: malformed operation " + op) + // Deferred frames only carry value operations: the compiler never defers + // Enter/Exit/Defer themselves. + let runFrameOps = (ops, overlay) => + let k = 0 + while k < ops.length do + set k = execValueOp(ops, k, overlay) + // Operations attached to ε-edges of the committed path. + let runOps = (opsId) => + let ops = prog.opsPool.[opsId] + let k = 0 + while k < ops.length do + let op = ops.[k] + if op is + 6 then // Enter: delimit the deferred frames of a recursive component + frames.push(null) + set k += 1 + 7 then // Exit: unwind the deferred frames, innermost first + let frame = frames.pop() + while frame !== null do + runFrameOps(prog.opsPool.[frame.[0]], frame.[1]) + set frame = frames.pop() + set k += 1 + 9 then // Defer: capture the listed slots and postpone the operations + let deferredOps = ops.[k + 1] + let capCount = ops.[k + 2] + let captures = new Map() + let j = 0 + while j < capCount do + let slot = ops.[k + 3 + j] + captures.set(slot, bindings.get(slot)) + set j += 1 + frames.push([deferredOps, captures]) + set k += 3 + capCount + else + set k = execValueOp(ops, k, null) + // Execute the operations of the ε-path from the search root to `state`. + let execChain = (state) => + let chain = mut [] + let w = state + while w !== cur do + let opsId = parentOps.get(w) + if opsId !== -1 do chain.push(opsId) + set w = parentState.get(w) + let idx = chain.length - 1 + while idx >= 0 do + runOps(chain.[idx]) + set idx -= 1 + let pushEdges = (stack, state) => + let edges = prog.states.[state] + let i = edges.length - 1 + while i >= 0 do + stack.push([state, edges.[i]]) + set i -= 1 + while (cur === prog.accept && pos === n) is false do + set gen += 1 + visited.set(cur, gen) + let stack = mut [] + pushEdges(stack, cur) + let committed = false + while committed is false do + if stack.length === 0 do + // The upfront viability check guarantees a path to acceptance, and + // the search only moves to viable states, so this cannot happen. + throw Error("StrPat: no viable transition (this is a compiler bug)") + let item = stack.pop() + let source = item.[0] + let edge = item.[1] + if edge.[0] === 0 then + // A character edge: the first viable consumption in priority order + // is the committed one. + if pos < n && unitInRanges(edge.[2], input.charCodeAt(pos)) + && viable(prog, revArr.[n - (pos + 1)], edge.[1]) do + execChain(source) + set cur = edge.[1] + set pos += 1 + set committed = true + else + let target = edge.[1] + if target === prog.accept then + // Acceptance is only viable once the input is exhausted (in + // prefix mode, the trailing lazy star pads the difference). + if pos === n do + parentState.set(target, source) + parentOps.set(target, edge.[2]) + execChain(target) + set cur = target + set committed = true + else if visited.get(target) !== gen && viable(prog, revArr.[n - pos], target) do + visited.set(target, gen) + parentState.set(target, source) + parentOps.set(target, edge.[2]) + pushEdges(stack, target) + let output = if valStack.length > 0 then valStack.pop() else input + let result = mut [] + result.push(output) + if prefix do result.push(input.slice(remStart)) + let s = 0 + while s < prog.slotCount do + result.push(bindings.get(s)) + set s += 1 + result + + fun parseWhole(table, actions, input) = parseRun(table, actions, input, false) + + fun parsePrefix(table, actions, input) = parseRun(table, actions, input, true) + // Re-export rendering functions val render = Rendering.render diff --git a/hkmc2/shared/src/test/mlscript/ucs/patterns/Compilation.mls b/hkmc2/shared/src/test/mlscript/ucs/patterns/Compilation.mls index 537eacf7e6..2e24978ecf 100644 --- a/hkmc2/shared/src/test/mlscript/ucs/patterns/Compilation.mls +++ b/hkmc2/shared/src/test/mlscript/ucs/patterns/Compilation.mls @@ -7,9 +7,6 @@ open annotations :fixme fun trimStart(str) = if str is @compile { (" " | "\t") ~ rest } then trimStart(rest) else str -//│ ╔══[COMPILATION ERROR] String concatenation is not supported in pattern compilation. -//│ ║ l.9: if str is @compile { (" " | "\t") ~ rest } then trimStart(rest) else str -//│ ╙── ^^^^^^^^^^^^^^^^^^ //│ ╔══[COMPILATION ERROR] No definition found in scope for member 'rest' //│ ║ l.9: if str is @compile { (" " | "\t") ~ rest } then trimStart(rest) else str //│ ╙── ^^^^ @@ -17,21 +14,15 @@ fun trimStart(str) = :fixme fun trimStart(str) = if str is (@compile ((" " | "\t") ~ rest)) then trimStart(rest) else str -//│ ╔══[COMPILATION ERROR] String concatenation is not supported in pattern compilation. -//│ ║ l.19: if str is (@compile ((" " | "\t") ~ rest)) then trimStart(rest) else str -//│ ╙── ^^^^^^^^^^^^^^^^^^ //│ ╔══[COMPILATION ERROR] No definition found in scope for member 'rest' -//│ ║ l.19: if str is (@compile ((" " | "\t") ~ rest)) then trimStart(rest) else str +//│ ║ l.16: if str is (@compile ((" " | "\t") ~ rest)) then trimStart(rest) else str //│ ╙── ^^^^ :fixme fun trimStart(str) = if str is @compile ((" " | "\t") ~ rest) then trimStart(rest) else str -//│ ╔══[COMPILATION ERROR] String concatenation is not supported in pattern compilation. -//│ ║ l.29: if str is @compile ((" " | "\t") ~ rest) then trimStart(rest) else str -//│ ╙── ^^^^^^^^^^^^^^^^^^ //│ ╔══[COMPILATION ERROR] No definition found in scope for member 'rest' -//│ ║ l.29: if str is @compile ((" " | "\t") ~ rest) then trimStart(rest) else str +//│ ║ l.23: if str is @compile ((" " | "\t") ~ rest) then trimStart(rest) else str //│ ╙── ^^^^ @@ -41,16 +32,16 @@ fun trimStart(str) = fun trimStart(str) = if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str //│ ╔══[PARSE ERROR] Expected start of expression in this position -//│ ║ l.42: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str +//│ ║ l.33: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str //│ ║ ^ //│ ╟── found a lone annotation instead -//│ ║ l.42: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str +//│ ║ l.33: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str //│ ╙── ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ //│ ╔══[COMPILATION ERROR] Unrecognized pattern (‹erroneous syntax›). -//│ ║ l.42: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str +//│ ║ l.33: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str //│ ╙── ^ //│ ╔══[COMPILATION ERROR] Name not found: rest -//│ ║ l.42: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str +//│ ║ l.33: if str is (@compile((" " | "\t") ~ rest)) then trimStart(rest) else str //│ ╙── ^^^^ diff --git a/hkmc2/shared/src/test/mlscript/ups/UpsBugsBacklog.mls b/hkmc2/shared/src/test/mlscript/ups/UpsBugsBacklog.mls index 75f3fbb801..f37b84e2c1 100644 --- a/hkmc2/shared/src/test/mlscript/ups/UpsBugsBacklog.mls +++ b/hkmc2/shared/src/test/mlscript/ups/UpsBugsBacklog.mls @@ -10,9 +10,12 @@ import "../../mlscript-compile/Char.mls" pattern Input = ((Char.Whitespace ~ (Input as inp)) => inp) | ((c ~ (Input as inp)) => [c, ..inp]) | ("" => []) -:fixme +// Note: the wildcard `c` is greedy — it consumes as much as the rest of the +// sequence permits — so it takes the whole input here (`Input` accepts the +// empty remainder). This used to overflow the stack in the backtracking +// translation; string patterns now compile to automata and always terminate. if "a b c" is Input as chars then chars -//│ ═══[RUNTIME ERROR] RangeError: Maximum call stack size exceeded +//│ = [a b c] // ——— ——— ——— @@ -47,7 +50,7 @@ if tokens is @compile Parse as stack do print(stack) if tokens is @compile Parse as stack do print(stack) //│ ╔══[COMPILATION ERROR] No definition found in scope for member 'stack' -//│ ║ l.48: @compile Parse as stack do print(stack) +//│ ║ l.51: @compile Parse as stack do print(stack) //│ ╙── ^^^^^ // What's actually intended: diff --git a/hkmc2/shared/src/test/mlscript/ups/regex/CompiledSemantics.mls b/hkmc2/shared/src/test/mlscript/ups/regex/CompiledSemantics.mls new file mode 100644 index 0000000000..9f4e78a43c --- /dev/null +++ b/hkmc2/shared/src/test/mlscript/ups/regex/CompiledSemantics.mls @@ -0,0 +1,88 @@ +:js + +// This file pins down the semantics of compiled string patterns. +// +// Recognition is relational: a sequence matches when *some* split of the +// scrutinee matches, regardless of how greedy sub-patterns are. Among all +// successful parses, the committed one — which determines bindings, transform +// applications, and outputs — is the parse that a recursive-descent matcher +// with unlimited backtracking would find first, trying alternatives from left +// to right. Transforms run exactly once, and only on the committed parse. + + +// Alternation is ordered: the leftmost alternative that lets the rest of the +// sequence succeed wins, even if another alternative would match more. + +pattern Split = ((("a" | "ab") as x) ~ (("c" | "bc") as y)) => [x, y] + +:expect ["a", "bc"] +if "abc" is Split as parts then parts +//│ = ["a", "bc"] + +// The first alternative is committed whenever it lets the parse complete — +// even when it consumes everything. + +:expect ["abd", ""] +if "abd" is (((("a" ~ "b" ~ "d") | "ab") as x) ~ (("" | "d") as y)) then [x, y] +//│ = ["abd", ""] + + +// The wildcard consumes greedily: it takes as much as the rest of the +// sequence permits. + +pattern LastB = ((_ as x) ~ ("b" as y)) => [x, y] + +:expect ["aba", "b"] +if "abab" is LastB as parts then parts +//│ = ["aba", "b"] + +// `Str` in string position means "any string", just like the wildcard. (The +// backtracking translation used to consume exactly one character here.) + +pattern Bracketed = ("[" ~ (Str as s) ~ "]") => s + +:expect "abc" +if "[abc]" is Bracketed as s then s +//│ = "abc" + +:expect "" +if "[]" is Bracketed as s then s +//│ = "" + + +// Transforms run exactly once, and only on the committed parse: a transform +// in an abandoned alternative or a failed parse never executes. (The +// backtracking translation used to run transforms on parses it later +// abandoned.) + +fun probe(x) = if x is ("a" => print("ran")) ~ "b" then "yes" else "no" + +probe("ab") +//│ > ran +//│ = "yes" + +// The prefix "a" matches, but the whole sequence fails: no transform runs. +probe("ax") +//│ = "no" + + +// Bindings work across the automaton boundary, including at inline use sites. + +fun trimStart(str) = if str is (" " | "\t") ~ rest then trimStart(rest) else str + +:expect "hello " +trimStart(" \t hello ") +//│ = "hello " + + +// Literals may span several UTF-16 code units. + +pattern Exclaimed = "😀" ~ "!" + +:expect true +"😀!" is Exclaimed +//│ = true + +:expect false +"😀?" is Exclaimed +//│ = false diff --git a/hkmc2/shared/src/test/mlscript/ups/regex/EmailAddress.mls b/hkmc2/shared/src/test/mlscript/ups/regex/EmailAddress.mls index 08a8e6af0c..38087954b2 100644 --- a/hkmc2/shared/src/test/mlscript/ups/regex/EmailAddress.mls +++ b/hkmc2/shared/src/test/mlscript/ups/regex/EmailAddress.mls @@ -4,6 +4,7 @@ import "../../../mlscript-compile/Char.mls" import "../../../mlscript-compile/Iter.mls" import "../../../mlscript-compile/Stack.mls" +open annotations open Iter { joined } // To match simple email addresses matched by the following regular expressions: @@ -78,7 +79,7 @@ let emails = [ ] emails - Iter.mapping of _ is Email + Iter.mapping of _ is @compile Email Iter.folded of true, _ && _ //│ = true @@ -89,8 +90,46 @@ pattern CommaSep(pattern S) = | (S as head) => head :: Nil | "" => Nil +:soir fun parseEmails(input) = - if input is CommaSep(Email) as emails then emails else Nil + if input is @compile CommaSep(Email) as emails then emails else Nil +//│ ——————————————| Optimized IR |—————————————————————————————————————————————————————————————————————— +//│ define parseEmails⁰ as fun parseEmails¹(input) { +//│ let inlinedVal, parseResult, stringOutput, lambda, lambda1, lambda2, tmp; +//│ match input +//│ Str⁰ => +//│ define lambda as fun lambda⁰(head, tail) { +//│ return Stack⁰.Cons⁰(head, tail) +//│ }; +//│ define lambda1 as fun lambda¹(head) { +//│ return Stack⁰.Cons⁰(head, Stack⁰.Nil⁰) +//│ }; +//│ define lambda2 as fun lambda²() { +//│ return Stack⁰.Nil⁰ +//│ }; +//│ set tmp = [lambda⁰, lambda¹, lambda²]; +//│ set parseResult = runtime⁰.StrPat⁰.parseWhole⁰("130,1,0,3,17,47,0;37,38,43,44,45,46,47,48,58,64,65,91,95,96,97,123;0,3,1,98,-1,1,128,-1,1,129,-1,1,1,0,0,3,1,35,-1,1,65,-1,1,66,-1,1,1,3,2,1,0,4,1,44,44,1,1,5,3,2,1,15,-1,1,6,-1,2,1,9,-1,1,7,-1,2,1,10,-1,1,11,-1,1,0,8,1,97,122,1,0,8,1,65,90,2,1,13,-1,1,14,-1,1,0,9,1,97,122,1,0,9,1,65,90,1,0,12,1,46,46,2,1,17,-1,1,15,-1,3,1,18,-1,1,21,-1,1,22,-1,2,1,19,-1,1,20,-1,1,0,16,1,97,122,1,0,16,1,65,90,1,0,16,1,48,57,1,0,16,1,45,45,1,0,17,1,64,64,2,1,25,-1,1,23,-1,7,1,26,-1,1,29,-1,1,30,-1,1,31,-1,1,32,-1,1,33,-1,1,34,-1,2,1,27,-1,1,28,-1,1,0,24,1,97,122,1,0,24,1,65,90,1,0,24,1,48,57,1,0,24,1,46,46,1,0,24,1,95,95,1,0,24,1,37,37,1,0,24,1,43,43,1,0,24,1,45,45,1,1,25,4,1,1,2,5,2,1,45,-1,1,36,-1,2,1,39,-1,1,37,-1,2,1,40,-1,1,41,-1,1,0,38,1,97,122,1,0,38,1,65,90,2,1,43,-1,1,44,-1,1,0,39,1,97,122,1,0,39,1,65,90,1,0,42,1,46,46,2,1,47,-1,1,45,-1,3,1,48,-1,1,51,-1,1,52,-1,2,1,49,-1,1,50,-1,1,0,46,1,97,122,1,0,46,1,65,90,1,0,46,1,48,57,1,0,46,1,45,45,1,0,47,1,64,64,2,1,55,-1,1,53,-1,7,1,56,-1,1,59,-1,1,60,-1,1,61,-1,1,62,-1,1,63,-1,1,64,-1,2,1,57,-1,1,58,-1,1,0,54,1,97,122,1,0,54,1,65,90,1,0,54,1,48,57,1,0,54,1,46,46,1,0,54,1,95,95,1,0,54,1,37,37,1,0,54,1,43,43,1,0,54,1,45,45,1,1,55,4,1,1,2,6,1,1,3,7,1,0,67,1,44,44,1,1,68,3,2,1,78,-1,1,69,-1,2,1,72,-1,1,70,-1,2,1,73,-1,1,74,-1,1,0,71,1,97,122,1,0,71,1,65,90,2,1,76,-1,1,77,-1,1,0,72,1,97,122,1,0,72,1,65,90,1,0,75,1,46,46,2,1,80,-1,1,78,-1,3,1,81,-1,1,84,-1,1,85,-1,2,1,82,-1,1,83,-1,1,0,79,1,97,122,1,0,79,1,65,90,1,0,79,1,48,57,1,0,79,1,45,45,1,0,80,1,64,64,2,1,88,-1,1,86,-1,7,1,89,-1,1,92,-1,1,93,-1,1,94,-1,1,95,-1,1,96,-1,1,97,-1,2,1,90,-1,1,91,-1,1,0,87,1,97,122,1,0,87,1,65,90,1,0,87,1,48,57,1,0,87,1,46,46,1,0,87,1,95,95,1,0,87,1,37,37,1,0,87,1,43,43,1,0,87,1,45,45,1,1,88,4,1,1,0,5,2,1,108,-1,1,99,-1,2,1,102,-1,1,100,-1,2,1,103,-1,1,104,-1,1,0,101,1,97,122,1,0,101,1,65,90,2,1,106,-1,1,107,-1,1,0,102,1,97,122,1,0,102,1,65,90,1,0,105,1,46,46,2,1,110,-1,1,108,-1,3,1,111,-1,1,114,-1,1,115,-1,2,1,112,-1,1,113,-1,1,0,109,1,97,122,1,0,109,1,65,90,1,0,109,1,48,57,1,0,109,1,45,45,1,0,110,1,64,64,2,1,118,-1,1,116,-1,7,1,119,-1,1,122,-1,1,123,-1,1,124,-1,1,125,-1,1,126,-1,1,127,-1,2,1,120,-1,1,121,-1,1,0,117,1,97,122,1,0,117,1,65,90,1,0,117,1,48,57,1,0,117,1,46,46,1,0,117,1,95,95,1,0,117,1,37,37,1,0,117,1,43,43,1,0,117,1,45,45,1,1,118,4,1,1,0,6;8,1,7,8,4,1,3,5,0,2,0,1,4,9,1,1,0,4,1,4,0,3,1,0,8,1,4,2,3,5,1,1,2,3,5,2,0,5,6,9,1,1,0;1,1,1,1,2,1,1,1,1,1,1,3,1,1,1,4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,1,1,1,6,1,1,1,1,1,1,1,1,1,1,1,1,7,1,1,1,8,1,1,1,1,1,1,1,1,1,1,1,1,7,1,1,1,8,1,1,1,1,1,1,1,1,1,1,1,1,9,1,1,1,10,1,1,1,1,1,1,1,1,1,1,1,1,9,1,1,1,10,1,1,1,1,1,1,1,11,1,1,1,1,7,1,1,1,8,1,1,1,1,1,1,1,11,1,1,1,1,7,1,1,1,8,1,1,1,1,1,1,1,12,1,1,1,1,9,1,1,1,10,1,1,1,1,1,1,1,12,1,1,1,1,9,1,1,1,10,1,1,1,1,1,1,13,1,1,14,1,1,15,1,1,1,16,1,1,1,1,1,1,17,1,1,18,1,1,19,1,1,1,20,1,1,1,1,1,1,13,1,1,14,1,21,22,1,1,1,23,1,1,1,1,1,1,13,1,1,14,1,21,22,1,1,1,23,1,1,1,1,1,1,13,1,1,14,1,21,24,1,1,1,25,1,1,1,1,1,1,13,1,1,14,1,21,24,1,1,1,25,1,1,1,1,1,1,17,1,1,18,1,26,27,1,1,1,28,1,1,1,1,1,1,17,1,1,18,1,26,27,1,1,1,28,1,1,1,1,1,1,17,1,1,18,1,26,29,1,1,1,30,1,1,1,1,1,1,17,1,1,18,1,26,29,1,1,1,30,1,1,31,1,32,1,33,34,1,35,1,1,36,1,37,1,38,1,1,1,1,1,1,13,1,1,14,1,21,22,1,1,1,23,1,1,1,1,1,1,13,1,1,14,1,21,22,1,1,1,23,1,1,1,1,1,1,13,11,1,14,1,21,24,1,1,1,25,1,1,1,1,1,1,13,11,1,14,1,21,24,1,1,1,25,1,1,39,1,40,1,41,42,1,43,1,1,44,1,45,1,46,1,1,1,1,1,1,17,1,1,18,1,26,27,1,1,1,28,1,1,1,1,1,1,17,1,1,18,1,26,27,1,1,1,28,1,1,1,1,1,1,17,12,1,18,1,26,29,1,1,1,30,1,1,1,1,1,1,17,12,1,18,1,26,29,1,1,1,30,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,31,1,32,2,33,34,1,35,1,1,36,1,37,1,38,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1,1,39,1,40,2,41,42,1,43,1,1,44,1,45,1,46,1;+AAAAA4AAAAwAAAAHAAAAEAAAAAAAAAAAAAAAAAAAAAAeAAAAAAAAADwAAAAAAAAAAAAAAANAAAAAAAAABoAAAAAAAAAA4AAAAAAAAAHAAAAADQAAAAAAAAAaAAAAAAAAAAOAAAAAAAAABwAAAAAAAAAAAAAAA2gAAAAAAAAG0AAAAAAAAADsAAAAAAAAAdgAAAANoAAAAAAAABtAAAAAAAAAA7AAAAAAAAAHYAAAAAAAAAAAAAAGBgAAAAAAAAwMAAAAYGAAAAAAAADAwAAAAAAAAAAAAAAAMIAAAAAAAABhAAAAAAAAAAxAAAAAAAAAGIAAAAAAAANDoAAAAAAABodAAAAAAAAA4PAAAAAAAAHB4AAAAAwgAAAAAAAAGEAAAAAAAAADEAAAAAAAAAYgAAAAAAAA0OgAAAAAAAGh0AAAAAAAADg8AAAAAAAAcHgAAAAAAAAAAAAAAABgAAAAAAAAAMAAAAAAAAAOgAAAAAAAAB0AAAAAAAAAA8AAAAAAAAAHgAAAAAAAANroAAAAAAABtdAAAAAAAAA7PAAAAAAAAHZ4AAAAAAYAAAAAAAAADAAAAAAAAADoAAAAAAAAAdAAAAAAAAAAPAAAAAAAAAB4AAAAAAAADa6AAAAAAAAbXQAAAAAAAAOzwAAAAAAAB2eAAAAAAABYAAAAAAADAlAAAAAAAAYElgAAAAAAAMBUAAAAAAABgKWAAAAAAAAwDQAAAAAAAGAZYAAAAAAADCFAAAAAAAAYQlgAAAAAAAMQUAAAAAAABiCWAAAAAAAA6BQAAAAAAAHQJYAAAAAAADBFAAAAAAAAYIlgAAAAAAAPAUAAAAAAAB4CWAAAwJAAAAAQAAGBIAAAABYAADAUAAAABAAAYCgAAAAFgAAMAwAAAAEAABgGAAAAAWAAAwhAAAAAQAAGEIAAAABYAADEEAAAABAAAYggAAAAFgAAOgQAAAAEAAB0CAAAAAWAAAwRAAAAAQAAGCIAAAABYAADwEAAAABAAAeAgAAAAA", tmp, input); +//│ match parseResult +//│ Array(1+) => +//│ set stringOutput = runtime⁰.Tuple⁰.get⁰(parseResult, 0); +//│ set inlinedVal = new runtime⁰.MatchSuccess⁰(stringOutput, null); +//│ match inlinedVal +//│ MatchSuccess⁰ => +//│ return inlinedVal.output⁰ +//│ else +//│ return Stack⁰.Nil⁰ +//│ end +//│ else +//│ do new runtime⁰.MatchFailure⁰("string mismatch"); +//│ return Stack⁰.Nil⁰ +//│ end +//│ else +//│ do new runtime⁰.MatchFailure⁰("never"); +//│ return Stack⁰.Nil⁰ +//│ end +//│ }; +//│ end +//│ —————————————————| Output |————————————————————————————————————————————————————————————————————————— parseEmails(emails.join(",")) Iter.fromStack() Iter.toArray() //│ = [ diff --git a/hkmc2/shared/src/test/mlscript/ups/regex/EmptyString.mls b/hkmc2/shared/src/test/mlscript/ups/regex/EmptyString.mls index c04ae45652..0bd5590fd6 100644 --- a/hkmc2/shared/src/test/mlscript/ups/regex/EmptyString.mls +++ b/hkmc2/shared/src/test/mlscript/ups/regex/EmptyString.mls @@ -1,18 +1,57 @@ :js +open annotations + pattern Oops = "" ~ (Oops | "") -:re -:todo +// The nullable recursion used to diverge (a stack overflow) in the naive +// backtracking translation; the automaton cuts the ε-cycle and terminates. "" is Oops -//│ ═══[RUNTIME ERROR] RangeError: Maximum call stack size exceeded +//│ = true +:soir +"" is @compile Oops +//│ ——————————————| Optimized IR |—————————————————————————————————————————————————————————————————————— +//│ let inlinedVal; +//│ match "" +//│ Str⁰ => +//│ set inlinedVal = runtime⁰.StrPat⁰.matchWhole⁰("1,0;;1,1;10", ""); +//│ match inlinedVal +//│ true => +//│ return true +//│ else +//│ return false +//│ end +//│ else +//│ return false +//│ end +//│ —————————————————| Output |————————————————————————————————————————————————————————————————————————— +//│ = true pattern Funny = "" ~ "" ~ "" "" is Funny //│ = true +:soir +"" is @compile Funny +//│ ——————————————| Optimized IR |—————————————————————————————————————————————————————————————————————— +//│ let inlinedVal; +//│ match "" +//│ Str⁰ => +//│ set inlinedVal = runtime⁰.StrPat⁰.matchWhole⁰("1,0;;1,1;10", ""); +//│ match inlinedVal +//│ true => +//│ return true +//│ else +//│ return false +//│ end +//│ else +//│ return false +//│ end +//│ —————————————————| Output |————————————————————————————————————————————————————————————————————————— +//│ = true + pattern Funny = "" ~ "hello" ~ "" "hello" is Funny diff --git a/hkmc2/shared/src/test/mlscript/ups/regex/Identifier.mls b/hkmc2/shared/src/test/mlscript/ups/regex/Identifier.mls index dd52222cdf..40427b3a4e 100644 --- a/hkmc2/shared/src/test/mlscript/ups/regex/Identifier.mls +++ b/hkmc2/shared/src/test/mlscript/ups/regex/Identifier.mls @@ -135,15 +135,10 @@ pattern Word = Letter ~ (Word | "") "b0rked" is Word //│ = false -:e fun isWord(x) = x is @compile Word -//│ ╔══[COMPILATION ERROR] String concatenation is not supported in pattern compilation. -//│ ║ l.120: pattern Word = Letter ~ (Word | "") -//│ ╙── ^^^^^^^^^^^^^^^^^^^ -// Unsupported patterns are equivalent to `Never`. isWord of "pattern" -//│ = false +//│ = true pattern ManyDigits = ("0" ..= "9") ~ (ManyDigits | "") @@ -159,6 +154,11 @@ pattern ManyDigits = ("0" ..= "9") ~ (ManyDigits | "") "1234" is ManyDigits //│ = true +fun isManyDigits(str) = str is @compile ManyDigits + +isManyDigits("5678") +//│ = true + pattern Integer = "0" | ("1" ..= "9") ~ (ManyDigits | "") :expect true diff --git a/hkmc2/shared/src/test/mlscript/ups/regex/NonRegular.mls b/hkmc2/shared/src/test/mlscript/ups/regex/NonRegular.mls new file mode 100644 index 0000000000..ce087b0699 --- /dev/null +++ b/hkmc2/shared/src/test/mlscript/ups/regex/NonRegular.mls @@ -0,0 +1,81 @@ +:js + +// Recursive string patterns compile to finite automata, which is possible +// exactly when every recursive reference sits in tail position of a sequence +// (the grammar is right-linear, hence regular). Self-embedding patterns are +// rejected with an error pinning the culprit reference, and nothing is +// compiled: the pattern never matches. + +:e +pattern Parens = ("(" ~ Parens ~ ")") | "" +//│ ╔══[COMPILATION ERROR] This recursive use of pattern `Parens` is not in tail position. +//│ ║ l.10: pattern Parens = ("(" ~ Parens ~ ")") | "" +//│ ║ ^^^^^^ +//│ ╟── Only self references at the end of a string sequence can be compiled to a finite string matcher. +//│ ╟── The recursion involves this pattern. +//│ ║ l.10: pattern Parens = ("(" ~ Parens ~ ")") | "" +//│ ╙── ^^^^^^ + +"(())" is Parens +//│ = false + +// Left recursion is not right-linear either. (The backtracking translation +// used to loop forever on it.) + +:e +pattern Lefty = (Lefty ~ "a") | "b" +//│ ╔══[COMPILATION ERROR] This recursive use of pattern `Lefty` is not in tail position. +//│ ║ l.26: pattern Lefty = (Lefty ~ "a") | "b" +//│ ║ ^^^^^ +//│ ╟── Only self references at the end of a string sequence can be compiled to a finite string matcher. +//│ ╟── The recursion involves this pattern. +//│ ║ l.26: pattern Lefty = (Lefty ~ "a") | "b" +//│ ╙── ^^^^^ + +"baa" is Lefty +//│ = false + +// Mutual recursion is fine as long as all the references are in tail +// position. Here `Even` matches (aba)* and `Odd` matches a(aba)*. + +pattern Even = ("ab" ~ Odd) | "" +pattern Odd = "a" ~ Even + +:expect true +"a" is Odd +//│ = true + +:expect true +"aaba" is Odd +//│ = true + +:expect true +"abaaba" is Even +//│ = true + +:expect false +"ab" is Even +//│ = false + +// ... and rejected when one of them is not. + +:e +pattern Outer = ("a" ~ Inner ~ "c") | "" +pattern Inner = "b" ~ Outer +//│ ╔══[COMPILATION ERROR] This recursive use of pattern `Inner` is not in tail position. +//│ ║ l.63: pattern Outer = ("a" ~ Inner ~ "c") | "" +//│ ║ ^^^^^ +//│ ╟── Only self references at the end of a string sequence can be compiled to a finite string matcher. +//│ ╟── The recursion involves this pattern. +//│ ║ l.64: pattern Inner = "b" ~ Outer +//│ ╙── ^^^^^ +//│ ╔══[COMPILATION ERROR] This recursive use of pattern `Inner` is not in tail position. +//│ ║ l.63: pattern Outer = ("a" ~ Inner ~ "c") | "" +//│ ║ ^^^^^ +//│ ╟── Only self references at the end of a string sequence can be compiled to a finite string matcher. +//│ ╟── The recursion involves this pattern. +//│ ║ l.64: pattern Inner = "b" ~ Outer +//│ ╙── ^^^^^ + +"abc" is Outer +//│ = false diff --git a/hkmc2/shared/src/test/mlscript/ups/regex/Separation.mls b/hkmc2/shared/src/test/mlscript/ups/regex/Separation.mls index 24443a8e4e..e83fe68389 100644 --- a/hkmc2/shared/src/test/mlscript/ups/regex/Separation.mls +++ b/hkmc2/shared/src/test/mlscript/ups/regex/Separation.mls @@ -2,6 +2,8 @@ import "../../../mlscript-compile/Iter.mls" +open annotations + pattern TailLines(pattern L) = | ("\n" ~ (Lines(L) as t)) => t | "" => [] @@ -10,15 +12,15 @@ pattern Lines(pattern L) = | "" => [] :expect true -"hello" is Lines("hello") +"hello" is @compile Lines("hello") //│ = true :expect true -"hello\nworld" is Lines("hello" | "world") +"hello\nworld" is @compile Lines("hello" | "world") //│ = true :expect false -"hello\nworld" is Lines("hello") +"hello\nworld" is @compile Lines("hello") //│ = false pattern Digit = "0" ..= "9" @@ -32,11 +34,11 @@ Integer.unapplyStringPrefix("123") //│ = MatchSuccess(["123", ""], null) // Beautiful! -"123\n456\n789" is Lines of Integer +"123\n456\n789" is @compile (Lines of Integer) //│ = true fun parseIntegers(input) = - if input is (Lines of (Integer as n) => parseInt(n, 10)) as n then n + if input is @compile (Lines of (Integer as n) => parseInt(n, 10)) as n then n parseIntegers of "123\n456\n789" //│ = [123, 456, 789] @@ -47,6 +49,6 @@ parseIntegers of "123\n456\n789" // `t through P` means match `t` against `P` and returns the result. "123\n456\n789" through Lines(Integer) // ==> [123, 456, 789] //│ ╔══[COMPILATION ERROR] Illegal juxtaposition right-hand side (identifier). -//│ ║ l.48: "123\n456\n789" through Lines(Integer) // ==> [123, 456, 789] +//│ ║ l.50: "123\n456\n789" through Lines(Integer) // ==> [123, 456, 789] //│ ╙── ^^^^^^^ //│ ═══[RUNTIME ERROR] TypeError: Lines1 is not a function diff --git a/hkmc2/shared/src/test/mlscript/ups/regex/Simplification.mls b/hkmc2/shared/src/test/mlscript/ups/regex/Simplification.mls index 2f4bebc00d..2602d1434f 100644 --- a/hkmc2/shared/src/test/mlscript/ups/regex/Simplification.mls +++ b/hkmc2/shared/src/test/mlscript/ups/regex/Simplification.mls @@ -17,28 +17,22 @@ pattern Xss = Xs ~ (Xss | "") pattern XsX = Xs ~ "x" -// In the following tests, the left-hand side of concatenation should not -// eagerly consume all letters. We should compile these string patterns even in -// naive compilation, rather than just transliterate them. +// In the following tests, the left-hand side of concatenation must not +// eagerly consume all letters. String patterns are compiled to automata whose +// splitting decisions are global to the whole sequence, so these all match. :expect true -:fixme "xxxxxxxxxxxxxx" is XsX -//│ ═══[RUNTIME ERROR] Expected: 'true', got: 'false' -//│ = false +//│ = true pattern XssX = Xss ~ "x" :expect true -:fixme "xxxxxxxx" is XssX -//│ ═══[RUNTIME ERROR] Expected: 'true', got: 'false' -//│ = false +//│ = true pattern XsXs = Xs ~ Xs :expect true -:fixme "xxxxxx" is XsXs -//│ ═══[RUNTIME ERROR] Expected: 'true', got: 'false' -//│ = false +//│ = true diff --git a/hkmc2/shared/src/test/mlscript/ups/specialization/SimpleLiterals.mls b/hkmc2/shared/src/test/mlscript/ups/specialization/SimpleLiterals.mls index 02aebb83b8..475005ee5d 100644 --- a/hkmc2/shared/src/test/mlscript/ups/specialization/SimpleLiterals.mls +++ b/hkmc2/shared/src/test/mlscript/ups/specialization/SimpleLiterals.mls @@ -48,10 +48,5 @@ fun checkSomeZeroOneTwo(x) = if x is pattern ManyZero = ("0" ~ (ManyZero | "")) | "" -:e -:todo let res = "1" is @compile ManyZero -//│ ╔══[COMPILATION ERROR] String concatenation is not supported in pattern compilation. -//│ ║ l.49: pattern ManyZero = ("0" ~ (ManyZero | "")) | "" -//│ ╙── ^^^^^^^^^^^^^^^^^^^^ //│ res = false