From 47f7d167efd0e3929e30e12af90e87e7de657b02 Mon Sep 17 00:00:00 2001 From: myfreess Date: Fri, 10 Jul 2026 11:55:37 +0800 Subject: [PATCH 1/2] add $$$ metavar prefix --- lexer/lexer.mbt | 100 +++++++++++++++--------- lexer/metavar_test.mbt | 167 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 232 insertions(+), 35 deletions(-) diff --git a/lexer/lexer.mbt b/lexer/lexer.mbt index 1ff21167..1fae8bd6 100644 --- a/lexer/lexer.mbt +++ b/lexer/lexer.mbt @@ -86,12 +86,19 @@ fn lex_tokens( ) } - fn typed_metavar_payload(name : StringView, kind : StringView) -> String { - "$(\{name}:\{kind})" + fn typed_metavar_payload( + prefix : StringView, + name : StringView, + kind : StringView, + ) -> String { + "\{prefix}(\{name}:\{kind})" } - fn inferred_metavar_payload(name : StringView) -> String { - "$\{name}" + fn inferred_metavar_payload( + prefix : StringView, + name : StringView, + ) -> String { + "\{prefix}\{name}" } lexmatch input with longest { @@ -731,8 +738,15 @@ fn lex_tokens( // Dot with identifier // Dot metavariable labels, only enabled for pattern parsing. + ("\.\$\$\([^\)\r\n]*\)" as raw, rest) => + add_metavar_error(raw, rest, '$', 1) + ("\.\$\$[A-Za-z_][A-Za-z0-9_]*" as raw, rest) => + add_metavar_error(raw, rest, '$', 1) + ("\.\$\$" as raw, rest) => add_metavar_error(raw, rest, '$', 1) ( - "\.\$\(" + "\." + ("\$(\$\$)?" as prefix) + "\(" "[ ]*" ("[A-Za-z_][A-Za-z0-9_]*" as name) "[ ]*" @@ -744,24 +758,33 @@ fn lex_tokens( rest ) => add_metavar_token( - @tokens.Token::DOT_LIDENT(typed_metavar_payload(name, kind)), + @tokens.Token::DOT_LIDENT(typed_metavar_payload(prefix, name, kind)), rest, '$', 1, ) - ("\.\$" ("[A-Za-z_][A-Za-z0-9_]*" as name), rest) => + ("\." ("\$(\$\$)?" as prefix) ("[A-Za-z_][A-Za-z0-9_]*" as name), rest) => add_metavar_token( - @tokens.Token::DOT_LIDENT(inferred_metavar_payload(name)), + @tokens.Token::DOT_LIDENT(inferred_metavar_payload(prefix, name)), rest, '$', 1, ) - ("\.\$\([ ]*:" as raw, rest) => add_metavar_error(raw, rest, '$', 1) - ("\.\$\([^\)\r\n]*\)" as raw, rest) => add_metavar_error(raw, rest, '$', 1) - ("\.\$\(" "[ ]*" "[A-Za-z_][A-Za-z0-9_]*" "[ ]*" ":" "[ ]*" "\)", rest) => - add_metavar_error_to_rest(rest, '$', 1) + ("\.\$(\$\$)?\([ ]*:" as raw, rest) => add_metavar_error(raw, rest, '$', 1) + ("\.\$(\$\$)?\([^\)\r\n]*\)" as raw, rest) => + add_metavar_error(raw, rest, '$', 1) + ( + "\.\$(\$\$)?\(" + "[ ]*" + "[A-Za-z_][A-Za-z0-9_]*" + "[ ]*" + ":" + "[ ]*" + "\)", + rest + ) => add_metavar_error_to_rest(rest, '$', 1) ( - "\.\$\(" + "\.\$(\$\$)?\(" "[ ]*" "[A-Za-z_][A-Za-z0-9_]*" "[ ]*" @@ -770,7 +793,7 @@ fn lex_tokens( "[a-z][a-z]*", rest ) => add_metavar_error_to_rest(rest, '$', 1) - ("\.\$[a-z][a-z]*:[a-z_][A-Za-z0-9_]*" as raw, rest) => + ("\.\$(\$\$)?[a-z][a-z]*:[a-z_][A-Za-z0-9_]*" as raw, rest) => add_metavar_error(raw, rest, '$', 1) ( "\." @@ -1115,8 +1138,14 @@ fn lex_tokens( } // Pattern metavariables, only enabled for pattern parsing. + ("\$\$\([^\)\r\n]*\)~?" as raw, rest) => + add_metavar_error(raw, rest, '$', 0) + ("\$\$[A-Za-z_][A-Za-z0-9_]*~?" as raw, rest) => + add_metavar_error(raw, rest, '$', 0) + ("\$\$" as raw, rest) => add_metavar_error(raw, rest, '$', 0) ( - "\$\(" + ("\$(\$\$)?" as prefix) + "\(" "[ ]*" ("[A-Za-z_][A-Za-z0-9_]*" as name) "[ ]*" @@ -1128,20 +1157,21 @@ fn lex_tokens( rest ) => add_metavar_token( - @tokens.Token::POST_LABEL(typed_metavar_payload(name, kind)), + @tokens.Token::POST_LABEL(typed_metavar_payload(prefix, name, kind)), rest, '$', 0, ) - ("\$" ("[A-Za-z_][A-Za-z0-9_]*" as name) "~", rest) => + (("\$(\$\$)?" as prefix) ("[A-Za-z_][A-Za-z0-9_]*" as name) "~", rest) => add_metavar_token( - @tokens.Token::POST_LABEL(inferred_metavar_payload(name)), + @tokens.Token::POST_LABEL(inferred_metavar_payload(prefix, name)), rest, '$', 0, ) ( - "\$\(" + ("\$(\$\$)?" as prefix) + "\(" "[ ]*" ("[A-Z][A-Za-z0-9_]*" as name) "[ ]*" @@ -1153,13 +1183,14 @@ fn lex_tokens( rest ) => add_metavar_token( - @tokens.Token::UIDENT(typed_metavar_payload(name, kind)), + @tokens.Token::UIDENT(typed_metavar_payload(prefix, name, kind)), rest, '$', 0, ) ( - "\$\(" + ("\$(\$\$)?" as prefix) + "\(" "[ ]*" ("[a-z_][A-Za-z0-9_]*" as name) "[ ]*" @@ -1171,31 +1202,32 @@ fn lex_tokens( rest ) => add_metavar_token( - @tokens.Token::LIDENT(typed_metavar_payload(name, kind)), + @tokens.Token::LIDENT(typed_metavar_payload(prefix, name, kind)), rest, '$', 0, ) - ("\$" ("[A-Z][A-Za-z0-9_]*" as name), rest) => + (("\$(\$\$)?" as prefix) ("[A-Z][A-Za-z0-9_]*" as name), rest) => add_metavar_token( - @tokens.Token::UIDENT(inferred_metavar_payload(name)), + @tokens.Token::UIDENT(inferred_metavar_payload(prefix, name)), rest, '$', 0, ) - ("\$" ("[a-z_][A-Za-z0-9_]*" as name), rest) => + (("\$(\$\$)?" as prefix) ("[a-z_][A-Za-z0-9_]*" as name), rest) => add_metavar_token( - @tokens.Token::LIDENT(inferred_metavar_payload(name)), + @tokens.Token::LIDENT(inferred_metavar_payload(prefix, name)), rest, '$', 0, ) - ("\$\([ ]*:" as raw, rest) => add_metavar_error(raw, rest, '$', 0) - ("\$\([^\)\r\n]*\)" as raw, rest) => add_metavar_error(raw, rest, '$', 0) - ("\$\(" "[ ]*" "[A-Za-z_][A-Za-z0-9_]*" "[ ]*" ":" "[ ]*" "\)", rest) => + ("\$(\$\$)?\([ ]*:" as raw, rest) => add_metavar_error(raw, rest, '$', 0) + ("\$(\$\$)?\([^\)\r\n]*\)" as raw, rest) => + add_metavar_error(raw, rest, '$', 0) + ("\$(\$\$)?\(" "[ ]*" "[A-Za-z_][A-Za-z0-9_]*" "[ ]*" ":" "[ ]*" "\)", rest) => add_metavar_error_to_rest(rest, '$', 0) ( - "\$\(" + "\$(\$\$)?\(" "[ ]*" "[A-Za-z_][A-Za-z0-9_]*" "[ ]*" @@ -1204,12 +1236,12 @@ fn lex_tokens( "[a-z][a-z]*", rest ) => add_metavar_error_to_rest(rest, '$', 0) - ("\$\(" "[ ]*" "[A-Za-z_][A-Za-z0-9_]*" "[ ]*" "\)", rest) => + ("\$(\$\$)?\(" "[ ]*" "[A-Za-z_][A-Za-z0-9_]*" "[ ]*" "\)", rest) => add_metavar_error_to_rest(rest, '$', 0) - ("\$\([ ]*\)" as raw, rest) => add_metavar_error(raw, rest, '$', 0) - ("\$[a-z][a-z]*:[A-Za-z_][A-Za-z0-9_]*~" as raw, rest) => + ("\$(\$\$)?\([ ]*\)" as raw, rest) => add_metavar_error(raw, rest, '$', 0) + ("\$(\$\$)?[a-z][a-z]*:[A-Za-z_][A-Za-z0-9_]*~" as raw, rest) => add_metavar_error(raw, rest, '$', 0) - ("\$[a-z][a-z]*:[a-z_][A-Za-z0-9_]*" as raw, rest) => + ("\$(\$\$)?[a-z][a-z]*:[a-z_][A-Za-z0-9_]*" as raw, rest) => add_metavar_error(raw, rest, '$', 0) // Multiline string #| diff --git a/lexer/metavar_test.mbt b/lexer/metavar_test.mbt index 2e2be853..5afc0241 100644 --- a/lexer/metavar_test.mbt +++ b/lexer/metavar_test.mbt @@ -1,3 +1,29 @@ +///| +fn assert_single_metavar_result( + result : LexResult, + source : String, + token_start : Int, +) -> Unit raise { + guard result.errors.length() == 0 else { + fail("expected no lexical errors for \{source}") + } + guard result.tokens.length() == 2 && result.tokens[1].0 is EOF else { + fail( + "expected exactly one metavariable token followed by EOF for \{source}", + ) + } + let (_, start, end) = result.tokens[0] + guard start.cnum == token_start && end.cnum == source.length() else { + fail( + "unexpected metavariable location for \{source}: \{start.cnum}..\{end.cnum}", + ) + } + let (_, eof_start, eof_end) = result.tokens[1] + guard eof_start.cnum == source.length() && eof_end.cnum == source.length() else { + fail("unexpected EOF location for \{source}") + } +} + ///| test "metavariable syntax is disabled by default" { let result = tokens_from_string("$x", comment=false) @@ -13,6 +39,23 @@ test "metavariable syntax is disabled by default" { guard dot.tokens[1].0 is LIDENT("field") else { fail("expected disabled dot metavariable syntax to resume after $") } + let triple = tokens_from_string("$$$x", comment=false) + guard triple.errors.length() == 3 else { + fail("expected each disabled $$$ prefix character to be rejected") + } + guard triple.tokens[0].0 is LIDENT("x") else { + fail("expected disabled $$$ metavariable syntax to resume after the prefix") + } + let dot_triple = tokens_from_string(".$$$field", comment=false) + guard dot_triple.errors.length() == 3 else { + fail("expected each disabled dot $$$ prefix character to be rejected") + } + guard dot_triple.tokens[0].0 is DOT_LIDENT("") && + dot_triple.tokens[1].0 is LIDENT("field") else { + fail( + "expected disabled dot $$$ metavariable syntax to recover after the prefix", + ) + } } ///| @@ -75,6 +118,83 @@ test "metavariable syntax encodes as ordinary tokens when enabled" { } } +///| +test "triple-dollar metavariable syntax preserves its prefix" { + let exp = tokens_from_string("$$$(x:exp)", comment=false, enable_metavar=true) + assert_single_metavar_result(exp, "$$$(x:exp)", 0) + guard exp.tokens[0].0 is LIDENT("$$$(x:exp)") else { + fail("expected LIDENT encoded typed $$$ metavariable") + } + let spaced = tokens_from_string( + "$$$( x : exp )", + comment=false, + enable_metavar=true, + ) + assert_single_metavar_result(spaced, "$$$( x : exp )", 0) + guard spaced.tokens[0].0 is LIDENT("$$$(x:exp)") else { + fail("expected typed $$$ metavariable payload to normalize whitespace") + } + let inferred = tokens_from_string("$$$x", comment=false, enable_metavar=true) + assert_single_metavar_result(inferred, "$$$x", 0) + guard inferred.tokens[0].0 is LIDENT("$$$x") else { + fail("expected LIDENT encoded untyped $$$ metavariable") + } + let inferred_type = tokens_from_string( + "$$$Foo", + comment=false, + enable_metavar=true, + ) + assert_single_metavar_result(inferred_type, "$$$Foo", 0) + guard inferred_type.tokens[0].0 is UIDENT("$$$Foo") else { + fail("expected UIDENT encoded untyped $$$ metavariable") + } + let type_name = tokens_from_string( + "$$$(Foo:type)", + comment=false, + enable_metavar=true, + ) + assert_single_metavar_result(type_name, "$$$(Foo:type)", 0) + guard type_name.tokens[0].0 is UIDENT("$$$(Foo:type)") else { + fail("expected UIDENT encoded typed $$$ metavariable") + } + let dot = tokens_from_string( + ".$$$(field:id)", + comment=false, + enable_metavar=true, + ) + assert_single_metavar_result(dot, ".$$$(field:id)", 1) + guard dot.tokens[0].0 is DOT_LIDENT("$$$(field:id)") else { + fail("expected DOT_LIDENT encoded typed $$$ metavariable") + } + let dot_inferred = tokens_from_string( + ".$$$field", + comment=false, + enable_metavar=true, + ) + assert_single_metavar_result(dot_inferred, ".$$$field", 1) + guard dot_inferred.tokens[0].0 is DOT_LIDENT("$$$field") else { + fail("expected DOT_LIDENT encoded untyped $$$ metavariable") + } + let post = tokens_from_string( + "$$$(label:id)~", + comment=false, + enable_metavar=true, + ) + assert_single_metavar_result(post, "$$$(label:id)~", 0) + guard post.tokens[0].0 is POST_LABEL("$$$(label:id)") else { + fail("expected POST_LABEL encoded typed $$$ metavariable") + } + let post_inferred = tokens_from_string( + "$$$label~", + comment=false, + enable_metavar=true, + ) + assert_single_metavar_result(post_inferred, "$$$label~", 0) + guard post_inferred.tokens[0].0 is POST_LABEL("$$$label") else { + fail("expected POST_LABEL encoded untyped $$$ metavariable") + } +} + ///| test "inferred metavariable annotation does not require whitespace" { let bare = tokens_from_string("$x:Int", comment=false, enable_metavar=true) @@ -101,11 +221,28 @@ test "inferred metavariable annotation does not require whitespace" { result.tokens[4].0 is RPAREN else { fail("expected parenthesized inferred metavariable annotation tokens") } + let triple = tokens_from_string( + "$$$x:Int", + comment=false, + enable_metavar=true, + ) + guard triple.errors.length() == 0 else { + fail("expected no lexical errors for inferred $$$ metavariable annotation") + } + guard triple.tokens[0].0 is LIDENT("$$$x") && + triple.tokens[1].0 is COLON && + triple.tokens[2].0 is UIDENT("Int") else { + fail("expected inferred $$$ metavariable annotation tokens") + } } ///| test "metavariable malformed forms report lexical errors when enabled" { - for source in ["$exp:x", "$(:exp)", "$(x:)", "$(x:exp", "$(\tx:exp)"] { + for + source in [ + "$exp:x", "$(:exp)", "$(x:)", "$(x:exp", "$(\tx:exp)", "$$$exp:x", "$$$(:exp)", + "$$$(x:)", "$$$(x:exp", "$$$(\tx:exp)", ".$$$(:exp)", + ] { let result = tokens_from_string(source, comment=false, enable_metavar=true) guard result.errors.length() > 0 && result.errors[0].2 is InvalidMetavarSyntax(_) else { @@ -114,6 +251,34 @@ test "metavariable malformed forms report lexical errors when enabled" { } } +///| +test "double-dollar metavariable prefix remains unsupported" { + for + case in [ + ("$$x", "$$x", 0), + ("$$(x:exp)", "$$(x:exp)", 0), + ("$$label~", "$$label~", 0), + (".$$field", ".$$field", 1), + (".$$(field:id)", ".$$(field:id)", 1), + ] { + let (source, expected_raw, error_start) = case + let result = tokens_from_string(source, comment=false, enable_metavar=true) + guard result.errors.length() == 1 else { + fail("expected one lexical error for unsupported prefix in \{source}") + } + let (start, end, error) = result.errors[0] + guard error is InvalidMetavarSyntax(raw) && + raw == expected_raw && + start.cnum == error_start && + end.cnum == source.length() else { + fail("unexpected double-dollar error or location for \{source}") + } + guard result.tokens.length() == 1 && result.tokens[0].0 is EOF else { + fail("unsupported \{source} must not recover as a valid metavariable") + } + } +} + ///| test "multiline interpolation keeps precedence over metavariable syntax" { let result = tokens_from_string("$|hello", comment=false, enable_metavar=true) From 1910314e54615daf768fe1cc1755db4b9e2525fc Mon Sep 17 00:00:00 2001 From: myfreess Date: Fri, 10 Jul 2026 13:32:19 +0800 Subject: [PATCH 2/2] update version --- lexer/moon.mod | 2 +- moon.mod | 6 +++--- moon_config/moon.mod | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lexer/moon.mod b/lexer/moon.mod index bfd6e810..0f3cc8a0 100644 --- a/lexer/moon.mod +++ b/lexer/moon.mod @@ -1,6 +1,6 @@ name = "moonbitlang/lexer" -version = "0.3.8" +version = "0.3.9" repository = "https://github.com/moonbitlang/parser" diff --git a/moon.mod b/moon.mod index 074ed2c9..89211a2a 100644 --- a/moon.mod +++ b/moon.mod @@ -1,11 +1,11 @@ name = "moonbitlang/parser" -version = "0.3.8" +version = "0.3.9" import { "moonbitlang/x@0.4.39", - "moonbitlang/lexer@0.3.8", - "moonbitlang/moon_config@0.3.7", + "moonbitlang/lexer@0.3.9", + "moonbitlang/moon_config@0.3.8", "moonbit-community/miniio@0.1.0", "moonbitlang/async@0.19.0", "moonbit-community/prettyprinter@0.4.10", diff --git a/moon_config/moon.mod b/moon_config/moon.mod index 548ae88d..978046ec 100644 --- a/moon_config/moon.mod +++ b/moon_config/moon.mod @@ -1,9 +1,9 @@ name = "moonbitlang/moon_config" -version = "0.3.7" +version = "0.3.8" import { - "moonbitlang/lexer@0.3.8", + "moonbitlang/lexer@0.3.9", } repository = "https://github.com/moonbitlang/parser"