diff --git a/report.md b/report.md new file mode 100644 index 00000000..9dcb4cbe --- /dev/null +++ b/report.md @@ -0,0 +1,136 @@ +# `untyped_cst` 与 `handrolled_parser` 语法解析差异审查报告 + +- 审查日期：2026-07-13 +- 更新日期：2026-07-15 +- 审查目标：验证以下关键不变量： + + > 对任意输入源码，`untyped_cst` 生成的 CST 转换为 AST 后，应等同于该源码经 `handrolled_parser` 直接生成的 AST。 + +## 结论 + +关键不变量整体上仍未成立。本报告只保留当前尚未解决或尚未完整验证的事项： + +1. 存在双方都无诊断、CST 转 AST 成功，但语义 AST 不同的输入。 +2. 即使忽略语义节点差异，两套实现生成的 AST 源码位置仍存在多类不一致。 +3. 尚未完整复扫未纳入 parser runner 的 token fixtures，不能据当前回归集合宣称全输入等价。 + +因此，当前仍不能将 `untyped_cst -> AST` 当作 `handrolled_parser` 的完全等价替代。 + +## 审查与验证基线 + +审查覆盖表达式、pattern、type、顶层声明、AST lowering、差分测试设施和源码位置。按照项目 `AGENTS.md` 的要求，没有读取由 `.mbty` 生成的大型 `.mbt` 文件；正式语法规则以原始 `.mbty` 文件为准。 + +当前验证结果： + +| 检查 | 结果 | +|---|---:| +| `moon check` | 通过 | +| `moon test untyped_cst/diagnostics_equiv_test.mbt -v` | 13/13 通过 | +| `moon test untyped_cst -v` | 85/85 通过 | +| `moon test test/sync_test/parser_test.mbt -v` | 967/967 通过 | +| `moon test test/manual_test -v` | 11/11 通过 | +| `moon info` | 无公开接口差异 | +| `git diff --check` | 通过 | +| 142 个非生成项目源码 | 双方均接受，发现 2 个 AST 差异 | +| 230 个未纳入 runner 的 token fixtures | 尚未完整复扫 | + +## 一、双方接受但语义 AST 不同 + +### 1. 单元素 tuple pattern 的尾逗号处理不同 + +输入： + +```moonbit +match x { (y,) => y } +``` + +- handrolled：`Pattern::Var(y)` +- CST：`Pattern::Tuple([Var(y)])` + +相关实现： + +- [`handrolled_parser/parser.mbt`](handrolled_parser/parser.mbt) +- [`untyped_cst/parse_pattern.mbt`](untyped_cst/parse_pattern.mbt) +- [`untyped_cst/lower_pattern.mbt`](untyped_cst/lower_pattern.mbt) + +正式 `.mbty` 规则要求逗号后还有一个非空 pattern，见 [`lexer/tokens/parser.mbty`](lexer/tokens/parser.mbty)。更合理的修复是让双方都拒绝该输入，而不是让 CST 模仿 handrolled 的 AST。 + +### 2. 单元素 tuple type 的尾逗号处理不同 + +输入： + +```moonbit +fn f(x : (Int,)) {} +``` + +- handrolled：`Type::Name(Int)` +- CST：`Type::Tuple([Int])` + +相关实现： + +- [`handrolled_parser/parser.mbt`](handrolled_parser/parser.mbt) +- [`untyped_cst/parse_type.mbt`](untyped_cst/parse_type.mbt) +- [`untyped_cst/lower_type.mbt`](untyped_cst/lower_type.mbt) + +正式 `.mbty` 规则同样要求逗号后存在非空 type。 + +## 二、AST 源码位置不变量仍不成立 + +默认 AST JSON 配置隐藏位置，见 [`lexer/basic/config.mbt`](lexer/basic/config.mbt)，因此大部分 AST equality 测试不会发现位置差异。 + +启用位置输出后，已确认的差异包括： + +- `fn init {}`、`fn main {}` 的空参数位置：CST 使用固定 `no_location`，handrolled 使用源文件中的空范围； +- `pub fn f(){}`：CST 的函数位置包含 `pub`，handrolled 从 `fn` 开始； +- `fn &T::f(){}`：owner type name 是否包含 `&` 不一致； +- `fn f(x~ : Int){}` 和 enum labelled argument：label/binder 位置是否包含 `~` 不一致； +- 局部命名函数：CST 使用源码位置，handrolled 使用 `no_location`； +- 负 pattern、索引/切片、object type 等节点也存在位置范围差异。 + +即使选择 `LocStyle::Json`，部分辅助位置字段仍没有被 serializer 输出。因此还需要直接比较 AST 位置字段或补全位置序列化，不能只依赖当前 JSON。 + +## 三、剩余测试覆盖盲区 + +### 1. 语义 AST runner 主要覆盖成功语料 + +[`test/sync_test/helper_test.mbt`](test/sync_test/helper_test.mbt) 会比较 handrolled、MoonYacc 和 CST 转 AST，但测试样例主要是三者都能成功解析的 parser fixtures。 + +[`untyped_cst/ast_equiv_test.mbt`](untyped_cst/ast_equiv_test.mbt) 也主要验证代表性成功输入，不能据此证明任意合法输入的 AST 都等价。 + +### 2. 默认位置序列化会隐藏差异 + +[`lexer/basic/loc.mbt`](lexer/basic/loc.mbt) 在默认配置下把 location 序列化为 `null`，使位置差异无法被常规 AST JSON equality 捕获。 + +### 3. 230 个 token fixtures 未纳入 parser runner + +现有 parser runner 有 967 个 fixtures，但 token fixtures 共 1197 个。剩余 230 个主要只用于 lexer 测试，本轮尚未重新完整扫描这些输入。 + +### 4. 测试生成器输出文件名与实际 runner 不一致 + +[`test/sync_test/generator/generator.mbt`](test/sync_test/generator/generator.mbt) 输出 `parser.mbt`/`lexer.mbt`，而当前实际运行的是 `parser_test.mbt`/`lexer_test.mbt`，存在重新生成语料后 runner 未同步的风险。 + +## 四、建议修复优先级 + +### P0：语义 AST 正确性 + +1. 按正式 `.mbty` 规则统一拒绝单元素 tuple pattern/type 的尾逗号。 +2. 继续核对 `as` 与 `is` 的解析层级及 lowering 结果。 + +### P1：AST 位置 + +1. 明确并测试每类 AST 节点的位置是否包含 modifier、sigil 和 soft keyword。 +2. 统一空参数、owner type、label/binder、局部函数及负 pattern 等位置范围。 +3. 补全位置 serializer，或在测试中直接比较完整 AST location 字段。 + +### P1：扩大差分覆盖 + +1. 完整复扫全部 token fixtures。 +2. 对双方成功的输入同时比较语义 AST 和完整位置。 +3. 修正测试生成器与 runner 的文件命名关系。 +4. 为每个剩余最小复现保留固定回归测试。 + +## 总结 + +当前 diagnostics、声明截断矩阵、967 个同步 parser tests 和 11 个 manual tests 均已通过。仍需解决的是合法输入的 AST 结构差异、AST 源码位置差异，以及尚未完整覆盖的 token fixtures。 + +要真正建立目标不变量，需要继续修复上述语义和位置问题，并把差分测试扩展到全部语料。 diff --git a/test/manual_test/extend_soft_keyword_regression_test.mbt b/test/manual_test/extend_soft_keyword_regression_test.mbt index 0fd4b0e0..147db617 100644 --- a/test/manual_test/extend_soft_keyword_regression_test.mbt +++ b/test/manual_test/extend_soft_keyword_regression_test.mbt @@ -26,9 +26,10 @@ test "extend soft keyword in remaining lower identifier positions" { } let hand_json = hand_impls.map(impl_ => impl_.json_repr()).to_json() let yacc_json = yacc_impls.map(impl_ => impl_.json_repr()).to_json() - if hand_json != yacc_json { + let cst_json = cst_impls_json(source, "extend_soft_keyword_regression.mbt") + if hand_json != yacc_json || hand_json != cst_json { fail( - "soft keyword extend produced different ASTs between moonyacc and handrolled parsers", + "soft keyword extend produced different ASTs between moonyacc, handrolled, and CST parsers", ) } } @@ -62,9 +63,12 @@ test "extend special constructor pattern location matches yacc" { let yacc_json = @basic.show_loc.protect(String, fn() { yacc_impls.map(impl_ => impl_.json_repr()).to_json() }) - if hand_json != yacc_json { + let cst_json = @basic.show_loc.protect(String, () => { + cst_impls_json(source, "extend_special_constructor_pattern_loc.mbt") + }) + if hand_json != yacc_json || hand_json != cst_json { fail( - "extend special constructor pattern location differs between moonyacc and handrolled parsers:\nhand=\{hand_json.stringify(indent=2)}\nyacc=\{yacc_json.stringify(indent=2)}", + "extend special constructor pattern location differs between moonyacc, handrolled, and CST parsers:\nhand=\{hand_json.stringify(indent=2)}\nyacc=\{yacc_json.stringify(indent=2)}\ncst=\{cst_json.stringify(indent=2)}", ) } } diff --git a/test/manual_test/for_condition_test.mbt b/test/manual_test/for_condition_test.mbt index 5fc9cb35..98b254d3 100644 --- a/test/manual_test/for_condition_test.mbt +++ b/test/manual_test/for_condition_test.mbt @@ -6,7 +6,7 @@ test "single semicolon for condition is accepted" { #| ignore(i) #| } #|} - let (_, hand_reports) = @parser.parse_string( + let (hand_impls, hand_reports) = @parser.parse_string( source, name="for_condition.mbt", parser=Handrolled, @@ -14,7 +14,7 @@ test "single semicolon for condition is accepted" { if !hand_reports.is_empty() { fail("handrolled parser rejected single-semicolon for condition") } - let (_, yacc_reports) = @parser.parse_string( + let (yacc_impls, yacc_reports) = @parser.parse_string( source, name="for_condition.mbt", parser=MoonYacc, @@ -22,4 +22,12 @@ test "single semicolon for condition is accepted" { if !yacc_reports.is_empty() { fail("moonyacc parser rejected single-semicolon for condition") } + let hand_json = hand_impls.map(impl_ => impl_.json_repr()).to_json() + let yacc_json = yacc_impls.map(impl_ => impl_.json_repr()).to_json() + let cst_json = cst_impls_json(source, "for_condition.mbt") + if hand_json != yacc_json || hand_json != cst_json { + fail( + "single-semicolon for condition produced different ASTs between moonyacc, handrolled, and CST parsers", + ) + } } diff --git a/test/manual_test/helper_test.mbt b/test/manual_test/helper_test.mbt index 76a54314..eec61856 100644 --- a/test/manual_test/helper_test.mbt +++ b/test/manual_test/helper_test.mbt @@ -1,6 +1,42 @@ +///| +fn reports_from_view(view : ArrayView[@basic.Report]) -> Array[@basic.Report] { + let reports : Array[@basic.Report] = [] + for report in view { + reports.push(report) + } + reports +} + +///| +fn cst_impls_json(source : String, name : String) -> Json raise Error { + let result = @untyped_cst.parse_string(source, name~, entry=Structure) + let diagnostics = reports_from_view(result.diagnostics_view()) + if !diagnostics.is_empty() { + fail( + "CST parser rejected \{name}: \{diagnostics.to_json().stringify(indent=2)}", + ) + } + let (impls, reports) = result.to_impls() + if !reports.is_empty() { + fail( + "converting CST for \{name} produced diagnostics: \{reports.to_json().stringify(indent=2)}", + ) + } + impls.map(impl_ => impl_.json_repr()).to_json() +} + +///| +fn expect_cst_rejected(source : String, name : String) -> Unit raise Error { + let result = @untyped_cst.parse_string(source, name~, entry=Structure) + if result.diagnostics_view().is_empty() { + fail("CST parser unexpectedly accepted \{name}") + } +} + ///| /// Verifies that both the hand-rolled parser and the moonyacc parser produce identical ASTs, -/// and that no diagnostics are generated for all test cases in `__snapshot__`. +/// and the CST parser produce identical ASTs, and that no diagnostics are +/// generated for all test cases in `__snapshot__`. fn @test.Test::run(t : Self) -> Unit raise Error { let name = t.name() let source_file = "test/manual_test/__snapshot__/\{name}.mbt" @@ -9,6 +45,7 @@ fn @test.Test::run(t : Self) -> Unit raise Error { } let hand_result = @parser.parse_string(source, name~, parser=Handrolled) let yacc_result = @parser.parse_string(source, name~, parser=MoonYacc) + let cst_json = cst_impls_json(source, name) if !(hand_result.1.is_empty() && yacc_result.1.is_empty()) { let buf = StringBuilder::new() let _ : Unit = @basic.show_loc.protect(String, fn() { @@ -24,7 +61,7 @@ fn @test.Test::run(t : Self) -> Unit raise Error { } let hand_json = hand_result.0.map(impl_ => impl_.json_repr()).to_json() let yacc_json = yacc_result.0.map(impl_ => impl_.json_repr()).to_json() - if hand_json != yacc_json { + if hand_json != yacc_json || hand_json != cst_json { @fs.create_dir(debug_dir) @fs.write_string_to_file("\{debug_dir}/\{name}.mbt", source) @fs.write_string_to_file( @@ -35,8 +72,12 @@ fn @test.Test::run(t : Self) -> Unit raise Error { "\{debug_dir}/\{name}_yacc.json", yacc_json.stringify(indent=2), ) + @fs.write_string_to_file( + "\{debug_dir}/\{name}_cst.json", + cst_json.stringify(indent=2), + ) fail( - "parsing \{name}.mbt produced different ASTs between moonyacc and handrolled parsers.", + "parsing \{name}.mbt produced different ASTs between moonyacc, handrolled, and CST parsers.", ) } let json : Json = hand_json diff --git a/test/manual_test/impl_method_quantifiers_test.mbt b/test/manual_test/impl_method_quantifiers_test.mbt index 61f3e37e..3a097e6e 100644 --- a/test/manual_test/impl_method_quantifiers_test.mbt +++ b/test/manual_test/impl_method_quantifiers_test.mbt @@ -30,9 +30,10 @@ test "parse impl method quantifiers" { } let hand_json = hand_impls.map(impl_ => impl_.json_repr()).to_json() let yacc_json = yacc_impls.map(impl_ => impl_.json_repr()).to_json() - if hand_json != yacc_json { + let cst_json = cst_impls_json(source, "impl_method_quantifiers.mbt") + if hand_json != yacc_json || hand_json != cst_json { fail( - "impl method quantifiers produced different ASTs between moonyacc and handrolled parsers", + "impl method quantifiers produced different ASTs between moonyacc, handrolled, and CST parsers", ) } } diff --git a/test/manual_test/loop_removed_test.mbt b/test/manual_test/loop_removed_test.mbt index 0a1af5b5..16ceab11 100644 --- a/test/manual_test/loop_removed_test.mbt +++ b/test/manual_test/loop_removed_test.mbt @@ -17,4 +17,5 @@ test "loop expression is rejected" { if yacc_reports.is_empty() { fail("moonyacc parser unexpectedly accepted loop expression") } + expect_cst_rejected(source, "loop_removed.mbt") } diff --git a/test/manual_test/moon.pkg b/test/manual_test/moon.pkg index 2ecc4bca..f739a239 100644 --- a/test/manual_test/moon.pkg +++ b/test/manual_test/moon.pkg @@ -1,6 +1,7 @@ import { "moonbitlang/parser", "moonbitlang/parser/syntax", + "moonbitlang/parser/untyped_cst", "moonbitlang/lexer/basic", "moonbitlang/x/fs", "moonbitlang/core/test", diff --git a/test/manual_test/outdated_syntax_removed_test.mbt b/test/manual_test/outdated_syntax_removed_test.mbt index 04a6cd3a..19600726 100644 --- a/test/manual_test/outdated_syntax_removed_test.mbt +++ b/test/manual_test/outdated_syntax_removed_test.mbt @@ -8,18 +8,35 @@ fn expect_rejected(source : String, name : String) -> Unit raise Error { if yacc_reports.is_empty() { fail("moonyacc parser unexpectedly accepted \{name}") } + expect_cst_rejected(source, name) } ///| fn expect_accepted(source : String, name : String) -> Unit raise Error { - let (_, hand_reports) = @parser.parse_string(source, name~, parser=Handrolled) + let (hand_impls, hand_reports) = @parser.parse_string( + source, + name~, + parser=Handrolled, + ) if !hand_reports.is_empty() { fail("handrolled parser unexpectedly rejected \{name}") } - let (_, yacc_reports) = @parser.parse_string(source, name~, parser=MoonYacc) + let (yacc_impls, yacc_reports) = @parser.parse_string( + source, + name~, + parser=MoonYacc, + ) if !yacc_reports.is_empty() { fail("moonyacc parser unexpectedly rejected \{name}") } + let hand_json = hand_impls.map(impl_ => impl_.json_repr()).to_json() + let yacc_json = yacc_impls.map(impl_ => impl_.json_repr()).to_json() + let cst_json = cst_impls_json(source, name) + if hand_json != yacc_json || hand_json != cst_json { + fail( + "parsing \{name} produced different ASTs between moonyacc, handrolled, and CST parsers", + ) + } } ///| diff --git a/test/manual_test/template_object_rhs_test.mbt b/test/manual_test/template_object_rhs_test.mbt index 8357bc1a..3f4e08a7 100644 --- a/test/manual_test/template_object_rhs_test.mbt +++ b/test/manual_test/template_object_rhs_test.mbt @@ -27,9 +27,10 @@ test "parse template object rhs" { } let yacc_json = yacc_impls.map(impl_ => impl_.json_repr()).to_json() let hand_json = hand_impls.map(impl_ => impl_.json_repr()).to_json() - if yacc_json != hand_json { + let cst_json = cst_impls_json(source, "template_object_rhs.mbt") + if yacc_json != hand_json || hand_json != cst_json { fail( - "template object rhs produced different ASTs between moonyacc and handrolled parsers", + "template object rhs produced different ASTs between moonyacc, handrolled, and CST parsers", ) } } diff --git a/test/sync_test/helper_test.mbt b/test/sync_test/helper_test.mbt index d13ae698..a0d91462 100644 --- a/test/sync_test/helper_test.mbt +++ b/test/sync_test/helper_test.mbt @@ -1,3 +1,12 @@ +///| +fn reports_from_view(view : ArrayView[@basic.Report]) -> Array[@basic.Report] { + let reports : Array[@basic.Report] = [] + for report in view { + reports.push(report) + } + reports +} + ///| /// Verifies that both the hand-rolled parser and the moonyacc parser produce identical ASTs, /// and that no diagnostics are generated for all test cases in `__snapshot__`. @@ -9,7 +18,11 @@ fn @test.Test::run(t : Self) -> Unit raise Error { } let hand_result = @parser.parse_string(source, name~, parser=Handrolled) let yacc_result = @parser.parse_string(source, name~, parser=MoonYacc) - if !(hand_result.1.is_empty() && yacc_result.1.is_empty()) { + let cst_result = @untyped_cst.parse_string(source, name~, entry=Structure) + let cst_diagnostics = reports_from_view(cst_result.diagnostics_view()) + if !(hand_result.1.is_empty() && + yacc_result.1.is_empty() && + cst_diagnostics.is_empty()) { let buf = StringBuilder::new() let _ : Unit = @basic.show_loc.protect(String, fn() { buf @@ -18,13 +31,26 @@ fn @test.Test::run(t : Self) -> Unit raise Error { ..write_string(hand_result.1.to_json().stringify(indent=2)) ..write_string("\nYacc parser diagnostics: ") ..write_string(yacc_result.1.to_json().stringify(indent=2)) + ..write_string("\nCST parser diagnostics: ") + ..write_string(cst_diagnostics.to_json().stringify(indent=2)) .write_string("\n") }) fail(buf.to_string()) } let hand_json = hand_result.0.map(impl_ => impl_.json_repr()).to_json() let yacc_json = yacc_result.0.map(impl_ => impl_.json_repr()).to_json() - if hand_json != yacc_json { + let (cst_impls, cst_reports) = cst_result.to_impls() + if !cst_reports.is_empty() { + let rendered = @basic.show_loc.protect(String, fn() { + cst_reports.to_json().stringify(indent=2) + }) + @fs.create_dir(debug_dir) + @fs.write_string_to_file("\{debug_dir}/\{name}.mbt", source) + @fs.write_string_to_file("\{debug_dir}/\{name}_cst_reports.json", rendered) + fail("converting CST for \{name}.mbt produced diagnostics.") + } + let cst_json = cst_impls.map(impl_ => impl_.json_repr()).to_json() + if hand_json != yacc_json || hand_json != cst_json { @fs.create_dir(debug_dir) @fs.write_string_to_file("\{debug_dir}/\{name}.mbt", source) @fs.write_string_to_file( @@ -35,8 +61,12 @@ fn @test.Test::run(t : Self) -> Unit raise Error { "\{debug_dir}/\{name}_yacc.json", yacc_json.stringify(indent=2), ) + @fs.write_string_to_file( + "\{debug_dir}/\{name}_cst.json", + cst_json.stringify(indent=2), + ) fail( - "parsing \{name}.mbt produced different ASTs between moonyacc and handrolled parsers.", + "parsing \{name}.mbt produced different ASTs between moonyacc, handrolled, and CST parsers.", ) } let json : Json = hand_json diff --git a/test/sync_test/moon.pkg b/test/sync_test/moon.pkg index bf2cb612..250dc279 100644 --- a/test/sync_test/moon.pkg +++ b/test/sync_test/moon.pkg @@ -4,6 +4,7 @@ import { "moonbitlang/lexer/basic", "moonbitlang/lexer", "moonbitlang/lexer/tokens", + "moonbitlang/parser/untyped_cst", "moonbitlang/x/fs", "moonbitlang/core/test", } for "test" diff --git a/untyped_cst/README.mbt.md b/untyped_cst/README.mbt.md new file mode 100644 index 00000000..9979ee19 --- /dev/null +++ b/untyped_cst/README.mbt.md @@ -0,0 +1,42 @@ +# moonbitlang/parser/untyped_cst + +`moonbitlang/parser/untyped_cst` parses MoonBit source into an untyped concrete +syntax tree that preserves tokens, comments, delimiters, separators, and source +spans. It is intended for tooling that needs source-faithful structure before +or alongside the existing typed syntax AST. + +The main entry point is `parse_string`. + +## Example + +```mbt check +///| +test "parse and restore source text" { + let source = "let x = 1" + let result = @untyped_cst.parse_string(source) + assert_eq(result.diagnostics_view().length(), 0) + assert_eq(result.root().text(source), source) +} +``` + +## Source Text + +CST nodes do not expose the original source text. Keep the original source and +pass it to `CstNode::text(source)` when reconstructing text for a node. + +`source_span` offsets are `String`/`StringView` slicing offsets. Keep using the +same source string when calling `text(source)`. + +Parse source text with `parse_string` so spans remain backed by the original +source string. + +## Note + +The `untyped_cst` package must remain behaviorally equivalent to +`handrolled_parser`: + +- For any input that `handrolled_parser` fails to parse, the CST parser must + produce the same reports. +- For any input that `handrolled_parser` parses without reports, the CST parser + must also produce no reports, and converting the resulting CST to an AST must + produce an AST equal to the one produced by `handrolled_parser`. diff --git a/untyped_cst/api.mbt b/untyped_cst/api.mbt new file mode 100644 index 00000000..9ecec04f --- /dev/null +++ b/untyped_cst/api.mbt @@ -0,0 +1,47 @@ +///| +fn initial_position(name : String) -> Position { + Position::{ fname: name, lnum: 1, bol: 0, cnum: 0 } +} + +///| +fn minimal_root(entry : EntryPoint, name : String, source_len : Int) -> CstNode { + let start = initial_position(name) + let kind = match entry { + Structure => NodeKind::SourceFile + Expression => NodeKind::Fragment(Expression) + } + node(kind, empty_loc(start), mk_span(0, source_len), []) +} + +///| +fn parse_with_state( + tokens : Triples, + source : String, + location_mode : LocationMode, + entry : EntryPoint, + diagnostics : Array[Report], + docstrings : Array[List[(Location, Comment)]], +) -> ParseResult { + let public_diagnostics = diagnostics.copy() + let state = new_state(tokens, source, location_mode, public_diagnostics) + let root = parse_fragment(state, entry) + parse_result(root, state.reports, docstrings) +} + +///| +pub fn parse_string( + source : String, + name? : String = "", + entry? : EntryPoint = Structure, +) -> ParseResult { + let lex = @lexer.tokens_from_string(source, name~, comment=true) + let diagnostics = reports_from_lex_errors(lex.errors) + parse_with_state( + lex.tokens, + source, + DefaultLocation, + entry, + diagnostics, + lex.docstrings, + ) +} diff --git a/untyped_cst/ast_equiv_test.mbt b/untyped_cst/ast_equiv_test.mbt new file mode 100644 index 00000000..39c1b9aa --- /dev/null +++ b/untyped_cst/ast_equiv_test.mbt @@ -0,0 +1,409 @@ +///| +fn impls_json(impls : @syntax.Impls) -> Json { + @basic.show_loc.protect(String, fn() { + impls.map(impl_ => impl_.json_repr()).to_json() + }) +} + +///| +fn expr_json(expr : @syntax.Expr) -> Json { + @basic.show_loc.protect(String, fn() { expr.json_repr() }) +} + +///| +fn reports_from_view(view : ArrayView[@basic.Report]) -> Array[@basic.Report] { + let reports : Array[@basic.Report] = [] + for report in view { + reports.push(report) + } + reports +} + +///| +fn fail_parser_reports( + source : String, + hand_reports : Array[@basic.Report], + yacc_reports : Array[@basic.Report], +) -> Unit raise { + let buf = StringBuilder::new() + let _ : Unit = @basic.show_loc.protect(String, fn() { + buf + ..write_string("parsing focused AST equivalence case failed.\n") + ..write_string("Source:\n") + ..write_string(source) + ..write_string("\nHandrolled parser diagnostics: ") + ..write_string(hand_reports.to_json().stringify(indent=2)) + ..write_string("\nYacc parser diagnostics: ") + ..write_string(yacc_reports.to_json().stringify(indent=2)) + .write_string("\n") + }) + fail(buf.to_string()) +} + +///| +fn assert_ast_equiv( + source : String, + label? : String = "", +) -> Unit raise { + let (hand_impls, hand_reports) = @parser.parse_string( + source, + parser=@parser.Handrolled, + ) + let (yacc_impls, yacc_reports) = @parser.parse_string( + source, + parser=@parser.MoonYacc, + ) + let cst_result = @untyped_cst.parse_string(source, entry=Structure) + if !(hand_reports.is_empty() && yacc_reports.is_empty()) { + fail_parser_reports(source, hand_reports, yacc_reports) + } + if cst_result.diagnostics_view().length() != 0 { + let reports = reports_from_view(cst_result.diagnostics_view()) + let rendered = @basic.show_loc.protect(String, fn() { + reports.to_json().stringify(indent=2) + }) + fail("CST diagnostics for \{label}: \{rendered}") + } + if cst_result.root().text(source) != source { + fail("source text mismatch for \{label}") + } + let (cst_impls, conversion_reports) = cst_result.to_impls() + if conversion_reports.length() != 0 { + fail("conversion diagnostics for \{label}: \{conversion_reports.length()}") + } + if impls_json(cst_impls) != impls_json(hand_impls) { + fail("handrolled AST mismatch for \{label}") + } + if impls_json(cst_impls) != impls_json(yacc_impls) { + fail("yacc AST mismatch for \{label}") + } +} + +///| +fn assert_handrolled_cst_ast_equiv(source : String) -> Unit raise { + let (hand_impls, hand_reports) = @parser.parse_string( + source, + parser=@parser.Handrolled, + ) + let cst_result = @untyped_cst.parse_string(source, entry=Structure) + if !hand_reports.is_empty() { + fail("handrolled diagnostics for focused CST equivalence case") + } + if cst_result.diagnostics_view().length() != 0 { + fail("CST diagnostics for focused handrolled equivalence case") + } + if cst_result.root().text(source) != source { + fail("source text mismatch for focused handrolled equivalence case") + } + let (cst_impls, conversion_reports) = cst_result.to_impls() + if conversion_reports.length() != 0 { + fail("conversion diagnostics for focused handrolled equivalence case") + } + if impls_json(cst_impls) != impls_json(hand_impls) { + fail("handrolled AST mismatch for focused CST equivalence case") + } +} + +///| +fn assert_expr_ast_equiv(source : String) -> Unit raise { + let lex = @lexer.tokens_from_string(source, comment=true) + let (expected, expected_reports) = @handrolled_parser.parse_expr(lex.tokens) + let cst_result = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(expected_reports.length(), 0) + @test.assert_eq(cst_result.diagnostics_view().length(), 0) + let (actual, conversion_reports) = cst_result.to_expr() + @test.assert_eq(conversion_reports.length(), 0) + guard actual is Some(expr) else { fail("expected expression for \{source}") } + @test.assert_eq(expr_json(expr), expr_json(expected)) +} + +///| +fn read_snapshot(name : String) -> String raise { + @fs.read_file_to_string("test/sync_test/__snapshot__/\{name}") catch { + _ => fail("failed to read snapshot \{name}") + } +} + +///| +fn docstring_source() -> String { + let source = + #|/// docs for fn f + #|fn f() { () } + #| + #|/// docs for type T + #|type T + #| + #|/// docs for enum E + #|enum E { + #| /// docs for constructor A + #| A + #|} + #| + #|/// docs for struct S + #|struct S { + #| /// docs for field x + #| x : Int + #|} + source +} + +///| +test "representative snapshots are AST equivalent" { + let snapshots = [ + "parse_test_toplevel_test.mbt", "parse_test_fn_exp_test.mbt", "pipeline_test_match_test.mbt", + "parse_test_async_test.mbt", "parse_test_text_segment_test.mbt", "pipeline_test_pattern_test.mbt", + "parse_test_pipe_arrow_parse.mbt", "parse_test_return.mbt", "pipeline_test_while_test.mbt", + "pipeline_test_for.mbt", "parse_test_lexmatch.mbt", "pipeline_test_option.mbt", + "pipeline_test_builtin_test.mbt", "pipeline_test_array.mbt", "pipeline_test_obj_in_struct.mbt", + "pipeline_test_record_update_typing.mbt", "parse_test_unicode_test.mbt", "pipeline_test_async_misc.mbt", + "pipeline_test_record.mbt", "pipeline_test_pipe.mbt", "pipeline_test_foreach.mbt", + "pipeline_test_return_test.mbt", + ] + for snapshot in snapshots { + assert_ast_equiv(read_snapshot(snapshot), label=snapshot) + } +} + +///| +test "focused declaration AST equivalence cases" { + let cases = [ + "fn f(x : Int) -> Int { x }", "const X = 1", "type T[A] = Array[A]", "struct S { pub mut x : Int }", + "enum E { A(x~ : Int) }", "trait Show { show(Self) -> String }", "impl Show for S with show(self) { \"S\" }", + "test { inspect(1, content=\"1\") }", "using @pkg {func as func2}", "type U derive(Eq, @pkg.Interface)", + ] + for source in cases { + assert_ast_equiv(source) + } +} + +///| +test "question-suffixed error type matches handrolled AST" { + assert_handrolled_cst_ast_equiv("fn f() -> Unit raise E? {}") +} + +///| +test "float unary plus matches handrolled AST" { + assert_expr_ast_equiv("+1.0F") +} + +///| +test "qualified trait impl matches handrolled AST" { + let cases = [ + "impl @debug.Debug for T", "impl @debug.Debug for T with to_repr(self) { self }", + ] + for source in cases { + assert_handrolled_cst_ast_equiv(source) + } +} + +///| +test "impl method error annotations and bodies match handrolled AST" { + let cases = [ + "impl T for S with f() raise E { 1 }", "impl T for S with f() noraise { 2 }", + ] + for source in cases { + assert_handrolled_cst_ast_equiv(source) + } +} + +///| +test "optional types and existing error annotations remain AST equivalent" { + let cases = [ + "fn f(x : E?) -> E? { x }", "fn f() -> Unit raise E {}", "fn f() -> Unit raise ? {}", + ] + for source in cases { + assert_ast_equiv(source) + } +} + +///| +test "attributed generic struct optional field type AST equivalence" { + let source = + #|#valtype + #|struct T[A, B] { + #| a : (A, B) + #| b : A? + #| c : FixedArray[B] + #|} derive(Debug) + assert_ast_equiv(source) +} + +///| +test "source-backed root covers leading and trailing whitespace" { + assert_ast_equiv("\n fn spaced() { () }\n\n") +} + +///| +test "expression conversion produces expression AST" { + let source = "{ let mut x = 1; x }" + let result = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(result.diagnostics_view().length(), 0) + let (expr, reports) = result.to_expr() + @test.assert_eq(reports.length(), 0) + guard expr is Some(_) else { fail("expected expression") } +} + +///| +test "string and labelled argument AST equivalence cases" { + let cases = [ + "\"hello \\{name}\"", "b\"hello \\{name}\"", "#| Hello\n#| World", "$| hello \\{name}\n#| tail", + "f(label~)", "f(label?, other?=value)", + ] + for source in cases { + assert_expr_ast_equiv(source) + } +} + +///| +test "special postfix operators bind after range and prefix expressions" { + let cases = [ + "1 ..< 2 as T", "-x as T", "1 ..< 2 is _", "-x is _", "1 ..< 2 =~ re\"a\"", "-x =~ re\"a\"", + "1 ..< 2 <+ \"value\"", "-x <+ \"value\"", "1 ..< 2 String raise { + @fs.read_file_to_string("untyped_cst/\{path}") catch { + _ => fail("failed to read untyped_cst/\{path}") + } +} + +///| +fn array_has_string(items : Array[String], expected : String) -> Bool { + for item in items { + if item == expected { + return true + } + } + false +} + +///| +test "AST conversion runtime does not replay tokens through copied parser" { + let runtime_files = [ + "api.mbt", "to_ast.mbt", "docstring.mbt", "node.mbt", "parser_state.mbt", + ] + let forbidden = [ + "parse_impls_from_tokens", "parse_expr_from_tokens", "parse_impls_for_cst", "parse_expr_for_cst", + "token_triples_from_cst", "collect_token_triples", "tokens_array", "tokens_view", + ] + for path in runtime_files { + let source = read_untyped_cst_source(path) + for needle in forbidden { + @test.assert_eq(source.contains(needle), false) + } + } +} + +///| +test "copied AST parser bridge files are removed from package sources" { + let sources = @fs.read_dir("untyped_cst") catch { + _ => fail("failed to read untyped_cst directory") + } + let removed_files = [ + "ast_bridge.mbt", "ast_parser.mbt", "ast_parser_core.mbt", "ast_parser_first_set.mbt", + ] + for path in removed_files { + @test.assert_eq(array_has_string(sources, path), false) + } +} diff --git a/untyped_cst/cst_debug.mbt b/untyped_cst/cst_debug.mbt new file mode 100644 index 00000000..da2b0dc2 --- /dev/null +++ b/untyped_cst/cst_debug.mbt @@ -0,0 +1,82 @@ +///| +fn loc_to_test_json(loc : Location) -> Json { + { "start": loc.start.cnum, "end": loc.end.cnum } +} + +///| +fn span_to_test_json(span : SourceSpan) -> Json { + { "start": span.start_offset, "end": span.end_offset } +} + +///| +fn label_to_test_json(label : String?) -> Json { + match label { + Some(label) => Json::string(label) + None => Json::null() + } +} + +///| +fn NodeKind::test_name(self : NodeKind) -> String { + match self { + SourceFile => "SourceFile" + Fragment(_) => "Fragment" + Token(_) => "Token" + Missing(_) => "Missing" + Error => "Error" + Impl(_) => "Impl" + Expr(_) => "Expr" + Pattern(_) => "Pattern" + Type(_) => "Type" + LexPattern(_) => "LexPattern" + RegexPattern(_) => "RegexPattern" + Case(_) => "Case" + Argument(_) => "Argument" + Stmt(_) => "Stmt" + Decl(_) => "Decl" + Clause(_) => "Clause" + Name(_) => "Name" + List(_) => "List" + InterpSegment(_) => "InterpSegment" + } +} + +///| +fn CstNode::to_test_json(self : CstNode, source? : String = "") -> Json { + let _ = source + let children : Array[Json] = [] + for i in 0.. + Json::object({ + "kind": Json::string(self.kind.test_name()), + "expected": Json::string(expected.to_string()), + "loc": loc_to_test_json(self.loc), + "source_span": span_to_test_json(self.source_span), + "children": Json::array(children), + }) + Token(token) => + Json::object({ + "kind": Json::string(self.kind.test_name()), + "token": token.to_json(), + "loc": loc_to_test_json(self.loc), + "source_span": span_to_test_json(self.source_span), + "children": Json::array(children), + }) + _ => + Json::object({ + "kind": Json::string(self.kind.test_name()), + "loc": loc_to_test_json(self.loc), + "source_span": span_to_test_json(self.source_span), + "children": Json::array(children), + }) + } +} diff --git a/untyped_cst/diagnostics_equiv_test.mbt b/untyped_cst/diagnostics_equiv_test.mbt new file mode 100644 index 00000000..1beca9c8 --- /dev/null +++ b/untyped_cst/diagnostics_equiv_test.mbt @@ -0,0 +1,515 @@ +///| +fn diagnostics_equiv_reports_from_view( + view : ArrayView[@basic.Report], +) -> Array[@basic.Report] { + let reports : Array[@basic.Report] = [] + for report in view { + reports.push(report) + } + reports +} + +///| +fn diagnostics_equiv_reports_json(reports : Array[@basic.Report]) -> Json { + @basic.show_loc.protect(@basic.Json, fn() { reports.to_json() }) +} + +///| +fn diagnostics_equiv_assert_reports( + label : String, + source : String, + accepted : Bool, + hand : Array[@basic.Report], + actual : Array[@basic.Report], +) -> Unit raise { + let hand_json = diagnostics_equiv_reports_json(hand) + let actual_json = diagnostics_equiv_reports_json(actual) + let actual_accepted = actual.is_empty() + if hand_json != actual_json || actual_accepted != accepted { + fail( + StringBuilder::new() + ..write_string("diagnostics mismatch for ") + ..write_string(label) + ..write_string(": expected accepted=") + ..write_string(accepted.to_string()) + ..write_string("\nsource:\n") + ..write_string(source) + ..write_string("\nhandrolled:\n") + ..write_string(hand_json.stringify(indent=2)) + ..write_string("\nCST:\n") + ..write_string(actual_json.stringify(indent=2)) + .to_string(), + ) + } +} + +///| +fn assert_structure_acceptance( + label : String, + source : String, + accepted : Bool, +) -> Unit raise { + let (_, hand_reports) = @parser.parse_string( + source, + name="diagnostics_equiv.mbt", + parser=@parser.Handrolled, + ) + let cst = @untyped_cst.parse_string( + source, + name="diagnostics_equiv.mbt", + entry=Structure, + ) + let cst_reports = diagnostics_equiv_reports_from_view(cst.diagnostics_view()) + diagnostics_equiv_assert_reports( + label, source, accepted, hand_reports, cst_reports, + ) + if cst.root().text(source) != source { + fail("source text mismatch for \{label}") + } +} + +///| +fn assert_expression_acceptance( + label : String, + source : String, + accepted : Bool, +) -> Unit raise { + let lex = @lexer.tokens_from_string( + source, + name="diagnostics_equiv.mbt", + comment=true, + ) + let (_, hand_reports) = @handrolled_parser.parse_expr(lex.tokens) + let cst = @untyped_cst.parse_string( + source, + name="diagnostics_equiv.mbt", + entry=Expression, + ) + let cst_reports = diagnostics_equiv_reports_from_view(cst.diagnostics_view()) + diagnostics_equiv_assert_reports( + label, source, accepted, hand_reports, cst_reports, + ) + if cst.root().text(source) != source { + fail("source text mismatch for \{label}") + } +} + +///| +test "structure diagnostics come from CST parser recovery" { + let cases = [ + ("missing parameter close", "fn f("), + ("bad leading parameter separator", "fn f(, x)"), + ("missing let pattern", "let = 1"), + ("top-level recovery", "fn f(\nlet y = 1"), + ("invalid question in block", "fn f() { ? }"), + ("removed struct constructor", "struct S { fn new() -> S }"), + ] + for case in cases { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } +} + +///| +test "expression diagnostics come from CST parser recovery" { + let cases = [ + ("missing call close", "f(1, 2"), + ("missing block close", "{ let x = 1; x"), + ("loop expression", "loop { 1 }"), + ("throw expression", "throw err"), + ("removed lexmatch question", "s lexmatch? \"a\""), + ] + for case in cases { + let (label, source) = case + assert_expression_acceptance(label, source, false) + } + assert_expression_acceptance("line break separator", "1\n2", true) +} + +///| +test "acceptance equivalence: labelled arguments" { + let invalid = [ + ("uppercase label", "f(X=1)"), + ("uppercase optional label", "f(X?=1)"), + ] + for case in invalid { + let (label, source) = case + assert_expression_acceptance(label, source, false) + } + let valid = [ + ("lowercase label", "f(x=1)"), + ("lowercase optional label", "f(x?=1)"), + ("label puns", "f(x~, y?)"), + ] + for case in valid { + let (label, source) = case + assert_expression_acceptance(label, source, true) + } +} + +///| +test "acceptance equivalence: delimited lists" { + let invalid = [ + ("enum separator", "enum E { A B }"), + ("struct separator", "struct S { x : Int y : Int }"), + ("trait separator", "trait T { f(Self) g(Self) }"), + ("using separator", "using @p { x y }"), + ("parameter separator", "fn f(x : Int y : Int) {}"), + ("type parameter separator", "type T[A B]"), + ("derive separator", "type T derive(Eq Show)"), + ("empty tuple struct", "struct S()"), + ("top-level separator", "fn f() {} fn g() {}"), + ] + for case in invalid { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid = [ + ("enum semicolon", "enum E { A; B }"), + ("struct semicolon", "struct S { x : Int; y : Int }"), + ("trait semicolon", "trait T { f(Self); g(Self) }"), + ("using comma", "using @p { x, y }"), + ("parameter comma", "fn f(x : Int, y : Int) {}"), + ("type parameter comma", "type T[A, B]"), + ("derive comma", "type T derive(Eq, Show)"), + ("nonempty tuple struct", "struct S(Int)"), + ("top-level semicolon", "fn f() {}; fn g() {}"), + ] + for case in valid { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} + +///| +test "acceptance equivalence: declaration completion" { + let invalid = [ + ("bare fn", "fn"), + ("fn name only", "fn f"), + ("fn parameters only", "fn f()"), + ("bare let", "let"), + ("let pattern only", "let x"), + ("const name only", "const X"), + ("bare impl", "impl"), + ("impl trait only", "impl Trait"), + ("bare test", "test"), + ("bare extern", "extern"), + ("type bang", "type! E"), + ("numeric function name", "fn 1() {}"), + ("numeric function parameter", "fn f(1) {}"), + ("missing return type", "fn f() -> {}"), + ("using without body", "using @p"), + ("declare using", "declare using @p { x }"), + ("extern without language", "extern fn f() = \"x\""), + ("extern type", "extern \"C\" type T"), + ] + for case in invalid { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid = [ + ("function body", "fn f() {}"), + ("function stub", "fn f() = \"x\""), + ("declare function", "declare fn f()"), + ("declare function return type", "declare fn f() -> Int"), + ("declare method return type", "declare fn T::f(x : T) -> Int"), + ("declare function error annotation", "declare fn f() -> Int noraise"), + ("let value", "let x = 1"), + ("constant value", "const X = 1"), + ("impl relation", "impl Trait for T"), + ("impl method", "impl Trait with f(self) { self }"), + ("test body", "test {}"), + ("extern function", "extern \"C\" fn f() = \"x\""), + ("using body", "using @p {}"), + ] + for case in valid { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} + +///| +test "acceptance equivalence: declaration constraints" { + let invalid = [ + ("type parameter constraint", "type T[A : Eq]"), + ("suberror type parameter", "suberror E[A]"), + ("enum label without tilde", "enum E { A(x : Int) }"), + ("destructuring function parameter", "fn f((x, y) : (Int, Int)) { x }"), + ] + for case in invalid { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid = [ + ("plain type parameter", "type T[A]"), + ("function type constraint", "fn[A : Eq] f(x : A) {}"), + ("impl type constraint", "impl[A : Eq] Trait for T[A]"), + ("plain suberror", "suberror E"), + ("enum labelled parameter", "enum E { A(x~ : Int) }"), + ( + "supported function parameters", "fn f(x : Int, y~ : Int, z? : Int, _ : Int) {}", + ), + ] + for case in valid { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} + +///| +test "acceptance equivalence: expressions patterns and types" { + let invalid_exprs = [ + ("regex pattern", "match x { re\"a\" => 0 }"), + ("regex map pattern key", "match x { {re\"a\" : y} => y }"), + ("top-level typed pattern", "match x { y : Int => y }"), + ("ellipsis array elements", "[...a, ...b]"), + ("dotdot range", "1 .. 2"), + ("leading negative map key", "{-1 : x}"), + ("late record spread", "{x, ..r}"), + ("try without catch", "try f()"), + ("raise call argument", "f(raise e)"), + ("non-block else", "if x {} else 1"), + ("parenthesized foreach binder", "for (x) in xs {}"), + ("removed for else", "for x in xs {} else {}"), + ("unparenthesized typed arrow", "x : Int => x"), + ("unbraced pipeline arrow", "a |> x => x"), + ("uppercase lex alias binder", "lexmatch s { (re\"a\" as X) => 0 }"), + ("missing lex alias binder", "lexmatch s { (re\"a\" as) => 0 }"), + ] + for case in invalid_exprs { + let (label, source) = case + assert_expression_acceptance(label, source, false) + } + let invalid_types = [ + ("unparenthesized function type", "type T = Int -> Int"), + ("async non-function type", "type T = async Int -> Int"), + ("object type literal", "type T = {a : Int}"), + ("standalone unit type", "type T = ()"), + ("parameterized error type", "fn f() -> Unit raise E[Int] {}"), + ("object error type", "fn f() -> Unit raise &E {}"), + ] + for case in invalid_types { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid_exprs = [ + ("lex regex pattern", "lexmatch s { re\"a\" => 0 }"), + ("parenthesized typed pattern", "match x { (y : Int) => y }"), + ("array spreads", "[..a, ..b]"), + ("record update", "{..r, x: 1}"), + ("try catch", "try f() catch { _ => 0 }"), + ("block else", "if x {} else {}"), + ("nobreak", "for x in xs {} nobreak {}"), + ("parenthesized typed arrow", "(x : Int) => x"), + ("braced pipeline arrow", "a |> x => { x }"), + ("later negative map key", "{\"x\": 1, -1: 2}"), + ("lowercase lex alias binder", "lexmatch s { (re\"a\" as x) => 0 }"), + ("standalone raise statement", "raise e"), + ("standalone return statement", "return e"), + ("standalone defer statement", "defer f()"), + ("standalone let statement", "let x = 1"), + ("standalone todo statement", "..."), + ("statement expressions in block", "{ raise e; defer f(); () }"), + ] + for case in valid_exprs { + let (label, source) = case + assert_expression_acceptance(label, source, true) + } + let valid_types = [ + ("parenthesized function type", "type T = (Int) -> Int"), + ("async function type", "type T = async (Int) -> Int"), + ("zero argument function type", "type T = () -> Int"), + ("tuple type", "type T = (Int, String)"), + ("named object type", "type T = &E"), + ("simple error type", "fn f() -> Unit raise E {}"), + ] + for case in valid_types { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} + +///| +test "diagnostics equivalence: reported declaration boundary cases" { + let cases = [ + ("impl missing self type", "impl Trait for"), + ( + "impl method after missing self type", "impl Trait for with f(self) { self }", + ), + ("enumview missing source type", "enumview V { C } for with view(x) { x }"), + ("type alias missing target", "type T ="), + ("trait missing supertrait", "trait T : {}"), + ("let annotation missing type", "let x : = 1"), + ("parameter annotation missing type", "fn f(x~ : = 1) {}"), + ] + for case in cases { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } +} + +///| +test "diagnostics equivalence: reported expression boundary cases" { + let cases = [ + ("catch missing block", "try f() catch"), + ("try missing body", "try catch { _ => 0 }"), + ("if missing body", "if x"), + ("while missing body", "while x"), + ("uppercase lex alias", "lexmatch s { re\"a\" as X => 0 }"), + ("missing lex alias", "lexmatch s { re\"a\" as => 0 }"), + ("else missing block", "if x {} else"), + ] + for case in cases { + let (label, source) = case + assert_expression_acceptance(label, source, false) + } +} + +///| +test "declaration truncation matrix: functions and bindings" { + let invalid = [ + ("function body at EOF", "fn f()"), + ("function body after newline", "fn f()\n"), + ("function body before next declaration", "fn f()\nlet y = 1"), + ("return type at EOF", "fn f() ->"), + ("return type after newline", "fn f() ->\n"), + ("return type before body", "fn f() -> {}"), + ("parameter type before close", "fn f(x :) {}"), + ("parameter type after newline", "fn f(x :\n) {}"), + ("parameter type before default", "fn f(x~ : = 1) {}"), + ("let pattern at EOF", "let"), + ("let pattern after newline", "let\n"), + ("let pattern before equal", "let = 1"), + ("let value at EOF", "let x ="), + ("let value after newline", "let x =\n"), + ("let value before next declaration", "let x =\nfn f() {}"), + ("constant name at EOF", "const"), + ("constant value at EOF", "const X ="), + ("constant value before next declaration", "const X =\nfn f() {}"), + ] + for case in invalid { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid = [ + ("function body", "fn f() {}"), + ("function return type", "fn f() -> Unit {}"), + ("typed parameter", "fn f(x : Int) {}"), + ("let value", "let x = 1"), + ("constant value", "const X = 1"), + ] + for case in valid { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} + +///| +test "declaration truncation matrix: type declarations" { + let invalid = [ + ("type name at EOF", "type"), + ("type target at EOF", "type T ="), + ("type target after newline", "type T =\n"), + ("type target before next declaration", "type T =\nfn f() {}"), + ("struct name at EOF", "struct"), + ("struct body at EOF", "struct S"), + ("struct body before next declaration", "struct S\nfn f() {}"), + ("struct field type", "struct S { x : }"), + ("enum name at EOF", "enum"), + ("enum body at EOF", "enum E"), + ("enum body before next declaration", "enum E\nfn f() {}"), + ("constructor parameter type", "enum E { A(x~ :) }"), + ("constructor tag", "enum E { A = }"), + ("suberror name at EOF", "suberror"), + ("trait name at EOF", "trait"), + ("trait body at EOF", "trait T"), + ("trait body before next declaration", "trait T\nfn f() {}"), + ("supertrait before body", "trait T : {}"), + ("trait method parameter type", "trait T { f(x :) }"), + ] + for case in invalid { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid = [ + ("abstract type", "type T"), + ("type alias", "type T = Int"), + ("record struct", "struct S { x : Int }"), + ("enum", "enum E { A }"), + ("suberror", "suberror E"), + ("trait", "trait T { f(Self) }"), + ] + for case in valid { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} + +///| +test "declaration truncation matrix: impl and enumview" { + let invalid = [ + ("impl trait at EOF", "impl"), + ("impl relation marker at EOF", "impl Trait"), + ("impl self type at EOF", "impl Trait for"), + ("impl self type before with", "impl Trait for with f(self) { self }"), + ("impl method name", "impl Trait with"), + ("impl method parameters", "impl Trait with f"), + ("impl method body", "impl Trait with f(self)"), + ("impl method return type", "impl Trait with f(self) -> {}"), + ("enumview name", "enumview"), + ("enumview constructors", "enumview V"), + ("enumview for marker", "enumview V { C }"), + ("enumview source type", "enumview V { C } for"), + ( + "enumview source type before with", "enumview V { C } for with view(x) { x }", + ), + ("enumview with marker", "enumview V { C } for T"), + ("enumview method name", "enumview V { C } for T with"), + ("enumview method parameters", "enumview V { C } for T with view"), + ("enumview body", "enumview V { C } for T with view(x)"), + ] + for case in invalid { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid = [ + ("impl relation", "impl Trait for T"), + ("impl method", "impl Trait with f(self) { self }"), + ("enumview", "enumview V { C } for T with view(x) { x }"), + ] + for case in valid { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} + +///| +test "declaration truncation matrix: test using and extern" { + let invalid = [ + ("test body at EOF", "test"), + ("test body after newline", "test\n"), + ("test body before next declaration", "test\nfn f() {}"), + ("using package at EOF", "using"), + ("using body at EOF", "using @p"), + ("using body after newline", "using @p\n"), + ("using body before next declaration", "using @p\nfn f() {}"), + ("extern language at EOF", "extern"), + ("extern function marker", "extern \"C\""), + ("extern function name", "extern \"C\" fn"), + ("extern equal", "extern \"C\" fn f()"), + ("extern stub", "extern \"C\" fn f() ="), + ] + for case in invalid { + let (label, source) = case + assert_structure_acceptance(label, source, false) + } + let valid = [ + ("test", "test {}"), + ("using", "using @p {}"), + ("extern", "extern \"C\" fn f() = \"f\""), + ] + for case in valid { + let (label, source) = case + assert_structure_acceptance(label, source, true) + } +} diff --git a/untyped_cst/docstring.mbt b/untyped_cst/docstring.mbt new file mode 100644 index 00000000..5bffffc6 --- /dev/null +++ b/untyped_cst/docstring.mbt @@ -0,0 +1,103 @@ +///| +fn attach_docstrings_to_impls( + docstrings_view : ArrayView[List[(Location, Comment)]], + toplevels : @syntax.Impls, +) -> Unit { + let docstrings : Array[List[(Location, Comment)]] = [] + for comments in docstrings_view { + docstrings.push(comments) + } + + fn search_docstrings( + last_pos : Position, + loc : Location, + ) -> List[(Location, Comment)]? { + let mut result = None + while docstrings.last() is Some(comments) && + comments.head().unwrap().0.start.lnum <= loc.start.lnum { + let comments = docstrings.pop().unwrap() + let line = comments.head().unwrap().0.start.lnum + if last_pos.lnum < line && line <= loc.start.lnum { + result = Some(comments) + } + } + result + } + + fn make_doc(comments : List[(Location, Comment)]) -> @syntax.DocString { + { + content: comments.map(p => { + match p.1.content { + [.. "///|", .. remain] => + if remain.is_blank() { + "" + } else { + remain.to_owned() + } + [.. "///", .. remain] => remain.to_owned() + _ => panic() + } + }), + loc: { + start: comments.head().unwrap().0.start, + end: comments.last().unwrap().0.end, + }, + } + } + + let mut last_pos = Position::{ fname: "", lnum: -1, bol: 0, cnum: 0 } + for toplevel in toplevels { + let loc = toplevel.loc() + let doc = search_docstrings(last_pos, loc) + .map(make_doc) + .unwrap_or(@syntax.DocString::empty()) + match toplevel { + @syntax.TopTypeDef(td) => { + td.doc = doc + last_pos = loc.start + match td.components { + @syntax.Abstract + | @syntax.Alias(_) + | @syntax.Error(@syntax.NoPayload) + | @syntax.TupleStruct(_) => () + @syntax.Error(@syntax.EnumPayload(constrs)) + | @syntax.Variant(constrs) + | @syntax.ExtensibleEnum(constrs) => + constrs.each(constr => { + let previous = search_docstrings(last_pos, constr.loc) + .map(make_doc) + .unwrap_or(@syntax.DocString::empty()) + constr.doc = previous + last_pos = constr.loc.end + }) + @syntax.ExtendEnum(constructors=constrs, ..) => + constrs.each(constr => { + let previous = search_docstrings(last_pos, constr.loc) + .map(make_doc) + .unwrap_or(@syntax.DocString::empty()) + constr.doc = previous + last_pos = constr.loc.end + }) + @syntax.Record(fields~) => + fields.each(field => { + let previous = search_docstrings(last_pos, field.loc) + .map(make_doc) + .unwrap_or(@syntax.DocString::empty()) + field.doc = previous + last_pos = field.loc.end + }) + } + } + @syntax.TopFuncDef(fun_decl~, ..) => fun_decl.doc = doc + @syntax.TopLetDef(..) as ld => ld.doc = doc + @syntax.TopExpr(..) => () + @syntax.TopImplRelation(..) as imp => imp.doc = doc + @syntax.TopTest(..) as test_ => test_.doc = doc + @syntax.TopTrait(decl) => decl.doc = doc + @syntax.TopView(..) as view => view.doc = doc + @syntax.TopImpl(..) as imp => imp.doc = doc + @syntax.TopUsing(..) as using_stmt => using_stmt.doc = doc + } + last_pos = loc.end + } +} diff --git a/untyped_cst/error_recovery_test.mbt b/untyped_cst/error_recovery_test.mbt new file mode 100644 index 00000000..9d2fd17c --- /dev/null +++ b/untyped_cst/error_recovery_test.mbt @@ -0,0 +1,167 @@ +///| +fn recovery_any_node( + node : @untyped_cst.CstNode, + pred : (@untyped_cst.CstNode) -> Bool, +) -> Bool { + if pred(node) { + return true + } + for i in 0.. return true + _ => () + } + } + false +} + +///| +fn recovery_is_missing( + kind : @tokens.TokenKind, +) -> (@untyped_cst.CstNode) -> Bool { + fn(node) { + match node.kind() { + @untyped_cst.Missing(found) => found == kind + _ => false + } + } +} + +///| +fn recovery_is_error(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Error => true + _ => false + } +} + +///| +fn recovery_is_lident(name : String) -> (@untyped_cst.CstNode) -> Bool { + fn(node) { + match node.kind() { + @untyped_cst.Token(@tokens.LIDENT(found)) => found == name + _ => false + } + } +} + +///| +fn recovery_is_int(value : String) -> (@untyped_cst.CstNode) -> Bool { + fn(node) { + match node.kind() { + @untyped_cst.Token(@tokens.INT(found)) => found == value + _ => false + } + } +} + +///| +test "missing function parameter close recovers" { + let source = "fn f(" + assert_structure_acceptance("missing function parameter close", source, false) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq( + recovery_any_node( + result.root(), + recovery_is_missing(@tokens.TokenKind::TK_RPAREN), + ), + true, + ) +} + +///| +test "bad leading parameter separator keeps following parameter" { + let source = "fn f(, x)" + assert_structure_acceptance("bad leading parameter separator", source, false) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq( + recovery_any_node(result.root(), recovery_is_lident("x")), + true, + ) +} + +///| +test "missing let pattern is represented" { + let source = "let = 1" + assert_structure_acceptance("missing let pattern", source, false) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq( + recovery_any_node( + result.root(), + recovery_is_missing(@tokens.TokenKind::TK_LIDENT), + ), + true, + ) +} + +///| +test "invalid question token in block becomes error node" { + let source = "fn f() { ? }" + assert_structure_acceptance("invalid question token in block", source, false) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq(recovery_any_node(result.root(), recovery_is_error), true) +} + +///| +test "statement recovery continues after semicolon" { + let source = "fn f() { ?; 1 }" + assert_structure_acceptance( + "statement recovery after semicolon", source, false, + ) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq(recovery_any_node(result.root(), recovery_is_error), true) + @test.assert_eq(recovery_any_node(result.root(), recovery_is_int("1")), true) +} + +///| +test "lexical diagnostics are preserved for absent bad tokens" { + let source = "fn f() { @ }" + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length() > 0, true) +} + +///| +test "top-level recovery keeps following declaration" { + let source = "fn f(\nlet y = 1" + assert_structure_acceptance("top-level recovery", source, false) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq( + recovery_any_node(result.root(), recovery_is_lident("y")), + true, + ) +} + +///| +test "function literal does not split top-level declaration" { + let source = "let f = fn(x) { x }\nlet y = 1" + assert_structure_acceptance( + "function literal stays in declaration", source, true, + ) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq( + recovery_any_node(result.root(), recovery_is_lident("y")), + true, + ) +} + +///| +test "declare prefix survives recovery" { + let source = "declare fn f()\n?;\ndeclare type T" + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq( + recovery_any_node(result.root(), recovery_is_lident("f")), + true, + ) + @test.assert_eq( + recovery_any_node(result.root(), recovery_is_lident("T")), + false, + ) + @test.assert_eq(result.diagnostics_view().length() > 0, true) +} diff --git a/untyped_cst/imports.mbt b/untyped_cst/imports.mbt new file mode 100644 index 00000000..db9baa3b --- /dev/null +++ b/untyped_cst/imports.mbt @@ -0,0 +1,17 @@ +///| +using @basic {type Location, type Position, type Report} + +///| +using @tokens { + type Comment, + type InterpSource, + type Token, + type TokenKind, + type Triples, +} + +///| +using @list {type List} + +///| +using @syntax {type Impls, type DocString} diff --git a/untyped_cst/kind.mbt b/untyped_cst/kind.mbt new file mode 100644 index 00000000..733409f0 --- /dev/null +++ b/untyped_cst/kind.mbt @@ -0,0 +1,240 @@ +///| +pub(all) enum EntryPoint { + Structure + Expression +} derive(@debug.Debug) + +///| +pub(all) enum LocationMode { + DefaultLocation + Utf16Location + SourceOffsetLocation +} derive(@debug.Debug) + +///| +pub(all) enum NodeKind { + SourceFile + Fragment(EntryPoint) + Token(Token) + Missing(TokenKind) + Error + Impl(ImplKind) + Expr(ExprKind) + Pattern(PatternKind) + Type(TypeKind) + LexPattern(LexPatternKind) + RegexPattern(RegexPatternKind) + Case(CaseKind) + Argument(ArgumentKind) + Stmt(StmtKind) + Decl(DeclKind) + Clause(ClauseKind) + Name(NameKind) + List(ListKind) + InterpSegment(InterpSegmentKind) +} derive(@debug.Debug) + +///| +pub(all) enum ImplKind { + Declaration + Test + Using + Extern + BalancedDecl +} derive(Eq, @debug.Debug) + +///| +pub(all) enum ExprKind { + Placeholder + Identifier + Literal + Block + Apply + DotApply + Field + ArrayGet + ArrayGetSlice + Array + ListComprehension + Tuple + Constraint + Record + Map + Infix + Prefix + Range + Assign + AugmentedAssign + If + Match + Try + TryOperator + For + ForEach + While + Function + Method + As + Is + LexMatch + LexScan + RegexMatch + TemplateWriting + Sequence + Let + LetMut + LetRec + LetFn + Return + Raise + Break + Continue + Defer + Guard + Quantifier + Hole + Group + Constructor + MultilineString + MissingExpr +} derive(@debug.Debug) + +///| +pub(all) enum PatternKind { + Placeholder + Binder + Underscore + Literal + Tuple + Array + Rest + Record + Map + Constructor + SpecialConstructor + Or + Alias + Range + Constraint + MissingPattern +} derive(@debug.Debug) + +///| +pub(all) enum TypeKind { + Placeholder + Name + Tuple + Function + Application + Option + Object + Hole + Group + MissingType +} derive(@debug.Debug) + +///| +pub(all) enum LexPatternKind { + Placeholder + Literal + RegexInterp + Sequence + Alias + Binder + Wildcard + ConstantRef +} derive(@debug.Debug) + +///| +pub(all) enum RegexPatternKind { + Placeholder + Literal + Reference + Sequence + Alternation + Alias +} derive(@debug.Debug) + +///| +pub(all) enum CaseKind { + MatchCase + LexCase + LexScanCase +} derive(@debug.Debug) + +///| +pub(all) enum ArgumentKind { + Positional + Labelled + LabelledPun + LabelledOption + LabelledOptionPun +} derive(@debug.Debug) + +///| +pub(all) enum StmtKind { + Let + LetMut + LetRec + Function + Expr + Guard + Defer + ProofAssert + ProofLet +} derive(@debug.Debug) + +///| +pub(all) enum InterpSegmentKind { + InterpLiteralSegment + InterpSourceSegment +} derive(@debug.Debug) + +///| +pub(all) enum DeclKind { + BalancedDecl + Function + Let + Const + Type + Struct + Enum + EnumView + Trait + Impl + Test + Using + Extern + Declare +} derive(Eq, @debug.Debug) + +///| +pub(all) enum ClauseKind { + Attribute + Visibility + Parameter + ReturnType + ErrorAnnotation + Where + Body +} derive(@debug.Debug) + +///| +pub(all) enum NameKind { + Identifier + Constructor + Field + Label + Qualified +} derive(@debug.Debug) + +///| +pub(all) enum ListKind { + ArgumentList + ParameterList + FieldList + StatementList + TypeArgumentList + PatternList + ConstructorList +} derive(@debug.Debug) diff --git a/untyped_cst/lower_common.mbt b/untyped_cst/lower_common.mbt new file mode 100644 index 00000000..fd4bd074 --- /dev/null +++ b/untyped_cst/lower_common.mbt @@ -0,0 +1,320 @@ +///| +fn lower_empty_doc() -> @syntax.DocString { + @syntax.DocString::empty() +} + +///| +fn lower_empty_loc() -> Location { + let pos = initial_position("") + Location::{ start: pos, end: pos } +} + +///| +fn CstNode::first_child_with_label(self : CstNode, label : String) -> CstNode? { + for i in 0.. + return Some(child) + _ => () + } + } + None +} + +///| +fn CstNode::children_with_label( + self : CstNode, + label : String, +) -> Array[CstNode] { + let result : Array[CstNode] = [] + for i in 0.. + result.push(child) + _ => () + } + } + result +} + +///| +fn CstNode::first_token(self : CstNode) -> Token? { + match self.kind() { + Token(tok) => Some(tok) + _ => { + for i in 0.. + match child.first_token() { + Some(tok) => return Some(tok) + None => () + } + None => () + } + } + None + } + } +} + +///| +fn CstNode::first_token_child_with_label( + self : CstNode, + label : String, +) -> CstNode? { + for child in self.children_with_label(label) { + match child.kind() { + Token(_) => return Some(child) + _ => () + } + } + None +} + +///| +fn lower_token_name(tok : Token) -> String? { + match tok { + LIDENT(name) + | UIDENT(name) + | PACKAGE_NAME(name) + | DOT_LIDENT(name) + | DOT_UIDENT(name) + | POST_LABEL(name) + | INFIX1(name) + | INFIX2(name) + | INFIX3(name) + | INFIX4(name) + | AUGMENTED_ASSIGNMENT(name) + | INT(name) + | DOUBLE(name) + | FLOAT(name) + | STRING(name) + | BYTES(name) + | REGEX_LITERAL(name) + | MULTILINE_STRING(name) + | BYTE(name) + | CHAR(name) => Some(name) + TRUE => Some("true") + FALSE => Some("false") + PLUS => Some("+") + PLUS_EQUAL => Some("+") + MINUS => Some("-") + EXCLAMATION => Some("!") + AMPER => Some("&") + AMPERAMPER => Some("&&") + BAR => Some("|") + BARBAR => Some("||") + CARET => Some("^") + PIPE => Some("|>") + PIPE_LEFT => Some("<|") + IMPLIES => Some("→") + DOTDOT => Some("..") + RANGE_EXCLUSIVE => Some("..<") + RANGE_INCLUSIVE => Some("..=") + RANGE_LT_INCLUSIVE => Some("..<=") + RANGE_INCLUSIVE_REV => Some(">=..") + RANGE_EXCLUSIVE_REV => Some(">..") + READONLY => Some("readonly") + EXTEND => Some("extend") + _ => None + } +} + +///| +fn lower_token_child_name(node : CstNode) -> String? { + match node.kind() { + Token(tok) => lower_token_name(tok) + _ => + match node.first_token() { + Some(tok) => lower_token_name(tok) + None => None + } + } +} + +///| +fn lower_label_from_node(node : CstNode) -> @syntax.Label { + let name = lower_token_child_name(node).unwrap_or("") + @syntax.Label::{ name, loc: node.loc() } +} + +///| +fn lower_binder_from_node(node : CstNode) -> @syntax.Binder { + let name = lower_token_child_name(node).unwrap_or("") + @syntax.Binder::{ name, loc: node.loc() } +} + +///| +fn lower_dummy_binder(loc : Location) -> @syntax.Binder { + @syntax.Binder::{ name: "", loc } +} + +///| +fn lower_long_ident_from_name_nodes( + nodes : Array[CstNode], +) -> @syntax.LongIdent { + if nodes.length() >= 2 { + let pkg = lower_token_child_name(nodes[0]).unwrap_or("") + let id = lower_token_child_name(nodes[1]).unwrap_or("") + @syntax.LongIdent::Dot(pkg~, id~) + } else if nodes.length() == 1 { + @syntax.LongIdent::Ident( + name=lower_token_child_name(nodes[0]).unwrap_or(""), + ) + } else { + @syntax.LongIdent::dummy() + } +} + +///| +fn lower_constr_id_from_name_nodes( + nodes : Array[CstNode], + loc : Location, +) -> @syntax.ConstrId { + @syntax.ConstrId::{ id: lower_long_ident_from_name_nodes(nodes), loc } +} + +///| +fn lower_var_from_token_node(node : CstNode) -> @syntax.Var { + @syntax.Var::{ + name: @syntax.LongIdent::Ident( + name=lower_token_child_name(node).unwrap_or(""), + ), + loc: node.loc(), + } +} + +///| +fn lower_hole_expr(loc : Location) -> @syntax.Expr { + @syntax.Expr::Hole(loc~, kind=@syntax.Hole::Synthesized) +} + +///| +fn lower_unit_expr(loc : Location) -> @syntax.Expr { + @syntax.Expr::Unit(loc~, faked=false) +} + +///| +fn lower_faked_unit_expr(loc : Location) -> @syntax.Expr { + @syntax.Expr::Unit(loc~, faked=true) +} + +///| +fn[A] lower_array_to_list(items : Array[A]) -> List[A] { + @list.List(items) +} + +///| +fn lower_item_children(node : CstNode) -> Array[CstNode] { + node.children_with_label("item") +} + +///| +fn lower_constant_from_token(tok : Token) -> @syntax.Constant? { + match tok { + TRUE => Some(@syntax.Constant::Bool(true)) + FALSE => Some(@syntax.Constant::Bool(false)) + INT(value) => Some(@syntax.make_int(value)) + BYTE(value) => Some(@syntax.Constant::Byte(value)) + CHAR(value) => Some(@syntax.Constant::Char(value)) + DOUBLE(value) => Some(@syntax.Constant::Double(value)) + FLOAT(value) => Some(@syntax.make_float(value)) + STRING(value) => Some(@syntax.Constant::String(value)) + BYTES(value) => Some(@syntax.Constant::Bytes(value)) + REGEX_LITERAL(value) => Some(@syntax.Constant::Regex(value)) + _ => None + } +} + +///| +fn lower_error_type_from_clause(node : CstNode?) -> @syntax.ErrorType { + match node { + Some(clause) => + match clause.kind() { + Clause(ClauseKind::ErrorAnnotation) => lower_error_annotation(clause) + _ => + if return_clause_error_belongs_to_type(clause) { + @syntax.ErrorType::NoErrorType + } else { + match clause.first_child_with_label("clause") { + Some(error_clause) => lower_error_annotation(error_clause) + None => @syntax.ErrorType::NoErrorType + } + } + } + None => @syntax.ErrorType::NoErrorType + } +} + +///| +fn return_clause_error_belongs_to_type(clause : CstNode) -> Bool { + match clause.kind() { + Clause(ClauseKind::ReturnType) => + match + ( + clause.first_child_with_label("type"), + clause.first_child_with_label("clause"), + ) { + (Some(ty), Some(_)) => ty.kind() is Type(TypeKind::Function) + _ => false + } + _ => false + } +} + +///| +fn lower_error_annotation(node : CstNode) -> @syntax.ErrorType { + match node.first_child_with_label("op").map(CstNode::first_token) { + Some(Some(NORAISE)) => @syntax.ErrorType::Noraise(loc=node.loc()) + Some(Some(RAISE)) => + match node.first_child_with_label("type") { + Some(ty_node) => + match ty_node.first_token() { + Some(QUESTION) => + @syntax.ErrorType::MaybeError( + ty=lower_fake_error_type(node.loc()), + ) + _ => + if ty_node.kind() is Type(TypeKind::Name) && + ty_node.first_child_with_label("question") is Some(_) { + @syntax.ErrorType::ErrorType( + ty=lower_question_suffixed_error_type(ty_node), + ) + } else { + @syntax.ErrorType::ErrorType(ty=lower_type(ty_node)) + } + } + None => @syntax.ErrorType::DefaultErrorType(loc=node.loc()) + } + _ => @syntax.ErrorType::NoErrorType + } +} + +///| +fn lower_question_suffixed_error_type(node : CstNode) -> @syntax.Type { + let base_name = match node.first_child_with_label("name") { + Some(name_node) => lower_token_child_name(name_node).unwrap_or("") + None => "" + } + let name = base_name + "?" + @syntax.Type::Name( + constr_id=@syntax.ConstrId::{ + id: @syntax.LongIdent::Ident(name~), + loc: node.loc(), + }, + tys=@list.empty(), + loc=node.loc(), + ) +} + +///| +fn lower_fake_error_type(loc : Location) -> @syntax.Type { + @syntax.Type::Name( + constr_id=@syntax.ConstrId::{ + id: @syntax.LongIdent::Ident(name="Error"), + loc, + }, + tys=@list.empty(), + loc~, + ) +} diff --git a/untyped_cst/lower_decl.mbt b/untyped_cst/lower_decl.mbt new file mode 100644 index 00000000..7f0646d5 --- /dev/null +++ b/untyped_cst/lower_decl.mbt @@ -0,0 +1,924 @@ +///| +fn lower_impls_from_cst(root : CstNode) -> @syntax.Impls { + let impls : Array[@syntax.Impl] = [] + match root.kind() { + SourceFile | Fragment(Structure) => + for item in root.children_with_label("item") { + impls.push(lower_top_item(item)) + } + _ => () + } + lower_array_to_list(impls) +} + +///| +fn lower_top_item(node : CstNode) -> @syntax.Impl { + match node.kind() { + Decl(DeclKind::Function) => lower_function_decl(node) + Decl(DeclKind::Let) => lower_value_decl(node, is_constant=false) + Decl(DeclKind::Const) => lower_value_decl(node, is_constant=true) + Decl(DeclKind::Type) => lower_type_decl(node) + Decl(DeclKind::Struct) => lower_struct_decl(node) + Decl(DeclKind::Enum) => lower_enum_decl(node) + Decl(DeclKind::EnumView) => lower_enumview_decl(node) + Decl(DeclKind::Trait) => lower_trait_decl(node) + Decl(DeclKind::Impl) => lower_impl_decl(node) + Decl(DeclKind::Test) => lower_test_decl(node) + Impl(ImplKind::Using) => lower_using_decl(node) + _ => + @syntax.Impl::TopExpr( + expr=lower_hole_expr(node.loc()), + is_main=false, + is_async=None, + loc=node.loc(), + ) + } +} + +///| +fn lower_visibility(node : CstNode) -> @syntax.Visibility { + for vis in node.children_with_label("vis") { + let attr = match vis.first_child_with_label("attr") { + Some(attr_node) => lower_token_child_name(attr_node) + None => None + } + let kw = vis.first_child_with_label("kw").unwrap_or(vis) + match kw.first_token() { + Some(PUB) => return @syntax.Visibility::Pub(attr~, loc=vis.loc()) + Some(PRIV) => return @syntax.Visibility::Priv(loc=vis.loc()) + _ => () + } + } + @syntax.Visibility::Default +} + +///| +fn lower_async_loc(node : CstNode) -> Location? { + for kw in node.children_with_label("kw") { + match kw.first_token() { + Some(ASYNC) => return Some(kw.loc()) + _ => () + } + } + None +} + +///| +fn lower_is_declare(node : CstNode) -> Bool { + for kw in node.children_with_label("kw") { + match kw.first_token() { + Some(DECLARE) => return true + _ => () + } + } + false +} + +///| +fn lower_decl_name_binder(node : CstNode) -> @syntax.Binder { + match node.first_child_with_label("name") { + Some(name) => lower_binder_from_node(name) + None => lower_dummy_binder(node.loc()) + } +} + +///| +fn lower_decl_name_string(node : CstNode) -> (String, Location) { + let names = node.children_with_label("name") + if names.length() > 0 { + let name = names[names.length() - 1] + (lower_token_child_name(name).unwrap_or(""), name.loc()) + } else { + ("", node.loc()) + } +} + +///| +fn lower_decl_type_name(node : CstNode) -> @syntax.TypeName { + let names = node.children_with_label("name") + let loc = if names.length() > 0 { + names[0].loc().merge(names[names.length() - 1].loc()) + } else { + node.loc() + } + @syntax.TypeName::{ + name: lower_long_ident_from_name_nodes(names), + is_object: false, + loc, + } +} + +///| +fn lower_function_decl(node : CstNode) -> @syntax.Impl { + let name = lower_function_name_binder(node) + let (parameters, parsed_params_loc) = lower_optional_parameters_from_list( + node.first_child_with_label("params"), + ) + let params_loc = match parameters { + Some(_) => parsed_params_loc + None => empty_loc(name.loc.end) + } + let body_expr = node + .first_child_with_label("body") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(node.loc())) + let return_type = lower_return_type_from_clause( + node.first_child_with_label("clause"), + ) + let header_loc = lower_function_header_loc(node) + let fun_decl = @syntax.FunDecl::{ + type_name: lower_function_type_name(node), + name, + is_async: lower_async_loc(node), + decl_params: parameters, + params_loc, + quantifiers: lower_fun_quantifiers(node), + return_type, + error_type: lower_error_type_from_clause( + node.first_child_with_label("clause"), + ), + vis: lower_visibility(node), + attrs: lower_attrs(node), + doc: lower_empty_doc(), + loc: header_loc, + } + @syntax.Impl::TopFuncDef( + fun_decl~, + decl_body=lower_function_decl_body(node, body_expr), + where_clause=node.first_child_with_label("where").map(lower_where_clause), + loc=node.loc(), + ) +} + +///| +fn lower_function_header_loc(node : CstNode) -> Location { + let start = node.loc().start + let mut end = start + for i in 0.. () + Some((Some("op"), child)) if child.first_token() is Some(EQUAL) => () + Some((_, child)) => end = child.loc().end + None => () + } + } + Location::{ start, end } +} + +///| +fn lower_function_name_binder(node : CstNode) -> @syntax.Binder { + match node.first_child_with_label("method") { + Some(method_name) => lower_binder_from_node(method_name) + None => lower_decl_name_binder(node) + } +} + +///| +fn lower_function_type_name(node : CstNode) -> @syntax.TypeName? { + match node.first_child_with_label("method") { + Some(_) => { + let owners = node.children_with_label("name") + match owners.get(0) { + Some(owner) => + Some(@syntax.TypeName::{ + name: lower_long_ident_from_name_nodes(owners), + is_object: lower_function_owner_is_object(node), + loc: owner.loc(), + }) + None => None + } + } + None => None + } +} + +///| +fn lower_function_owner_is_object(node : CstNode) -> Bool { + for i in 0.. if child.first_token() is Some(AMPER) { return true } + None => () + } + } + false +} + +///| +fn lower_function_decl_body( + node : CstNode, + body_expr : @syntax.Expr, +) -> @syntax.DeclBody { + if lower_is_declare(node) { + return @syntax.DeclBody::DeclNone + } + let language = match node.first_child_with_label("language") { + Some(language) => lower_token_child_name(language) + None => None + } + let stubs = node.children_with_label("stub") + match stubs.get(0) { + Some(stub) => + match stub.first_token() { + Some(STRING(code)) => + match stubs.get(1) { + Some(func_stub) => + match func_stub.first_token() { + Some(STRING(func_name)) => + @syntax.DeclBody::DeclStubs( + @syntax.FuncStubs::Import( + module_name=code, + func_name~, + language~, + ), + ) + _ => + @syntax.DeclBody::DeclStubs( + @syntax.FuncStubs::Embedded( + language~, + code=@syntax.EmbeddedCode::CodeString(code), + ), + ) + } + None => + @syntax.DeclBody::DeclStubs( + @syntax.FuncStubs::Embedded( + language~, + code=@syntax.EmbeddedCode::CodeString(code), + ), + ) + } + Some(MULTILINE_STRING(_)) => { + let lines : Array[String] = [] + for stub in stubs { + match stub.first_token() { + Some(MULTILINE_STRING(code)) => lines.push(code) + _ => () + } + } + @syntax.DeclBody::DeclStubs( + @syntax.FuncStubs::Embedded( + language~, + code=@syntax.EmbeddedCode::CodeMultilineString( + lower_array_to_list(lines), + ), + ), + ) + } + _ => @syntax.DeclBody::DeclBody(expr=body_expr) + } + None => @syntax.DeclBody::DeclBody(expr=body_expr) + } +} + +///| +fn lower_fun_quantifiers(node : CstNode) -> List[@syntax.TypeVarBinder] { + let quantifiers : Array[@syntax.TypeVarBinder] = [] + match node.first_child_with_label("quantifier") { + Some(list) => + for item in lower_item_children(list) { + let name_node = item.first_child_with_label("name").unwrap_or(item) + quantifiers.push(@syntax.TypeVarBinder::{ + name: lower_token_child_name(name_node).unwrap_or(""), + name_loc: name_node.loc(), + constraints: lower_type_var_constraints(item), + }) + } + None => () + } + lower_array_to_list(quantifiers) +} + +///| +fn lower_type_var_constraints( + node : CstNode, +) -> List[@syntax.TypeVarConstraint] { + let constraints : Array[@syntax.TypeVarConstraint] = [] + for ty_node in node.children_with_label("type") { + lower_type_var_constraints_from_type(ty_node, constraints) + } + lower_array_to_list(constraints) +} + +///| +fn lower_type_var_constraints_from_type( + node : CstNode, + constraints : Array[@syntax.TypeVarConstraint], +) -> Unit { + match node.kind() { + Type(TypeKind::Name) | Type(TypeKind::Object) => + constraints.push(@syntax.TypeVarConstraint::{ + trait_: lower_type_name_from_type_node(node).name, + loc: node.loc(), + }) + Type(TypeKind::Tuple) | Type(TypeKind::Group) => + for item in node.children_with_label("type") { + lower_type_var_constraints_from_type(item, constraints) + } + _ => + constraints.push(@syntax.TypeVarConstraint::{ + trait_: @syntax.LongIdent::dummy(), + loc: node.loc(), + }) + } +} + +///| +fn lower_value_decl(node : CstNode, is_constant~ : Bool) -> @syntax.Impl { + let binder = match node.first_child_with_label("pattern") { + Some(pat) => + match lower_pattern(pat) { + @syntax.Pattern::Var(binder) => binder + _ => lower_decl_name_binder(node) + } + None => lower_decl_name_binder(node) + } + @syntax.Impl::TopLetDef( + binder~, + ty=node.first_child_with_label("type").map(lower_type), + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_unit_expr(node.loc())), + vis=lower_visibility(node), + is_constant~, + loc=node.loc(), + attrs=lower_attrs(node), + doc=lower_empty_doc(), + is_declare=lower_is_declare(node), + ) +} + +///| +fn lower_type_decl(node : CstNode) -> @syntax.Impl { + let (name, name_loc) = lower_decl_name_string(node) + let components = match node.first_child_with_label("type") { + Some(ty) => @syntax.TypeDesc::Alias(lower_type(ty)) + None => @syntax.TypeDesc::Abstract + } + @syntax.Impl::TopTypeDef(@syntax.TypeDecl::{ + tycon: name, + tycon_loc: name_loc, + params: lower_type_decl_params(node), + components, + attrs: lower_attrs(node), + doc: lower_empty_doc(), + type_vis: lower_visibility(node), + deriving: lower_deriving(node), + loc: node.loc(), + is_declare: lower_is_declare(node), + }) +} + +///| +fn lower_simple_type_decl( + node : CstNode, + components : @syntax.TypeDesc, +) -> @syntax.Impl { + let (name, name_loc) = lower_decl_name_string(node) + @syntax.Impl::TopTypeDef(@syntax.TypeDecl::{ + tycon: name, + tycon_loc: name_loc, + params: lower_type_decl_params(node), + components, + attrs: lower_attrs(node), + doc: lower_empty_doc(), + type_vis: lower_visibility(node), + deriving: lower_deriving(node), + loc: node.loc(), + is_declare: lower_is_declare(node), + }) +} + +///| +fn lower_attrs(node : CstNode) -> List[@attribute.Attribute] { + let attrs : Array[@attribute.Attribute] = [] + for attr in node.children_with_label("attrs") { + match attr.first_token() { + Some(ATTRIBUTE(content)) => + attrs.push(@attribute.Attribute::new(loc=attr.loc(), content)) + _ => () + } + } + lower_array_to_list(attrs) +} + +///| +fn lower_deriving(node : CstNode) -> List[@syntax.DerivingDirective] { + let deriving : Array[@syntax.DerivingDirective] = [] + match node.first_child_with_label("deriving") { + Some(list) => + for item in lower_item_children(list) { + let name_nodes = item.children_with_label("name") + let name_loc = if name_nodes.length() > 0 { + name_nodes[0].loc().merge(name_nodes[name_nodes.length() - 1].loc()) + } else { + item.loc() + } + deriving.push(@syntax.DerivingDirective::{ + type_name: @syntax.TypeName::{ + name: lower_long_ident_from_name_nodes(name_nodes), + is_object: false, + loc: name_loc, + }, + args: item + .first_child_with_label("arg") + .map(lower_arguments) + .unwrap_or(@list.empty()), + loc: item.loc(), + }) + } + None => () + } + lower_array_to_list(deriving) +} + +///| +fn lower_struct_decl(node : CstNode) -> @syntax.Impl { + match node.first_child_with_label("body") { + Some(body) => + match body.kind() { + List(ListKind::TypeArgumentList) => + lower_simple_type_decl( + node, + @syntax.TypeDesc::TupleStruct(lower_tuple_struct_types(body)), + ) + _ => + lower_simple_type_decl( + node, + @syntax.TypeDesc::Record(fields=lower_struct_fields(node)), + ) + } + None => + lower_simple_type_decl( + node, + @syntax.TypeDesc::Record(fields=lower_struct_fields(node)), + ) + } +} + +///| +fn lower_tuple_struct_types(body : CstNode) -> List[@syntax.Type] { + let tys : Array[@syntax.Type] = [] + for item in lower_item_children(body) { + tys.push(lower_type(item)) + } + lower_array_to_list(tys) +} + +///| +fn lower_enum_decl(node : CstNode) -> @syntax.Impl { + let mut is_extenum = false + let mut is_extend = false + for kw in node.children_with_label("kw") { + match kw.first_token() { + Some(SUBERROR) => + return lower_simple_type_decl( + node, + @syntax.TypeDesc::Error(lower_suberror_exception_decl(node)), + ) + Some(EXTENUM) => is_extenum = true + _ => () + } + } + for i in 0.. + if child.first_token() is Some(PLUS_EQUAL) { + is_extend = true + } + None => () + } + } + if is_extenum && is_extend { + return lower_simple_type_decl( + node, + @syntax.TypeDesc::ExtendEnum( + target=lower_decl_type_name(node), + constructors=lower_enum_constructors(node), + ), + ) + } + if is_extenum { + return lower_simple_type_decl( + node, + @syntax.TypeDesc::ExtensibleEnum(lower_enum_constructors(node)), + ) + } + lower_simple_type_decl( + node, + @syntax.TypeDesc::Variant(lower_enum_constructors(node)), + ) +} + +///| +fn lower_enumview_decl(node : CstNode) -> @syntax.Impl { + let (parameters, params_loc) = lower_parameters_from_list( + node.first_child_with_label("params"), + ) + let (view_type_name, view_type_loc) = lower_decl_name_string(node) + @syntax.Impl::TopView( + quantifiers=lower_fun_quantifiers(node), + source_ty=node + .first_child_with_label("type") + .map(lower_type) + .unwrap_or(@syntax.Type::Any(loc=node.loc())), + view_type_name~, + view_type_loc~, + view_func_name=node + .first_child_with_label("method") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())), + parameters~, + params_loc~, + view_constrs=lower_enum_constructors(node), + body=node + .first_child_with_label("expr") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(node.loc())), + vis=lower_visibility(node), + loc=node.loc(), + attrs=lower_attrs(node), + doc=lower_empty_doc(), + ) +} + +///| +fn lower_suberror_exception_decl(node : CstNode) -> @syntax.ExceptionDecl { + let constructors = lower_enum_constructors(node) + if constructors.length() == 0 { + @syntax.ExceptionDecl::NoPayload + } else { + @syntax.ExceptionDecl::EnumPayload(constructors) + } +} + +///| +fn lower_enum_constructors(node : CstNode) -> List[@syntax.ConstrDecl] { + let constrs : Array[@syntax.ConstrDecl] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + match item.kind() { + Name(NameKind::Constructor) => + constrs.push(lower_enum_constructor(item)) + _ => () + } + } + None => () + } + lower_array_to_list(constrs) +} + +///| +fn lower_enum_constructor(node : CstNode) -> @syntax.ConstrDecl { + let name_node = node.first_child_with_label("name").unwrap_or(node) + let args = match node.first_child_with_label("params") { + Some(params) => Some(lower_constructor_params(params)) + None => None + } + @syntax.ConstrDecl::{ + name: @syntax.ConstrName::{ + name: lower_token_child_name(name_node).unwrap_or(""), + loc: name_node.loc(), + }, + args, + tag: lower_constructor_tag(node), + loc: node.loc(), + attrs: lower_attrs(node), + doc: lower_empty_doc(), + } +} + +///| +fn lower_constructor_tag(node : CstNode) -> (String, Location)? { + match node.first_child_with_label("tag") { + Some(tag_node) => + match tag_node.first_token() { + Some(INT(value)) => Some((value, tag_node.loc())) + _ => None + } + None => None + } +} + +///| +fn lower_constructor_params(list : CstNode) -> List[@syntax.ConstrParam] { + let params : Array[@syntax.ConstrParam] = [] + for item in lower_item_children(list) { + params.push(lower_constructor_param(item)) + } + lower_array_to_list(params) +} + +///| +fn lower_constructor_param(node : CstNode) -> @syntax.ConstrParam { + let label = match node.first_child_with_label("name") { + Some(name) => + Some(@syntax.Label::{ + name: lower_token_child_name(name).unwrap_or(""), + loc: name.loc().trim_last_char(), + }) + None => None + } + @syntax.ConstrParam::{ + ty: node + .first_child_with_label("type") + .map(lower_type) + .unwrap_or(@syntax.Type::Any(loc=node.loc())), + mut_: node.first_child_with_label("mut") is Some(_), + label, + } +} + +///| +fn lower_struct_fields(node : CstNode) -> List[@syntax.FieldDecl] { + let fields : Array[@syntax.FieldDecl] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + match item.kind() { + Clause(ClauseKind::Parameter) => fields.push(lower_struct_field(item)) + _ => () + } + } + None => () + } + lower_array_to_list(fields) +} + +///| +fn lower_struct_field(node : CstNode) -> @syntax.FieldDecl { + let name_node = node.first_child_with_label("name").unwrap_or(node) + let label = lower_token_child_name(name_node).unwrap_or("") + @syntax.FieldDecl::{ + name: @syntax.FieldName::{ label, loc: name_node.loc() }, + ty: node + .first_child_with_label("type") + .map(lower_type) + .unwrap_or(@syntax.Type::Any(loc=node.loc())), + mut_: node.first_child_with_label("mut") is Some(_), + vis: lower_visibility(node), + loc: node.loc(), + attrs: lower_attrs(node), + doc: lower_empty_doc(), + } +} + +///| +fn lower_type_decl_params(node : CstNode) -> List[@syntax.TypeDeclBinder] { + let params : Array[@syntax.TypeDeclBinder] = [] + match node.first_child_with_label("params") { + Some(list) => + for item in lower_item_children(list) { + let name_node = item.first_child_with_label("name").unwrap_or(item) + let name = match name_node.first_token() { + Some(UNDERSCORE) => None + Some(tok) => lower_token_name(tok) + None => None + } + params.push(@syntax.TypeDeclBinder::{ name, loc: name_node.loc() }) + } + None => () + } + lower_array_to_list(params) +} + +///| +fn lower_trait_decl(node : CstNode) -> @syntax.Impl { + @syntax.Impl::TopTrait(@syntax.TraitDecl::{ + name: lower_decl_name_binder(node), + supers: lower_type_var_constraints(node), + methods: lower_trait_methods(node), + vis: lower_visibility(node), + loc: node.loc(), + attrs: lower_attrs(node), + doc: lower_empty_doc(), + is_declare: lower_is_declare(node), + }) +} + +///| +fn lower_trait_methods(node : CstNode) -> List[@syntax.TraitMethodDecl] { + let methods : Array[@syntax.TraitMethodDecl] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + match item.kind() { + Decl(DeclKind::Function) => methods.push(lower_trait_method(item)) + _ => () + } + } + None => () + } + lower_array_to_list(methods) +} + +///| +fn lower_trait_method(node : CstNode) -> @syntax.TraitMethodDecl { + let (params, _) = lower_trait_method_parameters( + node.first_child_with_label("params"), + ) + @syntax.TraitMethodDecl::{ + name: lower_decl_name_binder(node), + is_async: lower_async_loc(node), + quantifiers: lower_fun_quantifiers(node), + params, + return_type: lower_return_type_from_clause( + node.first_child_with_label("clause"), + ), + error_type: lower_error_type_from_clause( + node.first_child_with_label("clause"), + ), + has_default: node.first_child_with_label("stub").map(CstNode::loc), + attrs: lower_attrs(node), + loc: node.loc(), + } +} + +///| +fn lower_trait_method_parameters( + list : CstNode?, +) -> (List[@syntax.Parameter], Location) { + match list { + Some(list) => { + let params : Array[@syntax.Parameter] = [] + for item in lower_item_children(list) { + params.push(lower_trait_method_parameter(item)) + } + (lower_array_to_list(params), list.loc()) + } + None => (@list.empty(), lower_empty_loc()) + } +} + +///| +fn lower_trait_method_parameter(node : CstNode) -> @syntax.Parameter { + let ty = node.first_child_with_label("type").map(lower_type) + match node.first_child_with_label("pattern") { + Some(pat_node) => { + let binder = lower_binder_from_node(pat_node) + match pat_node.first_token() { + Some(POST_LABEL(_)) => @syntax.Parameter::Labelled(binder~, ty~) + _ => + if lower_trait_parameter_is_question_optional(node) { + @syntax.Parameter::QuestionOptional(binder~, ty~) + } else { + @syntax.Parameter::Positional(binder~, ty~) + } + } + } + None => @syntax.Parameter::DiscardPositional(ty~, loc=node.loc()) + } +} + +///| +fn lower_trait_parameter_is_question_optional(node : CstNode) -> Bool { + match node.first_token_child_with_label("op") { + Some(op) => + match op.first_token() { + Some(QUESTION) => true + _ => false + } + None => false + } +} + +///| +fn lower_impl_decl(node : CstNode) -> @syntax.Impl { + match node.first_child_with_label("method") { + Some(method_node) => return lower_impl_method_decl(node, method_node) + None => () + } + let trait_name = lower_decl_type_name(node) + @syntax.Impl::TopImplRelation( + self_ty=node + .first_child_with_label("type") + .map(lower_type) + .unwrap_or(@syntax.Type::Any(loc=node.loc())), + trait_=trait_name, + quantifiers=lower_fun_quantifiers(node), + vis=lower_visibility(node), + attrs=lower_attrs(node), + loc=node.loc(), + doc=lower_empty_doc(), + is_declare=lower_is_declare(node), + ) +} + +///| +fn lower_impl_method_decl( + node : CstNode, + method_node : CstNode, +) -> @syntax.Impl { + let trait_name = lower_decl_type_name(node) + let (params, _) = lower_parameters_from_list( + method_node.first_child_with_label("params"), + ) + @syntax.Impl::TopImpl( + self_ty=node.first_child_with_label("type").map(lower_type), + trait_=trait_name, + method_name=lower_decl_name_binder(method_node), + quantifiers=lower_fun_quantifiers(node), + method_quantifiers=lower_fun_quantifiers(method_node), + params~, + ret_ty=lower_return_type_from_clause( + method_node.first_child_with_label("clause"), + ), + err_ty=lower_error_type_from_clause( + method_node.first_child_with_label("clause"), + ), + body=lower_function_decl_body( + method_node, + method_node + .first_child_with_label("body") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(method_node.loc())), + ), + vis=lower_visibility(node), + loc=node.loc(), + attrs=lower_attrs(node), + doc=lower_empty_doc(), + ) +} + +///| +fn lower_test_decl(node : CstNode) -> @syntax.Impl { + let expr = node + .first_child_with_label("body") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(node.loc())) + let name : @syntax.TestName = match node.first_child_with_label("name") { + Some(name_node) => + match name_node.first_token() { + Some(STRING(value)) => Some((value, name_node.loc())) + _ => None + } + None => None + } + let (params, _) = lower_optional_parameters_from_list( + node.first_child_with_label("params"), + ) + @syntax.Impl::TopTest( + expr~, + name~, + params~, + is_async=lower_async_loc(node), + loc=node.loc(), + attrs=lower_attrs(node), + doc=lower_empty_doc(), + ) +} + +///| +fn lower_using_decl(node : CstNode) -> @syntax.Impl { + let pkg = node + .first_child_with_label("name") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: node.loc() }) + @syntax.Impl::TopUsing( + pkg~, + names=lower_using_names(node), + vis=lower_visibility(node), + attrs=lower_attrs(node), + loc=node.loc(), + doc=lower_empty_doc(), + ) +} + +///| +fn lower_using_names( + node : CstNode, +) -> List[(@syntax.AliasTarget, @syntax.UsingKind)] { + let names : Array[(@syntax.AliasTarget, @syntax.UsingKind)] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + let target_node = item.first_child_with_label("target").unwrap_or(item) + let binder_node = item + .first_child_with_label("name") + .unwrap_or(target_node) + let target = if item.first_child_with_label("name") is Some(_) { + Some(lower_label_from_node(target_node)) + } else { + None + } + let using_kind = match + item.first_child_with_label("kind").map(CstNode::first_token) { + Some(Some(TYPE)) => @syntax.UsingKind::Type + Some(Some(TRAIT)) => @syntax.UsingKind::Trait + _ => @syntax.UsingKind::Value + } + names.push( + ( + @syntax.AliasTarget::{ + binder: lower_binder_from_node(binder_node), + target, + }, + using_kind, + ), + ) + } + None => () + } + lower_array_to_list(names) +} diff --git a/untyped_cst/lower_expr.mbt b/untyped_cst/lower_expr.mbt new file mode 100644 index 00000000..0d6ced56 --- /dev/null +++ b/untyped_cst/lower_expr.mbt @@ -0,0 +1,2275 @@ +///| +fn lower_expr(node : CstNode) -> @syntax.Expr { + match node.kind() { + Fragment(Expression) => + node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + Expr(ExprKind::Identifier) => + @syntax.Expr::Ident( + id=@syntax.Var::{ + name: lower_long_ident_from_name_nodes( + node.children_with_label("name"), + ), + loc: node.loc(), + }, + loc=node.loc(), + ) + Expr(ExprKind::Constructor) => lower_constructor_expr(node) + Expr(ExprKind::Literal) => lower_literal_expr(node) + Expr(ExprKind::MultilineString) => lower_multiline_string_expr(node) + Expr(ExprKind::Hole) => + match node.first_token() { + Some(ELLIPSIS) => + @syntax.Expr::Hole(loc=node.loc(), kind=@syntax.Hole::Todo) + _ => @syntax.Expr::Hole(loc=node.loc(), kind=@syntax.Hole::Incomplete) + } + Expr(ExprKind::MissingExpr) => lower_hole_expr(node.loc()) + Expr(ExprKind::Tuple) => lower_tuple_expr(node) + Expr(ExprKind::Constraint) => lower_constraint_expr(node) + Expr(ExprKind::Array) => lower_array_expr_node(node) + Expr(ExprKind::ListComprehension) => lower_list_comprehension_expr(node) + Expr(ExprKind::Record) => lower_record_expr_node(node) + Expr(ExprKind::Map) => lower_map_expr_node(node) + Expr(ExprKind::Block) => lower_block_expr_node(node) + Expr(ExprKind::Infix) => lower_infix_expr(node) + Expr(ExprKind::Range) => lower_infix_expr(node) + Expr(ExprKind::Prefix) => lower_prefix_expr(node) + Expr(ExprKind::Apply) => lower_apply_expr(node) + Expr(ExprKind::DotApply) => lower_dot_apply_expr(node) + Expr(ExprKind::Method) => lower_method_expr(node) + Expr(ExprKind::Field) => lower_field_expr(node) + Expr(ExprKind::ArrayGet) => lower_array_get_expr(node) + Expr(ExprKind::ArrayGetSlice) => lower_array_get_slice_expr(node) + Expr(ExprKind::If) => lower_if_expr(node) + Expr(ExprKind::Match) => lower_match_expr(node) + Expr(ExprKind::Try) => lower_try_expr(node) + Expr(ExprKind::TryOperator) => lower_try_operator_expr(node) + Expr(ExprKind::LexMatch) => lower_lexmatch_expr(node) + Expr(ExprKind::LexScan) => lower_lexscan_expr(node) + Expr(ExprKind::For) => lower_for_expr(node) + Expr(ExprKind::ForEach) => lower_foreach_expr(node) + Expr(ExprKind::While) => lower_while_expr(node) + Expr(ExprKind::Function) => lower_function_expr_node(node) + Expr(ExprKind::LetFn) => + lower_letfn_expr_with_body(node, lower_faked_unit_expr(node.loc())) + Expr(ExprKind::LetRec) => + lower_letrec_expr_with_body(node, lower_faked_unit_expr(node.loc())) + Expr(ExprKind::Let) | Expr(ExprKind::LetMut) => + lower_let_expr_without_body(node) + Expr(ExprKind::Assign) | Expr(ExprKind::AugmentedAssign) => + lower_assign_expr(node) + Expr(ExprKind::Guard) => + lower_guard_expr_with_body(node, lower_unit_expr(node.loc())) + Expr(ExprKind::Defer) => + lower_defer_expr_with_body(node, lower_unit_expr(node.loc())) + Stmt(StmtKind::ProofAssert) => lower_proof_assert_expr(node) + Stmt(StmtKind::ProofLet) => lower_proof_let_expr(node) + Expr(ExprKind::Return) => lower_return_expr(node) + Expr(ExprKind::Raise) => lower_raise_expr(node) + Expr(ExprKind::Break) => lower_break_expr(node) + Expr(ExprKind::Continue) => lower_continue_expr(node) + Expr(ExprKind::Group) => + node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + Expr(ExprKind::As) => lower_as_expr(node) + Expr(ExprKind::Is) => lower_is_expr(node) + Expr(ExprKind::RegexMatch) => lower_regex_match_expr(node) + Expr(ExprKind::TemplateWriting) => lower_template_writing_expr(node) + Expr(ExprKind::Quantifier) => lower_quantifier_expr(node) + _ => lower_hole_expr(node.loc()) + } +} + +///| +fn lower_quantifier_expr(node : CstNode) -> @syntax.Expr { + let kind = match node.first_child_with_label("kw").map(CstNode::first_token) { + Some(Some(EXISTS)) => @syntax.PredicateQuantifierKind::Exists + _ => @syntax.PredicateQuantifierKind::Forall + } + @syntax.Expr::Quantifier( + kind~, + binder=node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())), + binder_ty=node + .first_child_with_label("type") + .map(lower_type) + .unwrap_or(@syntax.Type::Any(loc=node.loc())), + body=node + .first_child_with_label("body") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + loc=node.loc(), + ) +} + +///| +fn lower_literal_expr(node : CstNode) -> @syntax.Expr { + match node.first_token() { + Some(INTERP(_)) => + @syntax.Expr::Interp(elems=lower_interp_elems(node), loc=node.loc()) + Some(BYTES_INTERP(_)) => + @syntax.Expr::BytesInterp(elems=lower_interp_elems(node), loc=node.loc()) + Some(tok) => + match lower_constant_from_token(tok) { + Some(c) => @syntax.Expr::Constant(c~, loc=node.loc()) + None => lower_hole_expr(node.loc()) + } + None => lower_hole_expr(node.loc()) + } +} + +///| +fn lower_template_writing_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::TemplateWriting( + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + template=node + .first_child_with_label("template") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + is_conditional=node.first_child_with_label("op").map(CstNode::first_token) + is Some(Some(LT_QUESTION)), + loc=node.loc(), + ) +} + +///| +fn lower_interp_elems(node : CstNode) -> List[@syntax.InterpElem] { + let elems : Array[@syntax.InterpElem] = [] + for segment in node.children_with_label("segment") { + elems.push(lower_interp_segment(segment)) + } + lower_array_to_list(elems) +} + +///| +fn lower_child_token_name(node : CstNode, label : String) -> String { + match node.first_child_with_label(label) { + Some(child) => lower_token_child_name(child).unwrap_or("") + None => "" + } +} + +///| +fn lower_interp_segment(node : CstNode) -> @syntax.InterpElem { + match node.kind() { + InterpSegment(InterpSegmentKind::InterpLiteralSegment) => + @syntax.InterpElem::Literal( + repr=lower_child_token_name(node, "repr"), + loc=node.loc(), + ) + InterpSegment(InterpSegmentKind::InterpSourceSegment) => + @syntax.InterpElem::Source(@tokens.InterpSource::{ + source: lower_child_token_name(node, "source"), + loc: node.loc(), + }) + _ => + @syntax.InterpElem::Source(@tokens.InterpSource::{ + source: "", + loc: node.loc(), + }) + } +} + +///| +fn lower_multiline_string_expr(node : CstNode) -> @syntax.Expr { + let elems : Array[@syntax.MultilineStringElem] = [] + let mut interp_elems : Array[@syntax.InterpElem] = [] + let mut collecting_interp = false + for i in 0.. + match token_node.first_token() { + Some(MULTILINE_STRING(value)) => { + if collecting_interp { + elems.push( + @syntax.MultilineStringElem::Interp( + lower_array_to_list(interp_elems), + ), + ) + interp_elems = [] + collecting_interp = false + } + elems.push(@syntax.MultilineStringElem::String(value)) + } + Some(MULTILINE_INTERP(_)) => { + if collecting_interp { + elems.push( + @syntax.MultilineStringElem::Interp( + lower_array_to_list(interp_elems), + ), + ) + interp_elems = [] + } + collecting_interp = true + } + _ => () + } + Some((Some("segment"), segment)) => + if collecting_interp { + interp_elems.push(lower_interp_segment(segment)) + } + _ => () + } + } + if collecting_interp { + elems.push( + @syntax.MultilineStringElem::Interp(lower_array_to_list(interp_elems)), + ) + } + @syntax.Expr::MultilineString( + elems=lower_array_to_list(elems), + loc=node.loc(), + ) +} + +///| +fn lower_constructor_expr(node : CstNode) -> @syntax.Expr { + let constr = match node.first_child_with_label("field") { + Some(field) => + @syntax.Constructor::{ + name: @syntax.ConstrName::{ + name: lower_token_child_name(field).unwrap_or(""), + loc: field.loc(), + }, + extra_info: lower_constructor_expr_extra_info(node), + loc: node.loc(), + } + None => + lower_constructor_from_names(node.children_with_label("name"), node.loc()) + } + @syntax.Expr::Constr(constr~, loc=node.loc()) +} + +///| +fn lower_constructor_expr_extra_info( + node : CstNode, +) -> @syntax.ConstructorExtraInfo { + match node.first_child_with_label("type") { + Some(type_node) => { + let type_name = lower_type_name_from_expr_node(type_node) + match node.first_child_with_label("pkg") { + Some(pkg_node) => + @syntax.ConstructorExtraInfo::TypeNameWithConstrPackage( + type_name~, + pkg=lower_token_child_name(pkg_node).unwrap_or(""), + ) + None => @syntax.ConstructorExtraInfo::TypeName(type_name) + } + } + None => + match node.first_child_with_label("pkg") { + Some(pkg_node) => + @syntax.ConstructorExtraInfo::Package( + lower_token_child_name(pkg_node).unwrap_or(""), + ) + None => @syntax.ConstructorExtraInfo::NoExtraInfo + } + } +} + +///| +fn lower_argument_kind( + item : CstNode, + labelled_pun_loc : Location, +) -> @syntax.ArgumentKind { + match item.kind() { + Argument(ArgumentKind::Labelled) => + @syntax.ArgumentKind::Labelled( + item + .first_child_with_label("name") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: item.loc() }), + ) + Argument(ArgumentKind::LabelledPun) => + @syntax.ArgumentKind::LabelledPun( + item + .first_child_with_label("name") + .map(name => @syntax.Label::{ + name: lower_token_child_name(name).unwrap_or(""), + loc: labelled_pun_loc, + }) + .unwrap_or(@syntax.Label::{ name: "", loc: item.loc() }), + ) + Argument(ArgumentKind::LabelledOption) => + @syntax.ArgumentKind::LabelledOption( + label=item + .first_child_with_label("name") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: item.loc() }), + question_loc=item + .first_child_with_label("op") + .map(CstNode::loc) + .unwrap_or(item.loc()), + ) + Argument(ArgumentKind::LabelledOptionPun) => + @syntax.ArgumentKind::LabelledOptionPun( + label=item + .first_child_with_label("name") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: item.loc() }), + question_loc=item + .first_child_with_label("op") + .map(CstNode::loc) + .unwrap_or(item.loc()), + ) + _ => @syntax.ArgumentKind::Positional + } +} + +///| +fn lower_arguments(list : CstNode) -> List[@syntax.Argument] { + let args : Array[@syntax.Argument] = [] + for i in 0.. + item + .first_child_with_label("name") + .map(name => { + lower_argument_pun_value(name, following_loc.trim_last_char()) + }) + .unwrap_or(lower_hole_expr(item.loc())) + Argument(ArgumentKind::LabelledOptionPun) => + item + .first_child_with_label("name") + .map(name => lower_argument_pun_value(name, name.loc())) + .unwrap_or(lower_hole_expr(item.loc())) + Argument(_) => + item + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(item.loc())) + _ => lower_expr(item) + } + args.push(@syntax.Argument::{ + value, + kind: lower_argument_kind(item, following_loc), + }) + } + lower_array_to_list(args) +} + +///| +fn lower_argument_following_loc( + list : CstNode, + item_index : Int, + fallback : Location, +) -> Location { + for i in (item_index + 1).. return child.loc() + _ => () + } + } + fallback +} + +///| +fn lower_argument_pun_value(node : CstNode, loc : Location) -> @syntax.Expr { + @syntax.Expr::Ident( + id=@syntax.Var::{ + name: @syntax.LongIdent::Ident( + name=lower_token_child_name(node).unwrap_or(""), + ), + loc, + }, + loc~, + ) +} + +///| +fn lower_tuple_expr(node : CstNode) -> @syntax.Expr { + let exprs : Array[@syntax.Expr] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + exprs.push(lower_expr(item)) + } + None => () + } + if exprs.length() == 0 { + @syntax.Expr::Unit(loc=node.loc(), faked=false) + } else if exprs.length() == 1 { + @syntax.Expr::Group( + expr=exprs[0], + group=@syntax.Group::Paren, + loc=node.loc(), + ) + } else { + @syntax.Expr::Tuple(exprs=lower_array_to_list(exprs), loc=node.loc()) + } +} + +///| +fn lower_constraint_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::Constraint( + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + ty=node + .first_child_with_label("type") + .map(lower_type) + .unwrap_or(@syntax.Type::Any(loc=node.loc())), + loc=node.loc(), + ) +} + +///| +fn lower_array_expr_node(node : CstNode) -> @syntax.Expr { + let elems : Array[@syntax.SpreadableElem] = [] + let mut is_iter = false + match node.first_child_with_label("body") { + Some(body) => { + is_iter = match + body.first_child_with_label("open").map(CstNode::first_token) { + Some(Some(LBRACKET_BAR)) => true + _ => false + } + for item in lower_item_children(body) { + elems.push(lower_array_expr_elem(item)) + } + } + None => () + } + @syntax.make_array_expr(loc=node.loc(), is_iter~, lower_array_to_list(elems)) +} + +///| +fn lower_array_expr_elem(item : CstNode) -> @syntax.SpreadableElem { + match item.first_child_with_label("op").map(CstNode::first_token) { + Some(Some(DOTDOT)) => + @syntax.SpreadableElem::Spread( + expr=item + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(item.loc())), + loc=item.loc(), + ) + _ => @syntax.SpreadableElem::Regular(lower_expr(item)) + } +} + +///| +fn lower_list_comprehension_expr(node : CstNode) -> @syntax.Expr { + let is_iter = match + node.first_child_with_label("open").map(CstNode::first_token) { + Some(Some(LBRACKET_BAR)) => true + _ => false + } + let kind = match node.first_child_with_label("pattern") { + Some(_) => + @syntax.ListComprehensionKind::Foreach( + binders=lower_foreach_binders(node), + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + init=lower_for_binding_list(node, "init"), + continue_block=lower_for_binding_list(node, "continue"), + ) + None => + @syntax.ListComprehensionKind::For( + binders=lower_for_binding_list(node, "init"), + condition=node.first_child_with_label("cond").map(lower_expr), + continue_block=lower_for_binding_list(node, "continue"), + for_loc=node + .first_child_with_label("kw") + .map(CstNode::loc) + .unwrap_or(node.loc()), + ) + } + @syntax.Expr::ListComprehension( + kind~, + guard_=node.first_child_with_label("guard").map(lower_expr), + body=node + .first_child_with_label("body") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + is_iter~, + loc=node.loc(), + ) +} + +///| +fn lower_record_expr_node(node : CstNode) -> @syntax.Expr { + let fields : Array[@syntax.FieldDef] = [] + let mut record : @syntax.Expr? = None + let mut trailing = @syntax.TrailingMark::None + match node.first_child_with_label("body") { + Some(body) => { + trailing = lower_record_trailing_mark(body) + for item in lower_item_children(body) { + match item.first_child_with_label("op") { + Some(op) => + match op.first_token() { + Some(DOTDOT) => + record = item.first_child_with_label("expr").map(lower_expr) + _ => fields.push(lower_record_field_def(item)) + } + None => fields.push(lower_record_field_def(item)) + } + } + } + None => () + } + let type_name = node + .first_child_with_label("type") + .map(lower_type_name_from_expr_node) + match record { + Some(record) => + @syntax.Expr::RecordUpdate( + type_name~, + record~, + fields=lower_array_to_list(fields), + loc=node.loc(), + ) + None => + @syntax.Expr::Record( + type_name~, + fields=lower_array_to_list(fields), + trailing~, + loc=node.loc(), + ) + } +} + +///| +fn lower_record_field_def(node : CstNode) -> @syntax.FieldDef { + let label_node = node.first_child_with_label("name") + let label = label_node + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: node.loc() }) + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or( + @syntax.Expr::Ident( + id=@syntax.Var::{ + name: @syntax.LongIdent::Ident(name=label.name), + loc: label.loc, + }, + loc=label.loc, + ), + ) + @syntax.FieldDef::{ + label, + expr, + is_pun: node.first_child_with_label("expr") is None, + loc: node.loc(), + } +} + +///| +fn lower_where_clause(node : CstNode) -> @syntax.WhereClause { + let fields : Array[@syntax.FieldDef] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + fields.push(lower_record_field_def(item)) + } + None => () + } + @syntax.WhereClause::{ fields: lower_array_to_list(fields), loc: node.loc() } +} + +///| +fn lower_record_trailing_mark(node : CstNode) -> @syntax.TrailingMark { + let mut trailing = @syntax.TrailingMark::None + for i in 0.. trailing = @syntax.TrailingMark::None + Some((Some("sep"), sep)) => + match sep.first_token() { + Some(COMMA) => trailing = @syntax.TrailingMark::Comma + Some(SEMI(_)) => trailing = @syntax.TrailingMark::Semi + _ => () + } + _ => () + } + } + trailing +} + +///| +fn lower_map_expr_node(node : CstNode) -> @syntax.Expr { + let elems : Array[@syntax.MapExprElem] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + elems.push(lower_map_expr_elem(item)) + } + None => () + } + @syntax.Expr::Map(elems=lower_array_to_list(elems), loc=node.loc()) +} + +///| +fn lower_map_expr_elem(node : CstNode) -> @syntax.MapExprElem { + let key_node = node.first_child_with_label("key") + let (key, key_loc) = match key_node { + Some(key_node) => + match key_node.first_token() { + Some(tok) => { + let key = if node + .first_child_with_label("op") + .map(CstNode::first_token) + is Some(Some(MINUS)) { + match tok { + INT(value) => Some(@syntax.make_int("-\{value}")) + DOUBLE(value) => Some(@syntax.Constant::Double("-\{value}")) + FLOAT(value) => Some(@syntax.make_float("-\{value}")) + _ => lower_constant_from_token(tok) + } + } else { + lower_constant_from_token(tok) + } + match key { + Some(key) => (key, key_node.loc()) + None => (@syntax.Constant::String(""), key_node.loc()) + } + } + None => (@syntax.Constant::String(""), key_node.loc()) + } + None => (@syntax.Constant::String(""), node.loc()) + } + @syntax.MapExprElem::{ + key, + expr: node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + key_loc, + loc: node.loc(), + } +} + +///| +fn lower_infix_expr(node : CstNode) -> @syntax.Expr { + let lhs = node + .first_child_with_label("lhs") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let rhs = node + .first_child_with_label("rhs") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let op_node = node.first_child_with_label("op") + let op_token = match op_node { + Some(n) => n.first_token() + None => None + } + match op_token { + Some(PIPE) => @syntax.Expr::Pipe(lhs~, rhs~, loc=node.loc()) + Some(PIPE_LEFT) => @syntax.Expr::RevPipe(lhs~, rhs~, loc=node.loc()) + Some(IMPLIES) => @syntax.Expr::Implies(lhs~, rhs~, loc=node.loc()) + Some(tok) => { + let op_var = @syntax.Var::{ + name: @syntax.LongIdent::Ident(name=lower_token_name(tok).unwrap_or("")), + loc: op_node.unwrap().loc(), + } + @syntax.Expr::Infix(op=op_var, lhs~, rhs~, loc=node.loc()) + } + None => + @syntax.Expr::Infix( + op=@syntax.Var::{ + name: @syntax.LongIdent::Ident(name=""), + loc: node.loc(), + }, + lhs~, + rhs~, + loc=node.loc(), + ) + } +} + +///| +fn lower_prefix_expr(node : CstNode) -> @syntax.Expr { + let op_token = match node.first_child_with_label("op") { + Some(op_node) => op_node.first_token() + None => None + } + let expr_token = match node.first_child_with_label("expr") { + Some(expr_node) => expr_node.first_token() + None => None + } + match (op_token, expr_token) { + (Some(PLUS), Some(INT(_))) | (Some(PLUS), Some(DOUBLE(_))) => + return node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + (Some(MINUS), Some(INT(value))) if !value.has_suffix("N") => + return @syntax.Expr::Constant( + c=@syntax.make_int("-\{value}"), + loc=node.loc(), + ) + _ => () + } + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let op_node = node.first_child_with_label("op").unwrap_or(node) + let op_name = match op_node.first_token() { + Some(tok) => lower_token_name(tok).unwrap_or("") + None => "" + } + if op_token is Some(PLUS) { + return @syntax.make_uplus(loc=node.loc(), "+", expr) + } + if op_token is Some(MINUS) { + return @syntax.make_uminus(loc=node.loc(), op_name, expr) + } + if op_token is Some(EXCLAMATION) { + return @syntax.make_unot(loc=node.loc(), expr) + } + let op = @syntax.Var::{ + name: @syntax.LongIdent::Ident(name=op_name), + loc: op_node.loc(), + } + @syntax.Expr::Unary(op~, expr~, loc=node.loc()) +} + +///| +fn lower_while_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::While( + loop_cond=node + .first_child_with_label("cond") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + loop_body=node + .first_child_with_label("body") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(node.loc())), + while_else=node.first_child_with_label("else").map(lower_function_body_expr), + label=node.first_child_with_label("label").map(lower_label_from_node), + loc=node.loc(), + ) +} + +///| +fn lower_assign_expr(node : CstNode) -> @syntax.Expr { + let lhs = node.first_child_with_label("lhs") + match lhs.map(lower_expr) { + Some(@syntax.Expr::Ident(id~, ..)) => { + let expr = node + .first_child_with_label("rhs") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let augmented_by = match node.kind() { + Expr(ExprKind::AugmentedAssign) => + node.first_child_with_label("op").map(lower_var_from_token_node) + _ => None + } + @syntax.Expr::Assign(var_=id, expr~, augmented_by~, loc=node.loc()) + } + Some(@syntax.Expr::ArrayGet(array~, index~, ..)) => { + let value = node + .first_child_with_label("rhs") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + match node.kind() { + Expr(ExprKind::AugmentedAssign) => { + let op = node + .first_child_with_label("op") + .map(lower_var_from_token_node) + .unwrap_or(@syntax.Var::{ + name: @syntax.LongIdent::Ident(name=""), + loc: node.loc(), + }) + @syntax.Expr::ArrayAugmentedSet( + op~, + array~, + index~, + value~, + loc=node.loc(), + ) + } + _ => @syntax.Expr::ArraySet(array~, index~, value~, loc=node.loc()) + } + } + Some(@syntax.Expr::Field(record~, accessor~, ..)) => { + let field = node + .first_child_with_label("rhs") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let augmented_by = match node.kind() { + Expr(ExprKind::AugmentedAssign) => + node.first_child_with_label("op").map(lower_var_from_token_node) + _ => None + } + @syntax.Expr::Mutate( + record~, + accessor~, + field~, + augmented_by~, + loc=node.loc(), + ) + } + Some(_) => lower_hole_expr(node.loc()) + None => lower_hole_expr(node.loc()) + } +} + +///| +fn lower_for_binding(node : CstNode) -> (@syntax.Binder, @syntax.Expr) { + ( + node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())), + node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_unit_expr(node.loc())), + ) +} + +///| +fn lower_for_binding_list( + node : CstNode, + label : String, +) -> List[(@syntax.Binder, @syntax.Expr)] { + let bindings : Array[(@syntax.Binder, @syntax.Expr)] = [] + match node.first_child_with_label(label) { + Some(list) => + for item in lower_item_children(list) { + bindings.push(lower_for_binding(item)) + } + None => () + } + lower_array_to_list(bindings) +} + +///| +fn lower_for_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::For( + binders=lower_for_binding_list(node, "init"), + condition=node.first_child_with_label("cond").map(lower_expr), + continue_block=lower_for_binding_list(node, "continue"), + body=node + .first_child_with_label("body") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(node.loc())), + for_else=node.first_child_with_label("else").map(lower_function_body_expr), + where_clause=node.first_child_with_label("where").map(lower_where_clause), + label=node.first_child_with_label("label").map(lower_label_from_node), + loc=node.loc(), + ) +} + +///| +fn lower_foreach_binders(node : CstNode) -> List[@syntax.Binder?] { + let binders : Array[@syntax.Binder?] = [] + match node.first_child_with_label("pattern") { + Some(pattern) => + match lower_pattern(pattern) { + @syntax.Pattern::Var(binder) => binders.push(Some(binder)) + @syntax.Pattern::Tuple(pats~, ..) => + for pat in pats { + match pat { + @syntax.Pattern::Var(binder) => binders.push(Some(binder)) + @syntax.Pattern::Any(..) => binders.push(None) + _ => binders.push(Some(lower_dummy_binder(pattern.loc()))) + } + } + @syntax.Pattern::Any(..) => binders.push(None) + _ => binders.push(Some(lower_dummy_binder(pattern.loc()))) + } + None => () + } + lower_array_to_list(binders) +} + +///| +fn lower_foreach_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::ForEach( + binders=lower_foreach_binders(node), + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + init=lower_for_binding_list(node, "init"), + continue_block=lower_for_binding_list(node, "continue"), + body=node + .first_child_with_label("body") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(node.loc())), + else_block=node.first_child_with_label("else").map(lower_function_body_expr), + where_clause=node.first_child_with_label("where").map(lower_where_clause), + label=node.first_child_with_label("label").map(lower_label_from_node), + loc=node.loc(), + ) +} + +///| +fn lower_apply_expr(node : CstNode) -> @syntax.Expr { + let func = node + .first_child_with_label("func") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let args = node + .first_child_with_label("arg") + .map(lower_arguments) + .unwrap_or(@list.empty()) + @syntax.Expr::Apply(func~, args~, loc=node.loc()) +} + +///| +fn lower_dot_apply_expr(node : CstNode) -> @syntax.Expr { + let self_expr = node + .first_child_with_label("self") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let method_label = node + .first_child_with_label("field") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: node.loc() }) + let args = node + .first_child_with_label("arg") + .map(lower_arguments) + .unwrap_or(@list.empty()) + let return_self = node.first_child_with_label("op").map(CstNode::first_token) + is Some(Some(DOTDOT)) + @syntax.Expr::DotApply( + self=self_expr, + method_name=method_label, + args~, + return_self~, + loc=node.loc(), + ) +} + +///| +fn lower_type_name_from_expr_node(node : CstNode) -> @syntax.TypeName { + match node.kind() { + Type(_) => lower_type_name_from_type_node(node) + Expr(ExprKind::Constructor) | Expr(ExprKind::Identifier) => + @syntax.TypeName::{ + name: lower_long_ident_from_name_nodes(node.children_with_label("name")), + is_object: false, + loc: node.loc(), + } + _ => + @syntax.TypeName::{ + name: @syntax.LongIdent::dummy(), + is_object: false, + loc: node.loc(), + } + } +} + +///| +fn lower_method_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::Method( + type_name=node + .first_child_with_label("type") + .map(lower_type_name_from_expr_node) + .unwrap_or(@syntax.TypeName::{ + name: @syntax.LongIdent::dummy(), + is_object: false, + loc: node.loc(), + }), + method_name=node + .first_child_with_label("field") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: node.loc() }), + loc=node.loc(), + ) +} + +///| +fn lower_field_expr(node : CstNode) -> @syntax.Expr { + let record = node + .first_child_with_label("self") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let accessor = match node.first_child_with_label("field") { + Some(field) => + match field.first_token() { + Some(DOT_INT(tuple_index)) => + @syntax.Accessor::Index(tuple_index~, loc=field.loc()) + _ => @syntax.Accessor::Label(lower_label_from_node(field)) + } + None => + @syntax.Accessor::Label(@syntax.Label::{ name: "", loc: node.loc() }) + } + @syntax.Expr::Field(record~, accessor~, loc=node.loc()) +} + +///| +fn lower_array_get_expr(node : CstNode) -> @syntax.Expr { + let array = node + .first_child_with_label("self") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let index = node + .first_child_with_label("index") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + @syntax.Expr::ArrayGet(array~, index~, loc=node.loc()) +} + +///| +fn lower_array_get_slice_expr(node : CstNode) -> @syntax.Expr { + let array = node + .first_child_with_label("self") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let indexes = node.children_with_label("index") + let start_index = match node.first_child_with_label("start") { + Some(start) => Some(lower_expr(start)) + None => indexes.get(0).map(lower_expr) + } + let end_index = match node.first_child_with_label("end") { + Some(end) => Some(lower_expr(end)) + None => indexes.get(1).map(lower_expr) + } + @syntax.Expr::ArrayGetSlice( + array~, + start_index~, + end_index~, + index_loc=node.loc(), + loc=node.loc(), + ) +} + +///| +fn lower_if_expr(node : CstNode) -> @syntax.Expr { + let cond = node + .first_child_with_label("cond") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let bodies = node.children_with_label("body") + let ifso = bodies + .get(0) + .map(lower_function_body_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let ifnot = node + .children_with_label("else") + .last() + .map(lower_function_body_expr) + @syntax.Expr::If(cond~, ifso~, ifnot~, loc=node.loc()) +} + +///| +fn lower_match_expr(node : CstNode) -> @syntax.Expr { + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let cases : Array[@syntax.Case] = [] + for case_node in node.children_with_label("case") { + cases.push(lower_case(case_node)) + } + @syntax.Expr::Match( + expr~, + cases=lower_array_to_list(cases), + match_loc=node.loc(), + loc=node.loc(), + ) +} + +///| +fn lower_case(node : CstNode) -> @syntax.Case { + @syntax.Case::{ + pattern: node + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())), + guard_: node.first_child_with_label("cond").map(lower_expr), + body: node + .first_child_with_label("body") + .map(lower_expr) + .unwrap_or(lower_unit_expr(node.loc())), + } +} + +///| +fn lower_try_expr(node : CstNode) -> @syntax.Expr { + let cases : Array[@syntax.Case] = [] + for case_list in node.children_with_label("case") { + for case_node in case_list.children_with_label("case") { + cases.push(lower_case(case_node)) + } + } + let else_cases : Array[@syntax.Case] = [] + for case_list in node.children_with_label("else") { + for case_node in case_list.children_with_label("case") { + else_cases.push(lower_case(case_node)) + } + } + let try_else = if else_cases.length() == 0 { + None + } else { + Some(lower_array_to_list(else_cases)) + } + let try_loc = node + .children_with_label("kw") + .get(0) + .map(CstNode::loc) + .unwrap_or(node.loc()) + let has_try = match node.children_with_label("kw").get(0) { + Some(kw) => + match kw.first_token() { + Some(TRY) => true + _ => false + } + None => false + } + let catch_loc = if has_try { + node + .children_with_label("kw") + .get(1) + .map(CstNode::loc) + .unwrap_or(node.loc()) + } else { + node + .children_with_label("kw") + .get(0) + .map(CstNode::loc) + .unwrap_or(node.loc()) + } + @syntax.Expr::Try( + body=node + .first_child_with_label("body") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + catch_=lower_array_to_list(cases), + try_else~, + has_try~, + try_loc~, + catch_loc~, + else_loc=node + .first_child_with_label("else_kw") + .map(CstNode::loc) + .unwrap_or(node.loc()), + loc=node.loc(), + ) +} + +///| +fn lower_try_operator_expr(node : CstNode) -> @syntax.Expr { + let kind = match node.first_child_with_label("kw").map(CstNode::first_token) { + Some(Some(TRY_EXCLAMATION)) => @syntax.TryOperatorKind::Exclamation + _ => @syntax.TryOperatorKind::Question + } + @syntax.Expr::TryOperator( + body=node + .first_child_with_label("body") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + kind~, + try_loc=node + .first_child_with_label("kw") + .map(CstNode::loc) + .unwrap_or(node.loc()), + loc=node.loc(), + ) +} + +///| +fn lower_lex_case_nodes(node : CstNode) -> Array[CstNode] { + let cases : Array[CstNode] = [] + for block in node.children_with_label("case") { + for case_node in block.children_with_label("case") { + cases.push(case_node) + } + } + cases +} + +///| +fn lower_lexmatch_expr(node : CstNode) -> @syntax.Expr { + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let cases : Array[@syntax.LexCase] = [] + for case_node in lower_lex_case_nodes(node) { + cases.push(lower_lex_case(case_node)) + } + @syntax.Expr::LexMatch( + strategy=node.first_child_with_label("name").map(lower_label_from_node), + expr~, + match_loc=node.loc(), + cases=lower_array_to_list(cases), + loc=node.loc(), + ) +} + +///| +fn lower_lexscan_expr(node : CstNode) -> @syntax.Expr { + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let cases : Array[@syntax.LexScanCase] = [] + for case_node in lower_lex_case_nodes(node) { + cases.push(lower_lex_scan_case(case_node)) + } + @syntax.Expr::LexScan( + expr~, + match_loc=node.loc(), + cases=lower_array_to_list(cases), + loc=node.loc(), + ) +} + +///| +fn lower_lex_case(node : CstNode) -> @syntax.LexCase { + let pat_node = node.first_child_with_label("pattern") + let pat_loc = pat_node.map(CstNode::loc).unwrap_or(node.loc()) + let body = match node.first_child_with_label("body") { + Some(body_node) => lower_expr(body_node) + None => + if pat_node.map(lower_node_is_ellipsis).unwrap_or(false) { + lower_todo_hole_expr(node.loc()) + } else { + lower_unit_expr(node.loc()) + } + } + @syntax.LexCase::{ + pat: pat_node.map(lower_lex_top_patterns).unwrap_or(@list.empty()), + pat_loc, + guard_: node.first_child_with_label("cond").map(lower_expr), + body, + } +} + +///| +fn lower_lex_scan_case(node : CstNode) -> @syntax.LexScanCase { + @syntax.LexScanCase::{ + pat: node + .first_child_with_label("pattern") + .map(lower_lex_scan_case_pattern) + .unwrap_or(@syntax.LexScanCasePattern::Wildcard(loc=node.loc())), + guard_: node.first_child_with_label("cond").map(lower_expr), + body: node + .first_child_with_label("body") + .map(lower_expr) + .unwrap_or(lower_unit_expr(node.loc())), + } +} + +///| +fn lower_pattern_tuple_items(node : CstNode) -> Array[CstNode] { + match node.first_child_with_label("body") { + Some(body) => lower_item_children(body) + None => [] + } +} + +///| +fn lower_pattern_tuple_has_separator(node : CstNode) -> Bool { + match node.first_child_with_label("body") { + Some(body) => { + for i in 0.. return true + _ => () + } + } + false + } + None => false + } +} + +///| +fn lower_lex_top_patterns(node : CstNode) -> List[@syntax.LexTopPattern] { + let pats : Array[@syntax.LexTopPattern] = [] + match node.kind() { + Pattern(PatternKind::Tuple) if lower_pattern_tuple_has_separator(node) => + lower_lex_top_pattern_groups(node).each(fn(pat) { pats.push(pat) }) + _ => pats.push(lower_lex_top_pattern(node)) + } + lower_array_to_list(pats) +} + +///| +fn lower_lex_top_pattern_groups(node : CstNode) -> Array[@syntax.LexTopPattern] { + let pats : Array[@syntax.LexTopPattern] = [] + match node.first_child_with_label("body") { + Some(body) => { + let mut group : Array[CstNode] = [] + for i in 0.. group.push(item) + Some((Some("sep"), _)) => { + pats.push(lower_lex_top_pattern_group(group, node.loc())) + group = [] + } + _ => () + } + } + if group.length() > 0 { + pats.push(lower_lex_top_pattern_group(group, node.loc())) + } + } + None => () + } + pats +} + +///| +fn lower_lex_top_pattern_group( + group : Array[CstNode], + loc : Location, +) -> @syntax.LexTopPattern { + let items : Array[CstNode] = [] + for item in group { + if !lower_lex_sequence_item_is_separator(item) { + items.push(item) + } + } + if items.length() == 0 { + @syntax.LexTopPattern::Wildcard(loc~) + } else if items.length() == 1 { + lower_lex_top_pattern(items[0]) + } else { + let pats : Array[@syntax.LexPattern] = [] + for item in items { + pats.push(lower_lex_pattern(item)) + } + @syntax.LexTopPattern::Pattern( + @syntax.LexPattern::Sequence(pats=lower_array_to_list(pats), loc~), + ) + } +} + +///| +fn lower_lex_top_pattern(node : CstNode) -> @syntax.LexTopPattern { + match node.kind() { + _ if lower_node_is_ellipsis(node) => + @syntax.LexTopPattern::Wildcard(loc=node.loc()) + Pattern(PatternKind::Tuple) if !lower_pattern_tuple_has_separator(node) => { + let items = lower_pattern_tuple_items(node) + if items.length() == 1 { + lower_lex_top_pattern(items[0]) + } else { + @syntax.LexTopPattern::Pattern(lower_lex_pattern(node)) + } + } + Pattern(PatternKind::Underscore) | Pattern(PatternKind::MissingPattern) => + @syntax.LexTopPattern::Wildcard(loc=node.loc()) + Pattern(PatternKind::Binder) => + @syntax.LexTopPattern::Binder( + node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())), + ) + _ => @syntax.LexTopPattern::Pattern(lower_lex_pattern(node)) + } +} + +///| +fn lower_node_is_ellipsis(node : CstNode) -> Bool { + match node.first_token() { + Some(ELLIPSIS) => true + _ => false + } +} + +///| +fn lower_todo_hole_expr(loc : Location) -> @syntax.Expr { + @syntax.Expr::Hole(loc~, kind=@syntax.Hole::Todo) +} + +///| +fn lower_lex_pattern(node : CstNode) -> @syntax.LexPattern { + match node.kind() { + Pattern(PatternKind::Literal) | LexPattern(LexPatternKind::Literal) => + lower_lex_literal_pattern(node) + LexPattern(LexPatternKind::RegexInterp) => + @syntax.LexPattern::RegexInterp( + elems=lower_interp_elems(node), + loc=node.loc(), + ) + Pattern(PatternKind::Alias) | LexPattern(LexPatternKind::Alias) => { + let pat = node + .first_child_with_label("pattern") + .map(lower_lex_pattern) + .unwrap_or(lower_lex_empty_regex(node.loc())) + let binder = node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())) + @syntax.LexPattern::Alias(pat~, binder~, loc=node.loc()) + } + Pattern(PatternKind::Tuple) | LexPattern(LexPatternKind::Sequence) => { + let items = lower_pattern_tuple_items(node) + if items.length() == 0 { + lower_lex_empty_regex(node.loc()) + } else if items.length() == 1 { + lower_lex_pattern(items[0]) + } else { + let pats : Array[@syntax.LexPattern] = [] + for item in items { + if !lower_lex_sequence_item_is_separator(item) { + pats.push(lower_lex_pattern(item)) + } + } + @syntax.LexPattern::Sequence( + pats=lower_array_to_list(pats), + loc=node.loc(), + ) + } + } + Pattern(PatternKind::Constructor) + | LexPattern(LexPatternKind::ConstantRef) => + @syntax.LexPattern::ConstantRef( + lid=lower_long_ident_from_name_nodes(node.children_with_label("name")), + loc=node.loc(), + ) + Pattern(PatternKind::Binder) | LexPattern(LexPatternKind::Binder) => + @syntax.LexPattern::ConstantRef( + lid=@syntax.LongIdent::Ident( + name=match node.first_child_with_label("name") { + Some(name_node) => lower_token_child_name(name_node).unwrap_or("") + None => "" + }, + ), + loc=node.loc(), + ) + _ => lower_lex_empty_regex(node.loc()) + } +} + +///| +fn lower_lex_sequence_item_is_separator(node : CstNode) -> Bool { + match node.first_token() { + Some(SEMI(_)) => true + _ => false + } +} + +///| +fn lower_lex_empty_regex(loc : Location) -> @syntax.LexPattern { + @syntax.LexPattern::Regex(lit="", offset=1, loc~) +} + +///| +fn lower_lex_literal_pattern(node : CstNode) -> @syntax.LexPattern { + match node.first_token() { + Some(REGEX_LITERAL(lit)) => + @syntax.LexPattern::Regex(lit~, offset=3, loc=node.loc()) + Some(STRING(lit)) => + @syntax.LexPattern::Regex(lit~, offset=1, loc=node.loc()) + Some(tok) => + @syntax.LexPattern::ConstantRef( + lid=@syntax.LongIdent::Ident(name=lower_token_name(tok).unwrap_or("")), + loc=node.loc(), + ) + None => lower_lex_empty_regex(node.loc()) + } +} + +///| +fn lower_regex_pattern(node : CstNode) -> @syntax.RegexPattern { + match node.kind() { + RegexPattern(RegexPatternKind::Placeholder) => + node + .first_child_with_label("pattern") + .map(lower_regex_pattern) + .unwrap_or(lower_empty_regex_pattern(node.loc())) + RegexPattern(RegexPatternKind::Literal) | Pattern(PatternKind::Literal) => + lower_regex_literal_pattern(node) + RegexPattern(RegexPatternKind::Alias) | Pattern(PatternKind::Alias) => { + let pat = node + .first_child_with_label("pattern") + .map(lower_regex_pattern) + .unwrap_or(lower_empty_regex_pattern(node.loc())) + let binder = node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())) + @syntax.RegexPattern::Alias(pat~, binder~, loc=node.loc()) + } + RegexPattern(RegexPatternKind::Sequence) | Pattern(PatternKind::Tuple) => + match + (node.first_child_with_label("lhs"), node.first_child_with_label("rhs")) { + (Some(lhs), Some(rhs)) => + @syntax.RegexPattern::Sequence( + pat1=lower_regex_pattern(lhs), + pat2=lower_regex_pattern(rhs), + loc=node.loc(), + ) + _ => { + let items = lower_pattern_tuple_items(node) + if items.length() == 0 { + lower_empty_regex_pattern(node.loc()) + } else { + let mut pat = lower_regex_pattern(items[0]) + for i in 1.. { + let lhs = node + .first_child_with_label("lhs") + .map(lower_regex_pattern) + .unwrap_or(lower_empty_regex_pattern(node.loc())) + let rhs = node + .first_child_with_label("rhs") + .map(lower_regex_pattern) + .unwrap_or(lower_empty_regex_pattern(node.loc())) + @syntax.RegexPattern::Alternation(pat1=lhs, pat2=rhs, loc=node.loc()) + } + RegexPattern(RegexPatternKind::Reference) + | Pattern(PatternKind::Constructor) + | Pattern(PatternKind::Binder) => + @syntax.RegexPattern::Reference( + lid=lower_long_ident_from_name_nodes(node.children_with_label("name")), + loc=node.loc(), + ) + _ => lower_empty_regex_pattern(node.loc()) + } +} + +///| +fn lower_regex_match_bindings( + pattern_node : CstNode?, +) -> List[(@syntax.Label, @syntax.Binder?)] { + let bindings : Array[(@syntax.Label, @syntax.Binder?)] = [] + match pattern_node { + Some(node) => + match node.first_child_with_label("bindings") { + Some(list) => + for item in lower_item_children(list) { + bindings.push(lower_regex_match_binding(item)) + } + None => () + } + None => () + } + lower_array_to_list(bindings) +} + +///| +fn lower_regex_match_binding( + node : CstNode, +) -> (@syntax.Label, @syntax.Binder?) { + let label = node + .first_child_with_label("name") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: node.loc() }) + let binder = node.first_child_with_label("binder").map(lower_binder_from_node) + (label, binder) +} + +///| +fn lower_empty_regex_pattern(loc : Location) -> @syntax.RegexPattern { + @syntax.RegexPattern::Literal(lit="", loc~) +} + +///| +fn lower_lex_scan_case_pattern(node : CstNode) -> @syntax.LexScanCasePattern { + match node.kind() { + _ if lower_node_is_ellipsis(node) => + @syntax.LexScanCasePattern::Wildcard(loc=node.loc()) + Pattern(PatternKind::Underscore) | Pattern(PatternKind::MissingPattern) => + @syntax.LexScanCasePattern::Wildcard(loc=node.loc()) + Pattern(PatternKind::Binder) => + @syntax.LexScanCasePattern::Binder( + node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())), + ) + _ => @syntax.LexScanCasePattern::Pattern(lower_regex_pattern(node)) + } +} + +///| +fn lower_regex_match_expr(node : CstNode) -> @syntax.Expr { + let pattern_node = node.first_child_with_label("pattern") + @syntax.Expr::RegexMatch( + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + pat=pattern_node + .map(lower_regex_pattern) + .unwrap_or(lower_empty_regex_pattern(node.loc())), + bindings=lower_regex_match_bindings(pattern_node), + loc=node.loc(), + ) +} + +///| +fn lower_regex_literal_pattern(node : CstNode) -> @syntax.RegexPattern { + match node.first_token() { + Some(STRING(lit)) | Some(REGEX_LITERAL(lit)) => + @syntax.RegexPattern::Literal(lit~, loc=node.loc()) + Some(tok) => + @syntax.RegexPattern::Reference( + lid=@syntax.LongIdent::Ident(name=lower_token_name(tok).unwrap_or("")), + loc=node.loc(), + ) + None => lower_empty_regex_pattern(node.loc()) + } +} + +///| +fn lower_parameter(node : CstNode) -> @syntax.Parameter { + let pat = node.first_child_with_label("pattern") + let ty = node.first_child_with_label("type").map(lower_type) + match pat { + Some(pat_node) => + match pat_node.kind() { + Pattern(PatternKind::Underscore) => + @syntax.Parameter::DiscardPositional(ty~, loc=pat_node.loc()) + Expr(ExprKind::Hole) => + @syntax.Parameter::DiscardPositional(ty~, loc=pat_node.loc()) + _ => { + let binder = match pat_node.first_child_with_label("name") { + Some(name) => lower_binder_from_node(name) + None => + match pat_node.first_token() { + Some(POST_LABEL(_)) => lower_binder_from_node(pat_node) + _ => lower_dummy_binder(pat_node.loc()) + } + } + match pat_node.first_token() { + Some(POST_LABEL(_)) => + match node.first_child_with_label("default") { + Some(default) => + @syntax.Parameter::Optional( + binder~, + default=lower_expr(default), + ty~, + ) + None => @syntax.Parameter::Labelled(binder~, ty~) + } + _ if node.first_child_with_label("default") is Some(_) => + @syntax.Parameter::Optional( + binder~, + default=node + .first_child_with_label("default") + .map(lower_expr) + .unwrap_or(lower_unit_expr(node.loc())), + ty~, + ) + _ if node.first_child_with_label("question") is Some(_) => + @syntax.Parameter::QuestionOptional(binder~, ty~) + _ => @syntax.Parameter::Positional(binder~, ty~) + } + } + } + None => @syntax.Parameter::DiscardPositional(ty~, loc=node.loc()) + } +} + +///| +fn lower_parameters_from_list( + list : CstNode?, +) -> (List[@syntax.Parameter], Location) { + match list { + Some(list) => { + let params : Array[@syntax.Parameter] = [] + for item in lower_item_children(list) { + params.push(lower_parameter(item)) + } + (lower_array_to_list(params), list.loc()) + } + None => (@list.empty(), lower_empty_loc()) + } +} + +///| +fn lower_optional_parameters_from_list( + list : CstNode?, +) -> (List[@syntax.Parameter]?, Location) { + match list { + Some(_) => { + let (params, loc) = lower_parameters_from_list(list) + (Some(params), loc) + } + None => (None, lower_empty_loc()) + } +} + +///| +fn lower_return_type_from_clause(node : CstNode?) -> @syntax.Type? { + match node { + Some(clause) => + match clause.first_child_with_label("type") { + Some(ty_node) => { + let ty = lower_type(ty_node) + match (ty, clause.first_child_with_label("clause")) { + ( + @syntax.Type::Arrow(args~, res~, err=_, is_async~, loc~), + Some(error_clause), + ) if ty_node.kind() is Type(TypeKind::Function) => + Some( + @syntax.Type::Arrow( + args~, + res~, + err=lower_error_annotation(error_clause), + is_async~, + loc~, + ), + ) + _ => Some(ty) + } + } + None => None + } + None => None + } +} + +///| +fn lower_function_expr_node(node : CstNode) -> @syntax.Expr { + let (parameters, params_loc) = lower_parameters_from_list( + node.first_child_with_label("params"), + ) + let body = match node.first_child_with_label("body") { + Some(body_node) => + if arrow_lambda_block_keeps_group(node, body_node) { + lower_expr(body_node) + } else if node.first_child_with_label("pipe") is Some(_) || + function_body_block_can_unwrap(body_node) { + lower_function_body_expr(body_node) + } else { + lower_expr(body_node) + } + None => lower_unit_expr(node.loc()) + } + let return_type = lower_return_type_from_clause( + node.first_child_with_label("type"), + ) + let kind = match node.first_child_with_label("op") { + Some(op) => + match op.first_token() { + Some(FAT_ARROW) => @syntax.FnKind::Arrow + _ => @syntax.FnKind::Lambda + } + None => @syntax.FnKind::Lambda + } + let func = @syntax.Func::{ + parameters, + params_loc, + body, + return_type, + error_type: lower_error_type_from_clause( + node.first_child_with_label("type"), + ), + kind, + is_async: lower_async_loc(node), + loc: node.loc(), + } + @syntax.Expr::Function(func~, loc=node.loc()) +} + +///| +fn arrow_lambda_block_keeps_group(node : CstNode, body_node : CstNode) -> Bool { + guard body_node.kind() is Expr(ExprKind::Block) else { return false } + if node.first_child_with_label("pipe") is Some(_) { + return false + } + match node.first_child_with_label("op").map(CstNode::first_token) { + Some(Some(FAT_ARROW)) => true + _ => false + } +} + +///| +fn function_body_block_can_unwrap(node : CstNode) -> Bool { + match node.kind() { + Expr(ExprKind::Block) => true + _ => false + } +} + +///| +fn lower_let_expr_without_body(node : CstNode) -> @syntax.Expr { + lower_let_expr_with_body(node, lower_unit_expr(node.loc())) +} + +///| +fn lower_let_expr_with_body( + node : CstNode, + body : @syntax.Expr, +) -> @syntax.Expr { + let base_pat = node + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + let pat = match node.kind() { + Expr(ExprKind::Let) => + match node.first_child_with_label("type") { + Some(type_node) => { + let ty = lower_type(type_node) + let loc = base_pat.loc().merge(ty.loc()) + @syntax.Pattern::Constraint(pat=base_pat, ty~, loc~) + } + None => base_pat + } + _ => base_pat + } + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_unit_expr(node.loc())) + match node.kind() { + Expr(ExprKind::LetMut) => { + let binder = match pat { + @syntax.Pattern::Var(binder) => binder + _ => lower_dummy_binder(pat.loc()) + } + @syntax.Expr::LetMut( + binder~, + ty=node.first_child_with_label("type").map(lower_type), + expr~, + body~, + loc=node.loc().merge(body.loc()), + ) + } + _ => + @syntax.Expr::Let( + pattern=pat, + expr~, + body~, + loc=node.loc().merge(body.loc()), + ) + } +} + +///| +fn lower_func_from_decl_like_node(node : CstNode) -> @syntax.Func { + let (parameters, params_loc) = lower_parameters_from_list( + node.first_child_with_label("params"), + ) + let body = node + .first_child_with_label("body") + .map(lower_function_body_expr) + .unwrap_or(lower_unit_expr(node.loc())) + let loc = match node.children_with_label("kw").get(0) { + Some(kw) => Location::{ start: kw.loc().start, end: node.loc().end } + None => node.loc() + } + @syntax.Func::{ + parameters, + params_loc, + body, + return_type: lower_return_type_from_clause( + node.first_child_with_label("type"), + ), + error_type: lower_error_type_from_clause( + node.first_child_with_label("type"), + ), + kind: @syntax.FnKind::Lambda, + is_async: lower_async_loc(node), + loc, + } +} + +///| +fn lower_func_from_letrec_binding(node : CstNode) -> @syntax.Func { + match node.first_child_with_label("expr").map(lower_expr) { + Some(@syntax.Expr::Function(func~, ..)) => func + Some(expr) => + @syntax.Func::{ + parameters: @list.empty(), + params_loc: lower_empty_loc(), + body: expr, + return_type: None, + error_type: @syntax.ErrorType::NoErrorType, + kind: @syntax.FnKind::Lambda, + is_async: None, + loc: node.loc(), + } + None => lower_func_from_decl_like_node(node) + } +} + +///| +fn lower_letfn_expr_with_body( + node : CstNode, + body : @syntax.Expr, +) -> @syntax.Expr { + @syntax.Expr::LetFn( + name=node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())), + func=lower_func_from_decl_like_node(node), + body~, + loc=node.loc().merge(body.loc()), + ) +} + +///| +fn lower_letrec_bindings( + node : CstNode, +) -> List[(@syntax.Binder, @syntax.Type?, @syntax.Func)] { + let bindings : Array[(@syntax.Binder, @syntax.Type?, @syntax.Func)] = [] + for binding in node.children_with_label("binding") { + let binding_type = binding + .first_child_with_label("binding_type") + .map(lower_type) + bindings.push( + ( + binding + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(binding.loc())), + binding_type, + lower_func_from_letrec_binding(binding), + ), + ) + } + lower_array_to_list(bindings) +} + +///| +fn lower_letrec_expr_with_body( + node : CstNode, + body : @syntax.Expr, +) -> @syntax.Expr { + @syntax.Expr::LetAnd( + bindings=lower_letrec_bindings(node), + body~, + loc=node.loc().merge(body.loc()), + ) +} + +///| +fn lower_guard_expr_with_body( + node : CstNode, + body : @syntax.Expr, +) -> @syntax.Expr { + @syntax.Expr::Guard( + cond=node + .first_child_with_label("cond") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + otherwise=node.first_child_with_label("else").map(lower_function_body_expr), + body~, + loc=node.loc().merge(body.loc()), + ) +} + +///| +fn lower_defer_expr_with_body( + node : CstNode, + body : @syntax.Expr, +) -> @syntax.Expr { + @syntax.Expr::Defer( + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + body~, + loc=node.loc().merge(body.loc()), + ) +} + +///| +fn lower_proof_assert_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::ProofAssert( + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + loc=node.loc(), + ) +} + +///| +fn lower_proof_let_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::ProofLet( + binder=node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())), + expr=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + loc=node.loc(), + ) +} + +///| +fn lower_statement_sequence( + items : Array[CstNode], + index : Int, + loc : Location, +) -> @syntax.Expr { + if index >= items.length() { + return lower_faked_unit_expr(loc) + } + let first = items[index] + if index == items.length() - 1 { + return lower_statement_tail_expr(first) + } + match first.kind() { + Expr(ExprKind::Let) | Expr(ExprKind::LetMut) => { + let body = lower_statement_sequence(items, index + 1, loc) + lower_let_expr_with_body(first, body) + } + Expr(ExprKind::LetFn) => { + let body = lower_statement_sequence(items, index + 1, loc) + lower_letfn_expr_with_body(first, body) + } + Expr(ExprKind::LetRec) => { + let body = lower_statement_sequence(items, index + 1, loc) + lower_letrec_expr_with_body(first, body) + } + Expr(ExprKind::Guard) => { + let body = lower_statement_sequence(items, index + 1, loc) + lower_guard_expr_with_body(first, body) + } + Expr(ExprKind::Defer) => { + let body = lower_statement_sequence(items, index + 1, loc) + lower_defer_expr_with_body(first, body) + } + _ => { + let prefix : Array[@syntax.Expr] = [] + let mut i = index + while i < items.length() - 1 { + match items[i].kind() { + Expr(ExprKind::Let) | Expr(ExprKind::LetMut) => { + let body = lower_statement_sequence(items, i, loc) + return if prefix.length() == 0 { + body + } else { + lower_sequence_expr(prefix, body) + } + } + Expr(ExprKind::LetFn) => { + let body = lower_statement_sequence(items, i + 1, loc) + let letfn = lower_letfn_expr_with_body(items[i], body) + return if prefix.length() == 0 { + letfn + } else { + lower_sequence_expr(prefix, letfn) + } + } + Expr(ExprKind::LetRec) => { + let body = lower_statement_sequence(items, i + 1, loc) + let letrec_expr = lower_letrec_expr_with_body(items[i], body) + return if prefix.length() == 0 { + letrec_expr + } else { + lower_sequence_expr(prefix, letrec_expr) + } + } + Expr(ExprKind::Guard) => { + let body = lower_statement_sequence(items, i + 1, loc) + let guard_expr = lower_guard_expr_with_body(items[i], body) + return if prefix.length() == 0 { + guard_expr + } else { + lower_sequence_expr(prefix, guard_expr) + } + } + Expr(ExprKind::Defer) => { + let body = lower_statement_sequence(items, i + 1, loc) + let defer_expr = lower_defer_expr_with_body(items[i], body) + return if prefix.length() == 0 { + defer_expr + } else { + lower_sequence_expr(prefix, defer_expr) + } + } + _ => { + prefix.push(lower_expr(items[i])) + i += 1 + } + } + } + lower_sequence_expr( + prefix, + lower_statement_tail_expr(items[items.length() - 1]), + ) + } + } +} + +///| +fn lower_sequence_expr( + exprs : Array[@syntax.Expr], + last_expr : @syntax.Expr, +) -> @syntax.Expr { + let loc = match exprs.get(0) { + Some(first) => first.loc().merge(last_expr.loc()) + None => last_expr.loc() + } + @syntax.Expr::Sequence(exprs=lower_array_to_list(exprs), last_expr~, loc~) +} + +///| +fn lower_statement_tail_expr(item : CstNode) -> @syntax.Expr { + match item.kind() { + Expr(ExprKind::Let) | Expr(ExprKind::LetMut) => + lower_let_expr_with_body(item, lower_faked_unit_expr(item.loc())) + Expr(ExprKind::Guard) => + lower_guard_expr_with_body(item, lower_faked_unit_expr(item.loc())) + Expr(ExprKind::Defer) => + lower_defer_expr_with_body(item, lower_faked_unit_expr(item.loc())) + _ => lower_expr(item) + } +} + +///| +fn lower_block_expr_node(node : CstNode) -> @syntax.Expr { + match lower_block_as_record_pun(node) { + Some(record) => return record + None => () + } + let expr = match node.first_child_with_label("body") { + Some(body) => + lower_statement_sequence(lower_item_children(body), 0, node.loc()) + None => lower_unit_expr(node.loc()) + } + @syntax.Expr::Group(expr~, group=@syntax.Group::Brace, loc=node.loc()) +} + +///| +fn lower_block_as_record_pun(node : CstNode) -> @syntax.Expr? { + match node.first_child_with_label("body") { + Some(body) => { + let items = lower_item_children(body) + match items.length() { + 1 => + match items[0].kind() { + Expr(ExprKind::Identifier) => + match items[0].children_with_label("name").get(0) { + Some(name_node) => { + let label = lower_label_from_node(name_node) + Some( + @syntax.Expr::Record( + type_name=None, + fields=@list.singleton(@syntax.FieldDef::{ + label, + expr: lower_expr(items[0]), + is_pun: true, + loc: items[0].loc(), + }), + trailing=lower_record_trailing_mark(body), + loc=node.loc(), + ), + ) + } + None => None + } + _ => None + } + _ => None + } + } + None => None + } +} + +///| +fn lower_function_body_expr(node : CstNode) -> @syntax.Expr { + match node.kind() { + Expr(ExprKind::Block) => + match node.first_child_with_label("body") { + Some(body) => + lower_statement_sequence(lower_item_children(body), 0, node.loc()) + None => lower_unit_expr(node.loc()) + } + _ => lower_expr(node) + } +} + +///| +fn lower_return_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::Return( + return_value=node.first_child_with_label("expr").map(lower_expr), + loc=node.loc(), + ) +} + +///| +fn lower_raise_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::Raise( + err_value=node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())), + loc=node.loc(), + ) +} + +///| +fn lower_break_expr(node : CstNode) -> @syntax.Expr { + @syntax.Expr::Break( + arg=node.first_child_with_label("expr").map(lower_expr), + label=node.first_child_with_label("label").map(lower_label_from_node), + loc=node.loc(), + ) +} + +///| +fn lower_continue_expr(node : CstNode) -> @syntax.Expr { + let args : Array[@syntax.Expr] = [] + match node.first_child_with_label("arg") { + Some(list) => + for item in lower_item_children(list) { + args.push(lower_expr(item)) + } + None => + match node.first_child_with_label("expr") { + Some(expr) => args.push(lower_expr(expr)) + None => () + } + } + let label = node.first_child_with_label("label").map(lower_label_from_node) + @syntax.Expr::Continue(args=lower_array_to_list(args), label~, loc=node.loc()) +} + +///| +fn lower_as_expr(node : CstNode) -> @syntax.Expr { + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let trait_ = node + .first_child_with_label("type") + .map(lower_type_name_from_type_node) + .unwrap_or(@syntax.TypeName::{ + name: @syntax.LongIdent::dummy(), + is_object: false, + loc: node.loc(), + }) + @syntax.Expr::As(expr~, trait_~, loc=node.loc()) +} + +///| +fn lower_is_expr(node : CstNode) -> @syntax.Expr { + let expr = node + .first_child_with_label("expr") + .map(lower_expr) + .unwrap_or(lower_hole_expr(node.loc())) + let pat = node + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + @syntax.Expr::Is(expr~, pat~, loc=node.loc()) +} diff --git a/untyped_cst/lower_pattern.mbt b/untyped_cst/lower_pattern.mbt new file mode 100644 index 00000000..e2ace308 --- /dev/null +++ b/untyped_cst/lower_pattern.mbt @@ -0,0 +1,509 @@ +///| +fn lower_pattern(node : CstNode) -> @syntax.Pattern { + match node.kind() { + Pattern(PatternKind::Binder) => + match node.first_child_with_label("name") { + Some(name) => @syntax.Pattern::Var(lower_binder_from_node(name)) + None => @syntax.Pattern::Any(loc=node.loc()) + } + Pattern(PatternKind::Underscore) | Pattern(PatternKind::MissingPattern) => + @syntax.Pattern::Any(loc=node.loc()) + Pattern(PatternKind::Literal) => + match node.first_token() { + Some(MINUS) => + match node.first_child_with_label("token").map(CstNode::first_token) { + Some(Some(INT(value))) => + @syntax.Pattern::Constant( + c=@syntax.make_int("-\{value}"), + loc=node.loc(), + ) + Some(Some(DOUBLE(value))) => + @syntax.Pattern::Constant( + c=@syntax.Constant::Double("-\{value}"), + loc=node.loc(), + ) + Some(Some(FLOAT(value))) => + @syntax.Pattern::Constant( + c=@syntax.make_float("-\{value}"), + loc=node.loc(), + ) + _ => @syntax.Pattern::Any(loc=node.loc()) + } + Some(tok) => + match lower_constant_from_token(tok) { + Some(c) => @syntax.Pattern::Constant(c~, loc=node.loc()) + None => @syntax.Pattern::Any(loc=node.loc()) + } + None => @syntax.Pattern::Any(loc=node.loc()) + } + Pattern(PatternKind::Tuple) => lower_tuple_pattern(node) + Pattern(PatternKind::Array) => + @syntax.Pattern::Array(pats=lower_array_patterns(node), loc=node.loc()) + Pattern(PatternKind::Record) => + @syntax.Pattern::Record( + fields=lower_record_pattern_fields(node), + is_closed=lower_record_pattern_is_closed(node), + loc=node.loc(), + ) + Pattern(PatternKind::Map) => + @syntax.Pattern::Map( + elems=lower_map_pattern_elems(node), + is_closed=lower_map_pattern_is_closed(node), + loc=node.loc(), + ) + Pattern(PatternKind::Constructor) => lower_constructor_pattern(node) + Pattern(PatternKind::SpecialConstructor) => + lower_special_constructor_pattern(node) + Pattern(PatternKind::Or) => { + let lhs = node + .first_child_with_label("lhs") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + let rhs = node + .first_child_with_label("rhs") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + @syntax.Pattern::Or(pat1=lhs, pat2=rhs, loc=node.loc()) + } + Pattern(PatternKind::Range) => { + let lhs = node + .first_child_with_label("lhs") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + let rhs = node + .first_child_with_label("rhs") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + @syntax.Pattern::Range( + lhs~, + rhs~, + kind=lower_pattern_range_kind(node), + loc=node.loc(), + ) + } + Pattern(PatternKind::Alias) => { + let pat = node + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + let binder = node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())) + @syntax.Pattern::Alias(pat~, alias_=binder, loc=node.loc()) + } + Pattern(PatternKind::Constraint) => { + let pat = node + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())) + let ty = node + .first_child_with_label("type") + .map(lower_type) + .unwrap_or(@syntax.Type::Any(loc=node.loc())) + @syntax.Pattern::Constraint(pat~, ty~, loc=node.loc()) + } + _ => @syntax.Pattern::Any(loc=node.loc()) + } +} + +///| +fn lower_pattern_range_kind(node : CstNode) -> @syntax.RangeKind { + let op_token = match node.first_child_with_label("op") { + Some(op) => op.first_token() + None => None + } + match op_token { + Some(RANGE_EXCLUSIVE) => @syntax.RangeKind::Exclusive + Some(RANGE_INCLUSIVE) => @syntax.RangeKind::Inclusive + _ => @syntax.RangeKind::InclusiveMissingEqual + } +} + +///| +fn lower_array_patterns(node : CstNode) -> @syntax.ArrayPatterns { + let prefix : Array[@syntax.ArrayPattern] = [] + let suffix : Array[@syntax.ArrayPattern] = [] + let mut rest : @syntax.DotDotBinder? = None + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + match item.kind() { + Pattern(PatternKind::Rest) if rest is None && + !array_rest_pattern_is_spread(item) => + rest = Some(lower_dot_dot_binder(item)) + _ => + if rest is Some(_) { + suffix.push(lower_array_pattern_item(item)) + } else { + prefix.push(lower_array_pattern_item(item)) + } + } + } + None => () + } + match rest { + Some(dot_dot_binder) => + @syntax.ArrayPatterns::Open( + lower_array_to_list(prefix), + lower_array_to_list(suffix), + dot_dot_binder, + ) + None => @syntax.ArrayPatterns::Closed(lower_array_to_list(prefix)) + } +} + +///| +fn array_rest_pattern_is_spread(node : CstNode) -> Bool { + node.first_child_with_label("str") is Some(_) || + node.first_child_with_label("bytes") is Some(_) || + node.first_child_with_label("const") is Some(_) +} + +///| +fn lower_array_pattern_item(node : CstNode) -> @syntax.ArrayPattern { + match node.kind() { + Pattern(PatternKind::Rest) => + match node.first_child_with_label("str") { + Some(str_node) => + @syntax.ArrayPattern::StringSpread( + str=lower_token_child_name(str_node).unwrap_or(""), + loc=node.loc(), + ) + None => + match node.first_child_with_label("bytes") { + Some(bytes_node) => + @syntax.ArrayPattern::BytesSpread( + bytes=lower_token_child_name(bytes_node).unwrap_or(""), + loc=node.loc(), + ) + None => + match node.first_child_with_label("const") { + Some(name_node) => { + let pkg = match node.first_child_with_label("pkg") { + Some(pkg_node) => lower_token_child_name(pkg_node) + None => None + } + @syntax.ArrayPattern::ConstSpread( + binder=lower_binder_from_node(name_node), + pkg~, + loc=node.loc(), + ) + } + None => + @syntax.ArrayPattern::Pattern( + @syntax.Pattern::Any(loc=node.loc()), + ) + } + } + } + _ => @syntax.ArrayPattern::Pattern(lower_pattern(node)) + } +} + +///| +fn lower_dot_dot_binder(node : CstNode) -> @syntax.DotDotBinder { + match node.first_child_with_label("name") { + Some(name_node) => + match name_node.first_token() { + Some(UNDERSCORE) => @syntax.DotDotBinder::Underscore + Some(LIDENT(_)) | Some(UIDENT(_)) => + if node.first_child_with_label("as") is Some(_) { + @syntax.DotDotBinder::BinderAs(lower_binder_from_node(name_node)) + } else { + @syntax.DotDotBinder::Binder(lower_binder_from_node(name_node)) + } + _ => @syntax.DotDotBinder::NoBinder + } + None => @syntax.DotDotBinder::NoBinder + } +} + +///| +fn lower_record_pattern_fields(node : CstNode) -> List[@syntax.FieldPat] { + let fields : Array[@syntax.FieldPat] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + if !(item.kind() is Pattern(PatternKind::Rest)) { + fields.push(lower_record_pattern_field(item)) + } + } + None => () + } + lower_array_to_list(fields) +} + +///| +fn lower_record_pattern_is_closed(node : CstNode) -> Bool { + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + if item.kind() is Pattern(PatternKind::Rest) { + return false + } + } + None => () + } + true +} + +///| +fn lower_record_pattern_field(node : CstNode) -> @syntax.FieldPat { + let label_node = node.first_child_with_label("name") + let label = label_node + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: node.loc() }) + let pattern = node + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or( + @syntax.Pattern::Var(@syntax.Binder::{ name: label.name, loc: label.loc }), + ) + @syntax.FieldPat::{ + label, + pattern, + is_pun: node.first_child_with_label("pattern") is None, + loc: node.loc(), + } +} + +///| +fn lower_map_pattern_is_closed(node : CstNode) -> Bool { + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + if item.kind() is Pattern(PatternKind::Rest) { + return false + } + } + None => () + } + true +} + +///| +fn lower_map_pattern_elems(node : CstNode) -> List[@syntax.MapPatElem] { + let elems : Array[@syntax.MapPatElem] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + if !(item.kind() is Pattern(PatternKind::Rest)) { + elems.push(lower_map_pattern_elem(item)) + } + } + None => () + } + lower_array_to_list(elems) +} + +///| +fn lower_map_pattern_elem(node : CstNode) -> @syntax.MapPatElem { + let (key, key_loc) = lower_map_pattern_key(node) + @syntax.MapPatElem::{ + key, + pat: node + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=node.loc())), + match_absent: node.first_child_with_label("question") is Some(_), + key_loc, + loc: node.loc(), + } +} + +///| +fn lower_map_pattern_key(node : CstNode) -> (@syntax.Constant, Location) { + match node.first_child_with_label("key") { + Some(key_node) => { + let key = match + (node.first_child_with_label("op"), key_node.first_token()) { + (Some(op), Some(INT(value))) if op.first_token() is Some(MINUS) => + @syntax.make_int("-\{value}") + (Some(op), Some(DOUBLE(value))) if op.first_token() is Some(MINUS) => + @syntax.Constant::Double("-\{value}") + (Some(op), Some(FLOAT(value))) if op.first_token() is Some(MINUS) => + @syntax.make_float("-\{value}") + (_, Some(tok)) => + lower_constant_from_token(tok).unwrap_or(@syntax.Constant::String("")) + _ => @syntax.Constant::String("") + } + (key, key_node.loc()) + } + None => (@syntax.Constant::String(""), node.loc()) + } +} + +///| +fn lower_pattern_items(node : CstNode) -> Array[@syntax.Pattern] { + let result : Array[@syntax.Pattern] = [] + match node.first_child_with_label("body") { + Some(body) => + for item in lower_item_children(body) { + result.push(lower_pattern(item)) + } + None => () + } + result +} + +///| +fn lower_tuple_pattern(node : CstNode) -> @syntax.Pattern { + let items = lower_pattern_items(node) + let has_separator = match node.first_child_with_label("body") { + Some(body) => body.children_with_label("sep").length() > 0 + None => false + } + if items.length() == 1 && !has_separator { + items[0] + } else { + @syntax.Pattern::Tuple(pats=lower_array_to_list(items), loc=node.loc()) + } +} + +///| +fn lower_constructor_from_names( + names : Array[CstNode], + loc : Location, +) -> @syntax.Constructor { + let name_node = if names.length() > 0 { + names[names.length() - 1] + } else { + node(Token(UIDENT("")), loc, mk_span(0, 0), []) + } + let extra_info = if names.length() == 2 && + names[0].first_token() is Some(PACKAGE_NAME(_)) { + @syntax.ConstructorExtraInfo::Package( + lower_token_child_name(names[0]).unwrap_or(""), + ) + } else if names.length() >= 2 { + let type_nodes : Array[CstNode] = [] + for i in 0..<(names.length() - 1) { + type_nodes.push(names[i]) + } + let type_node = type_nodes[0] + @syntax.ConstructorExtraInfo::TypeName(@syntax.TypeName::{ + name: lower_long_ident_from_name_nodes(type_nodes), + is_object: false, + loc: type_node.loc(), + }) + } else { + @syntax.ConstructorExtraInfo::NoExtraInfo + } + let loc = if names.length() > 0 { + names[0].loc().merge(names[names.length() - 1].loc()) + } else { + loc + } + @syntax.Constructor::{ + name: @syntax.ConstrName::{ + name: lower_token_child_name(name_node).unwrap_or(""), + loc: name_node.loc(), + }, + extra_info, + loc, + } +} + +///| +fn lower_constructor_pattern_args( + node : CstNode, +) -> List[@syntax.ConstrPatArg]? { + match node.first_child_with_label("params") { + Some(list) => { + let args : Array[@syntax.ConstrPatArg] = [] + for item in lower_item_children(list) { + if !(item.kind() is Pattern(PatternKind::Rest)) { + args.push(lower_constructor_pattern_arg(item)) + } + } + Some(lower_array_to_list(args)) + } + None => None + } +} + +///| +fn constructor_pattern_is_open(node : CstNode) -> Bool { + match node.first_child_with_label("params") { + Some(list) => + for item in lower_item_children(list) { + if item.kind() is Pattern(PatternKind::Rest) { + return true + } + } + None => () + } + false +} + +///| +fn lower_constructor_pattern_arg(item : CstNode) -> @syntax.ConstrPatArg { + match item.kind() { + Argument(ArgumentKind::LabelledPun) => { + let label_node = item.first_child_with_label("name").unwrap_or(item) + let binder = lower_binder_from_node(label_node) + @syntax.ConstrPatArg::{ + pat: @syntax.Pattern::Var(binder), + kind: @syntax.ArgumentKind::LabelledPun( + lower_label_from_node(label_node), + ), + } + } + Argument(ArgumentKind::Labelled) => + @syntax.ConstrPatArg::{ + pat: item + .first_child_with_label("pattern") + .map(lower_pattern) + .unwrap_or(@syntax.Pattern::Any(loc=item.loc())), + kind: @syntax.ArgumentKind::Labelled( + item + .first_child_with_label("name") + .map(lower_label_from_node) + .unwrap_or(@syntax.Label::{ name: "", loc: item.loc() }), + ), + } + _ => + @syntax.ConstrPatArg::{ + pat: lower_pattern(item), + kind: @syntax.ArgumentKind::Positional, + } + } +} + +///| +fn lower_constructor_pattern(node : CstNode) -> @syntax.Pattern { + @syntax.Pattern::Constr( + constr=lower_constructor_from_names( + node.children_with_label("name"), + node.loc(), + ), + args=lower_constructor_pattern_args(node), + is_open=constructor_pattern_is_open(node), + loc=node.loc(), + ) +} + +///| +fn lower_special_constructor_pattern(node : CstNode) -> @syntax.Pattern { + let binder = node + .first_child_with_label("name") + .map(lower_binder_from_node) + .unwrap_or(lower_dummy_binder(node.loc())) + let args : Array[@syntax.ConstrPatArg] = [] + match node.first_child_with_label("params") { + Some(list) => + for item in lower_item_children(list) { + args.push(@syntax.ConstrPatArg::{ + pat: lower_pattern(item), + kind: @syntax.ArgumentKind::Positional, + }) + } + None => () + } + @syntax.Pattern::SpecialConstr( + binder~, + args=lower_array_to_list(args), + loc=node.loc(), + ) +} diff --git a/untyped_cst/lower_type.mbt b/untyped_cst/lower_type.mbt new file mode 100644 index 00000000..bb3cf83b --- /dev/null +++ b/untyped_cst/lower_type.mbt @@ -0,0 +1,196 @@ +///| +fn lower_type(node : CstNode) -> @syntax.Type { + match node.kind() { + Type(TypeKind::Name) => lower_name_type(node, is_object=false) + Type(TypeKind::Object) => lower_object_type(node) + Type(TypeKind::Application) => lower_application_type(node) + Type(TypeKind::Option) => lower_option_type(node) + Type(TypeKind::Tuple) => lower_tuple_type(node) + Type(TypeKind::Group) => + match node.first_child_with_label("type") { + Some(ty) => lower_type(ty) + None => + match node.first_child_with_label("body") { + Some(body) => + match lower_type_items_from_list(body).get(0) { + Some(ty) => ty + None => @syntax.Type::Tuple(tys=@list.empty(), loc=node.loc()) + } + None => @syntax.Type::Any(loc=node.loc()) + } + } + Type(TypeKind::Function) => lower_function_type(node) + Type(TypeKind::Hole) | Type(TypeKind::MissingType) => + @syntax.Type::Any(loc=node.loc()) + _ => @syntax.Type::Any(loc=node.loc()) + } +} + +///| +fn lower_name_type(node : CstNode, is_object~ : Bool) -> @syntax.Type { + let names = node.children_with_label("name") + let constr_id = lower_constr_id_from_name_nodes(names, node.loc()) + if is_object { + @syntax.Type::Object(constr_id) + } else { + @syntax.Type::Name(constr_id~, tys=@list.empty(), loc=node.loc()) + } +} + +///| +fn lower_object_type(node : CstNode) -> @syntax.Type { + match node.first_child_with_label("type") { + Some(ty) => + match ty.kind() { + Type(TypeKind::Name) => { + let names = ty.children_with_label("name") + @syntax.Type::Object(lower_constr_id_from_name_nodes(names, ty.loc())) + } + _ => @syntax.Type::Any(loc=node.loc()) + } + None => @syntax.Type::Any(loc=node.loc()) + } +} + +///| +fn lower_application_type(node : CstNode) -> @syntax.Type { + let head = match node.first_child_with_label("func") { + Some(func) => Some(func) + None => node.first_child_with_label("head") + } + let args = match node.first_child_with_label("arg") { + Some(list) => lower_type_items_from_list(list) + None => [] + } + match head { + Some(head_node) => + match lower_type(head_node) { + @syntax.Type::Name(constr_id~, loc=_, tys=_) => + @syntax.Type::Name( + constr_id~, + tys=lower_array_to_list(args), + loc=node.loc(), + ) + other => other + } + None => @syntax.Type::Any(loc=node.loc()) + } +} + +///| +fn lower_option_type(node : CstNode) -> @syntax.Type { + let inner = match node.first_child_with_label("type") { + Some(ty) => lower_type(ty) + None => @syntax.Type::Any(loc=node.loc()) + } + let question_loc = match node.first_child_with_label("op") { + Some(op) => op.loc() + None => node.loc() + } + @syntax.Type::Option(ty=inner, loc=node.loc(), question_loc~) +} + +///| +fn lower_type_items_from_list(list : CstNode) -> Array[@syntax.Type] { + let result : Array[@syntax.Type] = [] + for item in lower_item_children(list) { + result.push(lower_type(item)) + } + result +} + +///| +fn lower_tuple_type(node : CstNode) -> @syntax.Type { + let items = match node.first_child_with_label("body") { + Some(body) => lower_type_items_from_list(body) + None => [] + } + @syntax.Type::Tuple(tys=lower_array_to_list(items), loc=node.loc()) +} + +///| +fn lower_type_arrow_args(param : CstNode) -> List[@syntax.Type] { + match param.kind() { + Type(TypeKind::Tuple) => + match param.first_child_with_label("body") { + Some(body) => lower_array_to_list(lower_type_items_from_list(body)) + None => @list.empty() + } + _ => @list.singleton(lower_type(param)) + } +} + +///| +fn lower_function_type(node : CstNode) -> @syntax.Type { + match node.first_child_with_label("type") { + Some(ty) => + match lower_type(ty) { + @syntax.Type::Arrow(args~, res~, err~, loc=_, is_async=_) => + @syntax.Type::Arrow( + args~, + res~, + err~, + is_async=Some( + node + .first_child_with_label("kw") + .map(CstNode::loc) + .unwrap_or(node.loc()), + ), + loc=node.loc(), + ) + other => other + } + None => { + let args = match node.first_child_with_label("param") { + Some(param) => lower_type_arrow_args(param) + None => @list.empty() + } + let res = match node.first_child_with_label("result") { + Some(result) => lower_type(result) + None => @syntax.Type::Any(loc=node.loc()) + } + @syntax.Type::Arrow( + args~, + res~, + err=lower_error_type_from_clause(node.first_child_with_label("clause")), + is_async=None, + loc=node.loc(), + ) + } + } +} + +///| +fn lower_type_name_from_type_node(node : CstNode) -> @syntax.TypeName { + match node.kind() { + Type(TypeKind::Object) => + match node.first_child_with_label("type") { + Some(ty) => { + let base = lower_type_name_from_type_node(ty) + @syntax.TypeName::{ + name: base.name, + is_object: true, + loc: node.loc(), + } + } + None => + @syntax.TypeName::{ + name: @syntax.LongIdent::dummy(), + is_object: true, + loc: node.loc(), + } + } + Type(TypeKind::Name) => + @syntax.TypeName::{ + name: lower_long_ident_from_name_nodes(node.children_with_label("name")), + is_object: node.first_child_with_label("op") is Some(_), + loc: node.loc(), + } + _ => + @syntax.TypeName::{ + name: @syntax.LongIdent::dummy(), + is_object: false, + loc: node.loc(), + } + } +} diff --git a/untyped_cst/moon.pkg b/untyped_cst/moon.pkg new file mode 100644 index 00000000..b3574f6c --- /dev/null +++ b/untyped_cst/moon.pkg @@ -0,0 +1,23 @@ +import { + "moonbitlang/lexer", + "moonbitlang/lexer/tokens", + "moonbitlang/lexer/basic", + "moonbitlang/parser/syntax", + "moonbitlang/parser/attribute", + "moonbitlang/core/list", + "moonbitlang/core/debug", + "moonbitlang/core/json", +} + +import { + "moonbitlang/parser", + "moonbitlang/parser/handrolled_parser", + "moonbitlang/core/test", + "moonbitlang/x/fs", +} for "test" + +import { + "moonbitlang/core/test", +} for "wbtest" + +warnings = "-1-3-7-29" diff --git a/untyped_cst/node.mbt b/untyped_cst/node.mbt new file mode 100644 index 00000000..6b6a73e8 --- /dev/null +++ b/untyped_cst/node.mbt @@ -0,0 +1,158 @@ +///| +pub(all) struct SourceSpan { + start_offset : Int + end_offset : Int +} derive(Eq, @debug.Debug) + +///| +pub struct CstNode { + kind : NodeKind + loc : Location + source_span : SourceSpan + children : ReadOnlyArray[(String?, CstNode)] +} derive(@debug.Debug) + +///| +pub struct ParseResult { + root : CstNode + diagnostics : ReadOnlyArray[Report] + docstrings : ReadOnlyArray[List[(Location, Comment)]] +} derive(@debug.Debug) + +///| +pub fn CstNode::kind(self : CstNode) -> NodeKind { + self.kind +} + +///| +pub fn CstNode::loc(self : CstNode) -> Location { + self.loc +} + +///| +pub fn CstNode::source_span(self : CstNode) -> SourceSpan { + self.source_span +} + +///| +pub fn CstNode::children_view(self : CstNode) -> ArrayView[(String?, CstNode)] { + self.children.view() +} + +///| +pub fn CstNode::child_at(self : CstNode, index : Int) -> (String?, CstNode)? { + self.children.get(index) +} + +///| +pub fn CstNode::children_length(self : CstNode) -> Int { + self.children.length() +} + +///| +pub fn ParseResult::root(self : ParseResult) -> CstNode { + self.root +} + +///| +pub fn ParseResult::diagnostics_view(self : ParseResult) -> ArrayView[Report] { + self.diagnostics.view() +} + +///| +pub fn ParseResult::docstrings_view( + self : ParseResult, +) -> ArrayView[List[(Location, Comment)]] { + self.docstrings.view() +} + +///| +fn mk_loc(start : Position, end : Position) -> Location { + Location::{ start, end } +} + +///| +fn empty_loc(pos : Position) -> Location { + Location::{ start: pos, end: pos } +} + +///| +fn mk_span(start_offset : Int, end_offset : Int) -> SourceSpan { + SourceSpan::{ start_offset, end_offset } +} + +///| +fn empty_span(offset : Int) -> SourceSpan { + SourceSpan::{ start_offset: offset, end_offset: offset } +} + +///| +fn node( + kind : NodeKind, + loc : Location, + source_span : SourceSpan, + children : Array[(String?, CstNode)], +) -> CstNode { + CstNode::{ + kind, + loc, + source_span, + children: ReadOnlyArray::from_array(children.copy()), + } +} + +///| +fn token_node( + token : Token, + start : Position, + end : Position, + source_span : SourceSpan, +) -> CstNode { + node(NodeKind::Token(token), mk_loc(start, end), source_span, []) +} + +///| +fn missing_node(kind : TokenKind, pos : Position, offset : Int) -> CstNode { + node(NodeKind::Missing(kind), empty_loc(pos), empty_span(offset), []) +} + +///| +fn error_node( + loc : Location, + source_span : SourceSpan, + children : Array[(String?, CstNode)], +) -> CstNode { + node(NodeKind::Error, loc, source_span, children) +} + +///| +fn push_child( + children : Array[(String?, CstNode)], + label : String?, + child : CstNode, +) -> Unit { + children.push((label, child)) +} + +///| +fn push_children( + children : Array[(String?, CstNode)], + more : Array[(String?, CstNode)], +) -> Unit { + for child in more { + children.push(child) + } +} + +///| +fn parse_result( + root : CstNode, + diagnostics : Array[Report], + docstrings : Array[List[(Location, Comment)]], +) -> ParseResult { + ParseResult::{ + root, + diagnostics: ReadOnlyArray::from_array(diagnostics.copy()), + docstrings: ReadOnlyArray::from_array(docstrings.copy()), + } +} diff --git a/untyped_cst/node_wbtest.mbt b/untyped_cst/node_wbtest.mbt new file mode 100644 index 00000000..6138dcbe --- /dev/null +++ b/untyped_cst/node_wbtest.mbt @@ -0,0 +1,87 @@ +///| +test "node getters expose immutable shape" { + let start = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 0 } + let end = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 3 } + let child = token_node(Token::LIDENT("x"), start, end, mk_span(0, 1)) + let root = node(NodeKind::SourceFile, mk_loc(start, end), mk_span(0, 3), [ + (Some("item"), child), + ]) + match root.kind() { + SourceFile => () + _ => fail("expected SourceFile") + } + @test.assert_eq(root.loc(), mk_loc(start, end)) + @test.assert_eq(root.source_span(), mk_span(0, 3)) + @test.assert_eq(root.children_length(), 1) + match root.child_at(0) { + Some((Some(label), node)) => { + @test.assert_eq(label, "item") + match node.kind() { + Token(LIDENT(name)) => @test.assert_eq(name, "x") + _ => fail("expected LIDENT token") + } + } + _ => fail("expected labeled child") + } +} + +///| +test "node test json is stable" { + let pos = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 0 } + let root = node(NodeKind::SourceFile, empty_loc(pos), empty_span(0), [ + (Some("missing"), missing_node(TokenKind::TK_EOF, pos, 0)), + ]) + json_inspect(root.to_test_json(), content={ + "kind": "SourceFile", + "loc": { "start": 0, "end": 0 }, + "source_span": { "start": 0, "end": 0 }, + "children": [ + { + "label": "missing", + "node": { + "kind": "Missing", + "expected": "EOF", + "loc": { "start": 0, "end": 0 }, + "source_span": { "start": 0, "end": 0 }, + "children": [], + }, + }, + ], + }) +} + +///| +test "node text returns source span" { + let source = "let x = 1" + let start = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 0 } + let end = Position::{ + fname: "test.mbt", + lnum: 1, + bol: 0, + cnum: source.length(), + } + let root = node( + NodeKind::SourceFile, + mk_loc(start, end), + mk_span(0, source.length()), + [], + ) + @test.assert_eq(root.text(source), source) +} + +///| +test "token text returns token span" { + let source = "let x = 1" + let start = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 4 } + let end = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 5 } + let token = token_node(Token::LIDENT("x"), start, end, mk_span(4, 5)) + @test.assert_eq(token.text(source), "x") +} + +///| +test "missing text is empty" { + let source = "let x = 1" + let pos = Position::{ fname: "test.mbt", lnum: 1, bol: 0, cnum: 4 } + let missing = missing_node(TokenKind::TK_LIDENT, pos, 4) + @test.assert_eq(missing.text(source), "") +} diff --git a/untyped_cst/parse_decl.mbt b/untyped_cst/parse_decl.mbt new file mode 100644 index 00000000..f055770f --- /dev/null +++ b/untyped_cst/parse_decl.mbt @@ -0,0 +1,2009 @@ +///| +fn delimiter_depth_delta(tok : Token) -> Int { + match tok { + LPAREN | LBRACKET | LBRACKET_BAR | LBRACE => 1 + RPAREN | RBRACKET | BAR_RBRACKET | RBRACE => -1 + _ => 0 + } +} + +///| +fn next_depth(depth : Int, tok : Token) -> Int { + let depth = depth + delimiter_depth_delta(tok) + if depth < 0 { + 0 + } else { + depth + } +} + +///| +fn next_paren_depth(depth : Int, tok : Token) -> Int { + match tok { + LPAREN => depth + 1 + RPAREN => if depth > 0 { depth - 1 } else { 0 } + _ => depth + } +} + +///| +fn is_top_level_boundary(tok : Token, depth : Int) -> Bool { + if depth == 0 { + match tok { + SEMI(_) | EOF => true + _ => false + } + } else { + false + } +} + +///| +fn top_level_decl_can_start(tok : Token) -> Bool { + match tok { + ATTRIBUTE(_) + | PUB + | PRIV + | DECLARE + | ASYNC + | FN + | LET + | CONST + | TYPE + | STRUCT + | ENUM + | SUBERROR + | EXTENUM + | ENUMVIEW + | TRAIT + | IMPL + | TEST + | USING + | EXTERN => true + _ => false + } +} + +///| +fn top_level_decl_expected_tokens() -> Array[TokenKind] { + [ + TokenKind::TK_PUB, + TokenKind::TK_PRIV, + TokenKind::TK_TYPE, + TokenKind::TK_SUBERROR, + TokenKind::TK_ASYNC, + TokenKind::TK_FN, + TokenKind::TK_STRUCT, + TokenKind::TK_ENUM, + TokenKind::TK_EXTENUM, + TokenKind::TK_LET, + TokenKind::TK_CONST, + TokenKind::TK_EXTERN, + TokenKind::TK_TEST, + TokenKind::TK_IMPL, + TokenKind::TK_TRAIT, + TokenKind::TK_ENUMVIEW, + TokenKind::TK_ATTRIBUTE, + TokenKind::TK_USING, + ] +} + +///| +priv enum DeclPhase { + Start + AwaitingName + AwaitingLetPattern + AwaitingParameters + AwaitingEqual + AwaitingValue + AwaitingType + AwaitingBody + AwaitingImplRelation + AwaitingEnumViewConstructors + AwaitingEnumViewFor + AwaitingEnumViewWith + AwaitingMethodName + AwaitingUsingName + AwaitingExternLanguage + AwaitingExternFunction + AwaitingStub + Complete +} + +///| +priv enum DeclExpectation { + ExpectedTokens(Array[TokenKind]) + ExpectedDescription(String) + NoExpectation +} + +///| +fn decl_name_expectation(kind : NodeKind) -> Array[TokenKind] { + match kind { + Decl(DeclKind::Function) => + [TokenKind::TK_LIDENT, TokenKind::TK_EXTEND, TokenKind::TK_PACKAGE_NAME] + Decl(DeclKind::Impl) => [TokenKind::TK_UIDENT, TokenKind::TK_PACKAGE_NAME] + _ => [TokenKind::TK_LIDENT] + } +} + +///| +fn decl_phase_expectation( + phase : DeclPhase, + kind : NodeKind, +) -> DeclExpectation { + match phase { + AwaitingName => ExpectedTokens(decl_name_expectation(kind)) + AwaitingLetPattern => + ExpectedTokens([TokenKind::TK_LIDENT, TokenKind::TK_EXTEND]) + AwaitingParameters => ExpectedTokens([TokenKind::TK_LPAREN]) + AwaitingEqual => ExpectedTokens([TokenKind::TK_EQUAL]) + AwaitingValue => ExpectedDescription("simple expression") + AwaitingType => ExpectedDescription("type") + AwaitingBody | AwaitingEnumViewConstructors => + ExpectedTokens([TokenKind::TK_LBRACE]) + AwaitingImplRelation => + ExpectedTokens([TokenKind::TK_FOR, TokenKind::TK_WITH]) + AwaitingEnumViewFor => ExpectedTokens([TokenKind::TK_FOR]) + AwaitingEnumViewWith => ExpectedTokens([TokenKind::TK_WITH]) + AwaitingMethodName => + ExpectedTokens([TokenKind::TK_LIDENT, TokenKind::TK_EXTEND]) + AwaitingUsingName => ExpectedTokens([TokenKind::TK_PACKAGE_NAME]) + AwaitingExternLanguage | AwaitingStub => + ExpectedTokens([TokenKind::TK_STRING]) + AwaitingExternFunction => ExpectedTokens([TokenKind::TK_FN]) + Start | Complete => NoExpectation + } +} + +///| +fn decl_phase_after_name(kind : NodeKind, is_suberror : Bool) -> DeclPhase { + match kind { + Decl(DeclKind::Function) => AwaitingBody + Decl(DeclKind::Const) => AwaitingEqual + Decl(DeclKind::Type) => Complete + Decl(DeclKind::Struct) => AwaitingBody + Decl(DeclKind::Enum) => if is_suberror { Complete } else { AwaitingBody } + Decl(DeclKind::Trait) => AwaitingBody + Decl(DeclKind::Impl) => AwaitingImplRelation + Decl(DeclKind::EnumView) => AwaitingEnumViewConstructors + _ => Complete + } +} + +///| +fn ParserState::report_decl_phase( + self : ParserState, + phase : DeclPhase, + kind : NodeKind, +) -> Unit { + match decl_phase_expectation(phase, kind) { + ExpectedTokens(tokens) => self.report_expected_tokens(tokens) + ExpectedDescription(description) => + self.report_expected_description(description) + NoExpectation => () + } +} + +///| +fn ParserState::report_decl_phase_at( + self : ParserState, + phase : DeclPhase, + kind : NodeKind, + found : Token, + loc : Location, +) -> Unit { + match decl_phase_expectation(phase, kind) { + ExpectedTokens(tokens) => self.report_expected_tokens_at(found, loc, tokens) + ExpectedDescription(description) => + self.report_expected_description_at(found, loc, description) + NoExpectation => () + } +} + +///| +fn decl_keyword_expects_name(tok : Token) -> Bool { + match tok { + FN + | CONST + | TYPE + | STRUCT + | ENUM + | SUBERROR + | EXTENUM + | ENUMVIEW + | TRAIT + | IMPL => true + _ => false + } +} + +///| +fn decl_keyword_kind(tok : Token) -> NodeKind { + match tok { + FN => Decl(DeclKind::Function) + CONST => Decl(DeclKind::Const) + TYPE => Decl(DeclKind::Type) + STRUCT => Decl(DeclKind::Struct) + ENUM | SUBERROR | EXTENUM => Decl(DeclKind::Enum) + ENUMVIEW => Decl(DeclKind::EnumView) + TRAIT => Decl(DeclKind::Trait) + IMPL => Decl(DeclKind::Impl) + _ => Decl(DeclKind::BalancedDecl) + } +} + +///| +fn decl_token_label(tok : Token) -> String? { + match tok { + ATTRIBUTE(_) => Some("attrs") + PUB | PRIV => Some("vis") + DECLARE + | ASYNC + | FN + | LET + | CONST + | TYPE + | STRUCT + | ENUM + | SUBERROR + | EXTENUM + | ENUMVIEW + | TRAIT + | IMPL + | TEST + | USING + | EXTERN => Some("kw") + SEMI(_) => Some("sep") + EOF => Some("eof") + _ => Some("item") + } +} + +///| +fn decl_prefix_token(tok : Token) -> Bool { + match tok { + ATTRIBUTE(_) | PUB | PRIV | DECLARE | ASYNC | TEST | EXTERN => true + _ => false + } +} + +///| +fn parse_visibility_clause(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut is_pub = false + match state.peek() { + Some((PUB, _, _)) => { + is_pub = true + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + } + Some((PRIV, _, _)) => + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + _ => + push_child( + children, + Some("kw"), + state.missing_token_node(TokenKind::TK_PUB), + ) + } + if is_pub && state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("open"), state.expect_token(TokenKind::TK_LPAREN)) + match state.peek() { + Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) + | Some((READONLY, _, _)) + | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(attr) => push_child(children, Some("attr"), attr) + None => () + } + _ => + push_child( + children, + Some("attr"), + state.missing_token_node(TokenKind::TK_LIDENT), + ) + } + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RPAREN), + ) + } + node_from_children(Clause(ClauseKind::Visibility), children) +} + +///| +fn decl_allows_kind_update(kind : NodeKind) -> Bool { + match kind { + Impl(ImplKind::Extern) | Impl(ImplKind::Using) => false + _ => true + } +} + +///| +fn node_has_missing_token(node : CstNode, kind : TokenKind) -> Bool { + for entry in node.children_view() { + let (_, child) = entry + match child.kind() { + Missing(missing) if missing == kind => return true + _ => () + } + } + false +} + +///| +fn is_name_token(tok : Token) -> Bool { + match tok { + LIDENT(_) | UIDENT(_) | PACKAGE_NAME(_) | EXTEND => true + _ => false + } +} + +///| +fn decl_name_token_is_valid(tok : Token, kind : NodeKind) -> Bool { + match kind { + Decl(DeclKind::Function) => + tok is (LIDENT(_) | UIDENT(_) | PACKAGE_NAME(_) | EXTEND) + Decl(DeclKind::Type) + | Decl(DeclKind::Struct) + | Decl(DeclKind::Enum) + | Decl(DeclKind::EnumView) + | Decl(DeclKind::Trait) + | Decl(DeclKind::Impl) => tok is (UIDENT(_) | PACKAGE_NAME(_)) + _ => is_name_token(tok) + } +} + +///| +fn parse_parameter_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut is_labelled = false + let mut has_question = false + let pat = match state.peek() { + Some((EOF, _, _)) => { + state.report_expected_description("parameter name or label") + let pos = state.current_position() + node( + Pattern(PatternKind::MissingPattern), + empty_loc(pos), + empty_span(state.offset_for_position(pos)), + [], + ) + } + Some((POST_LABEL(_), _, _)) => { + is_labelled = true + parse_single_token_pattern(state, PatternKind::Binder) + } + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + parse_single_token_pattern(state, PatternKind::Binder) + Some((UNDERSCORE, _, _)) => + parse_single_token_pattern(state, PatternKind::Underscore) + Some((found, start, end)) => { + state.report_expected_description_at( + found, + mk_loc(start, end), + "parameter name or label", + ) + let invalid : Array[(String?, CstNode)] = [] + while !state.current_is(TokenKind::TK_COMMA) && + !state.current_is(TokenKind::TK_RPAREN) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(invalid, Some("error"), bad) + None => () + } + } + node_from_children(Pattern(PatternKind::MissingPattern), invalid) + } + None => node_from_children(Pattern(PatternKind::MissingPattern), []) + } + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + if state.current_is(TokenKind::TK_QUESTION) { + has_question = true + match state.consume_any() { + Some(question) => push_child(children, Some("question"), question) + None => () + } + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [ + TokenKind::TK_EQUAL, + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + push_child( + children, + Some("type"), + state.required_type_node(ty, reports_before_type), + ) + } + if state.current_is(TokenKind::TK_EQUAL) { + if !is_labelled && !has_question { + state.report( + empty_loc(state.current_position()), + "only labelled parameters may have default values", + ) + } + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + let default = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + if default.children_length() > 0 { + push_child(children, Some("default"), default) + } + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_parameter_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_parameter_item, + ParameterList, + sep_kind=TokenKind::TK_COMMA, + item_expected="parameter name or label", + ) +} + +///| +fn parse_function_stub_tokens( + state : ParserState, + children : Array[(String?, CstNode)], +) -> Unit { + match state.peek() { + Some((STRING(_), _, _)) => { + match state.consume_any() { + Some(stub) => push_child(children, Some("stub"), stub) + None => () + } + push_children(children, state.consume_trivia()) + match state.peek() { + Some((STRING(_), _, _)) => + match state.consume_any() { + Some(stub) => push_child(children, Some("stub"), stub) + None => () + } + _ => () + } + } + Some((MULTILINE_STRING(_), _, _)) => { + let mut done = false + while !done { + match state.peek() { + Some((MULTILINE_STRING(_), _, _)) => { + match state.consume_any() { + Some(stub) => push_child(children, Some("stub"), stub) + None => done = true + } + if state.next_significant_is_multiline_string() { + push_children(children, state.consume_trivia()) + } + } + _ => done = true + } + } + } + _ => () + } +} + +///| +fn decl_accepts_type_params(kind : NodeKind) -> Bool { + match kind { + Decl(DeclKind::Type) | Decl(DeclKind::Struct) | Decl(DeclKind::Enum) => true + _ => false + } +} + +///| +fn parse_type_parameter_item( + state : ParserState, + allow_constraints : Bool, +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((UIDENT(_), _, _)) | Some((UNDERSCORE, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => + push_child( + children, + Some("name"), + state.missing_token_node(TokenKind::TK_UIDENT), + ) + } + if state.current_is(TokenKind::TK_COLON) { + if !allow_constraints { + state.report_expected_tokens([TokenKind::TK_COMMA, TokenKind::TK_RBRACKET]) + } + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let ty = parse_type_until(state, [ + TokenKind::TK_PLUS, + TokenKind::TK_COMMA, + TokenKind::TK_RBRACKET, + TokenKind::TK_EOF, + ]) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + while state.current_is(TokenKind::TK_PLUS) { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let ty = parse_type_until(state, [ + TokenKind::TK_PLUS, + TokenKind::TK_COMMA, + TokenKind::TK_RBRACKET, + TokenKind::TK_EOF, + ]) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + } + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_type_decl_param_item(state : ParserState) -> CstNode { + parse_type_parameter_item(state, false) +} + +///| +fn parse_fun_type_param_item(state : ParserState) -> CstNode { + parse_type_parameter_item(state, true) +} + +///| +fn parse_type_decl_parameter_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACKET, + TokenKind::TK_RBRACKET, + parse_type_decl_param_item, + TypeArgumentList, + sep_kind=TokenKind::TK_COMMA, + require_nonempty=true, + ) +} + +///| +fn parse_fun_type_parameter_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACKET, + TokenKind::TK_RBRACKET, + parse_fun_type_param_item, + TypeArgumentList, + sep_kind=TokenKind::TK_COMMA, + require_nonempty=true, + ) +} + +///| +fn parse_tuple_struct_type_item(state : ParserState) -> CstNode { + parse_type_arrow(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) +} + +///| +fn parse_tuple_struct_type_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_tuple_struct_type_item, + TypeArgumentList, + sep_kind=TokenKind::TK_COMMA, + require_nonempty=true, + nonempty_expected="type", + ) +} + +///| +fn decl_body_uses_balanced_group(kind : NodeKind) -> Bool { + match kind { + Decl(Struct) | Decl(Enum) | Decl(Trait) => true + _ => false + } +} + +///| +fn decl_body_list_kind(kind : NodeKind) -> ListKind { + match kind { + Decl(Enum) => ConstructorList + Decl(Struct) | Decl(Trait) => FieldList + _ => StatementList + } +} + +///| +fn parse_balanced_brace_group( + state : ParserState, + list_kind : ListKind, +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut depth = 0 + match state.peek() { + Some((LBRACE, _, _)) => + match state.consume_any() { + Some(open) => { + push_child(children, Some("open"), open) + depth = 1 + } + None => () + } + _ => () + } + while depth > 0 { + match state.peek() { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => + push_children(children, state.consume_trivia()) + Some((EOF, _, _)) | None => depth = 0 + Some((LBRACE, _, _)) => + match state.consume_any() { + Some(open) => { + push_child(children, Some("open"), open) + depth += 1 + } + None => depth = 0 + } + Some((RBRACE, _, _)) => + match state.consume_any() { + Some(close) => { + push_child(children, Some("close"), close) + depth -= 1 + } + None => depth = 0 + } + Some((tok, _, _)) => + match state.consume_any() { + Some(consumed) => push_child(children, token_label(tok), consumed) + None => depth = 0 + } + } + } + node_from_children(List(list_kind), children) +} + +///| +fn parse_struct_member_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_struct_member_item, + FieldList, + sep_kind=TokenKind::TK_SEMI, + ) +} + +///| +fn parse_enum_constructor_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_enum_constructor_item, + ConstructorList, + sep_kind=TokenKind::TK_SEMI, + ) +} + +///| +fn parse_trait_member_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_trait_member_item, + FieldList, + sep_kind=TokenKind::TK_SEMI, + ) +} + +///| +fn parse_trait_member_item(state : ParserState) -> CstNode { + let diagnostics_before = state.reports.length() + let children : Array[(String?, CstNode)] = [] + let mut saw_params = false + while state.current_is(TokenKind::TK_ATTRIBUTE) { + match state.consume_any() { + Some(attr) => push_child(children, Some("attrs"), attr) + None => () + } + push_children(children, state.consume_trivia()) + } + if state.current_is(TokenKind::TK_ASYNC) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + push_children(children, state.consume_trivia()) + } + if state.current_is(TokenKind::TK_FN) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACKET) { + push_child( + children, + Some("quantifier"), + parse_fun_type_parameter_list(state), + ) + } + push_children(children, state.consume_trivia()) + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + Some((UIDENT(name), start, end)) => { + state.report( + mk_loc(start, end), + "Unexpected uppercase identifier \{name}, expected lowercase identifier in trait method name.", + ) + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + } + _ => { + state.report_expected_tokens([TokenKind::TK_LIDENT, TokenKind::TK_EXTEND]) + push_child( + children, + Some("name"), + state.missing_token_node_silent(TokenKind::TK_LIDENT), + ) + } + } + if state.reports.length() == diagnostics_before && + state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_trait_method_param_list(state)) + saw_params = true + } + if state.reports.length() == diagnostics_before && + state.current_is(TokenKind::TK_THIN_ARROW) { + push_child(children, Some("clause"), parse_func_return_type(state)) + } + if state.reports.length() == diagnostics_before && + ( + state.current_is(TokenKind::TK_RAISE) || + state.current_is(TokenKind::TK_NORAISE) + ) { + push_child(children, Some("clause"), parse_func_return_type(state)) + } + if state.reports.length() == diagnostics_before && + state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + match state.peek() { + Some((UNDERSCORE, _, _)) => + match state.consume_any() { + Some(stub) => push_child(children, Some("stub"), stub) + None => () + } + _ => state.report_expected_tokens([TokenKind::TK_UNDERSCORE]) + } + } + if state.reports.length() == diagnostics_before && !saw_params { + state.report_expected_tokens([TokenKind::TK_LPAREN]) + } + node_from_children(Decl(DeclKind::Function), children) +} + +///| +fn parse_trait_method_param_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_trait_method_param_item, + ParameterList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_trait_method_param_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LIDENT(_), _, _)) + | Some((POST_LABEL(_), _, _)) + | Some((EXTEND, _, _)) => + match state.peek(offset=1) { + Some((QUESTION, _, _)) | Some((COLON, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("pattern"), name) + None => () + } + _ => () + } + _ => () + } + if state.current_is(TokenKind::TK_QUESTION) { + match state.consume_any() { + Some(question) => push_child(children, Some("op"), question) + None => () + } + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + push_child( + children, + Some("type"), + state.required_type_node(ty, reports_before_type), + ) + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_enum_constructor_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + while state.current_is(TokenKind::TK_ATTRIBUTE) { + match state.consume_any() { + Some(attr) => push_child(children, Some("attrs"), attr) + None => () + } + push_children(children, state.consume_trivia()) + } + match state.peek() { + Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => + push_child( + children, + Some("name"), + state.missing_token_node(TokenKind::TK_UIDENT), + ) + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_constructor_param_list(state)) + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + match state.peek() { + Some((INT(_), _, _)) => + match state.consume_any() { + Some(tag) => push_child(children, Some("tag"), tag) + None => () + } + _ => + push_child( + children, + Some("tag"), + state.missing_token_node(TokenKind::TK_INT), + ) + } + } + node_from_children(Name(NameKind::Constructor), children) +} + +///| +fn parse_constructor_param_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_constructor_param_item, + ParameterList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_constructor_param_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + if state.current_is(TokenKind::TK_MUTABLE) { + match state.consume_any() { + Some(mut_token) => push_child(children, Some("mut"), mut_token) + None => () + } + } + match (state.peek(), state.peek(offset=1)) { + (Some((LIDENT(name), start, end)), Some((COLON, _, _))) => { + state.report( + mk_loc(start, end), + "Unexpected lowercase identifier \{name}, expected uppercase identifier in type name.", + ) + match state.consume_any() { + Some(name_node) => push_child(children, Some("type"), name_node) + None => () + } + return node_from_children(Clause(ClauseKind::Parameter), children) + } + _ => () + } + match state.peek() { + Some((POST_LABEL(_), _, _)) + | Some((LIDENT(_), _, _)) + | Some((EXTEND, _, _)) => + match state.peek(offset=1) { + Some((COLON, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + push_child( + children, + Some("type"), + state.required_type_node(ty, reports_before_type), + ) + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_struct_member_item(state : ParserState) -> CstNode { + parse_struct_field_item(state) +} + +///| +fn parse_struct_field_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut done_prefix = false + while !done_prefix { + match state.peek() { + Some((ATTRIBUTE(_), _, _)) => { + match state.consume_any() { + Some(attr) => push_child(children, Some("attrs"), attr) + None => () + } + push_children(children, state.consume_trivia()) + } + Some((PUB, _, _)) | Some((PRIV, _, _)) => { + push_child(children, Some("vis"), parse_visibility_clause(state)) + push_children(children, state.consume_trivia()) + } + Some((MUTABLE, _, _)) => { + match state.consume_any() { + Some(mut_token) => push_child(children, Some("mut"), mut_token) + None => () + } + push_children(children, state.consume_trivia()) + } + _ => done_prefix = true + } + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + Some((UIDENT(name), start, end)) => { + state.report( + mk_loc(start, end), + "Unexpected uppercase identifier \{name}, expected lowercase identifier in struct field name.", + ) + match state.consume_any() { + Some(name_node) => push_child(children, Some("name"), name_node) + None => () + } + } + Some((found, start, end)) => { + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + ]) + push_child( + children, + Some("name"), + state.missing_token_node_silent(TokenKind::TK_LIDENT), + ) + } + None => + push_child( + children, + Some("name"), + state.missing_token_node_with_expected(TokenKind::TK_LIDENT, [ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + ]), + ) + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [ + TokenKind::TK_NEWLINE, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + push_child( + children, + Some("type"), + state.required_type_node(ty, reports_before_type), + ) + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_impl_method_item(state : ParserState) -> CstNode { + let diagnostics_before = state.reports.length() + let children : Array[(String?, CstNode)] = [] + let mut saw_params = false + let mut saw_body = false + let mut saw_stub = false + if state.current_is(TokenKind::TK_FN) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACKET) { + push_child( + children, + Some("quantifier"), + parse_fun_type_parameter_list(state), + ) + } + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + Some((UIDENT(name), start, end)) => { + state.report( + mk_loc(start, end), + "Unexpected uppercase identifier \{name}, expected lowercase identifier in name of method.", + ) + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + } + _ => { + state.report_expected_tokens([TokenKind::TK_LIDENT, TokenKind::TK_EXTEND]) + push_child( + children, + Some("name"), + state.missing_token_node_silent(TokenKind::TK_LIDENT), + ) + } + } + if state.reports.length() == diagnostics_before && + state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_parameter_list(state)) + saw_params = true + } + if state.reports.length() == diagnostics_before && + ( + state.current_is(TokenKind::TK_THIN_ARROW) || + state.current_is(TokenKind::TK_RAISE) || + state.current_is(TokenKind::TK_NORAISE) + ) { + push_child(children, Some("clause"), parse_func_return_type(state)) + } + if state.reports.length() == diagnostics_before && + state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + saw_body = true + } + if state.reports.length() == diagnostics_before && + state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + match state.peek() { + Some((STRING(_), _, _)) | Some((MULTILINE_STRING(_), _, _)) => { + saw_stub = true + parse_function_stub_tokens(state, children) + } + _ => state.report_expected_tokens([TokenKind::TK_STRING]) + } + } + if state.reports.length() == diagnostics_before { + if !saw_params { + state.report_expected_tokens([TokenKind::TK_LPAREN]) + } else if !saw_body && !saw_stub { + state.report_expected_tokens([TokenKind::TK_LBRACE]) + } + } + node_from_children(Decl(DeclKind::Function), children) +} + +///| +fn parse_using_name_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_using_name_item, + FieldList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_using_name_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((TYPE, _, _)) | Some((TRAIT, _, _)) => + match state.consume_any() { + Some(kind) => push_child(children, Some("kind"), kind) + None => () + } + _ => () + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("target"), name) + None => () + } + _ => + push_child( + children, + Some("target"), + state.missing_token_node(TokenKind::TK_LIDENT), + ) + } + if state.current_is(TokenKind::TK_AS) { + match state.consume_any() { + Some(as_token) => push_child(children, Some("kw"), as_token) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) + | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(alias_name) => push_child(children, Some("name"), alias_name) + None => () + } + _ => () + } + } + node_from_children(Name(NameKind::Identifier), children) +} + +///| +fn parse_deriving_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_deriving_item, + TypeArgumentList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_deriving_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match (state.peek(), state.peek(offset=1)) { + ( + Some((PACKAGE_NAME(_), _, _)), + Some((DOT_UIDENT(_), _, _)) + | Some((DOT_LIDENT(_), _, _)), + ) => { + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + } + (Some((UIDENT(_), _, _)) | Some((LIDENT(_), _, _)), _) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => + push_child( + children, + Some("name"), + state.missing_token_node(TokenKind::TK_UIDENT), + ) + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("arg"), parse_argument_list(state)) + } + node_from_children(Name(NameKind::Identifier), children) +} + +///| +fn parse_balanced_decl_body(state : ParserState, kind : NodeKind) -> CstNode { + let diagnostics_before = state.reports.length() + let children : Array[(String?, CstNode)] = [] + let mut effective_kind = kind + let mut depth = 0 + let mut paren_depth = 0 + let mut done = false + let mut phase = DeclPhase::Start + let mut accepts_params = false + let mut enumview_has_constructors = false + let mut recovered_body_without_close = false + let mut saw_core_keyword = false + let mut params_complete = true + let mut saw_body = false + let mut saw_stub = false + let mut saw_language = false + let mut is_declare = false + let mut is_extern = false + let mut is_suberror = false + let mut is_extenum = false + let mut suberror_had_type_params = false + while !done { + match state.peek() { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => + push_children(children, state.consume_trivia()) + Some((tok, _, _)) if is_top_level_boundary(tok, depth) => done = true + Some((tok, start, end)) if saw_core_keyword && + top_level_decl_can_start(tok) => { + if state.reports.length() == diagnostics_before && + !recovered_body_without_close { + match decl_phase_expectation(phase, effective_kind) { + NoExpectation => + state.report(mk_loc(start, end), "Missing newline here.") + _ => + state.report_decl_phase_at( + phase, + effective_kind, + tok, + mk_loc(start, end), + ) + } + } + done = true + } + Some((tok, start, end)) if phase is AwaitingName && is_name_token(tok) => { + if !decl_name_token_is_valid(tok, effective_kind) { + state.report(mk_loc(start, end), "expected uppercase type name") + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + phase = decl_phase_after_name(effective_kind, is_suberror) + accepts_params = true + } + Some((LBRACKET, _, _)) if phase is AwaitingName && + effective_kind is Decl(DeclKind::Function) => + push_child( + children, + Some("quantifier"), + parse_fun_type_parameter_list(state), + ) + Some((LBRACKET, _, _)) if phase is AwaitingName && + ( + effective_kind is Decl(DeclKind::Impl) || + effective_kind is Decl(DeclKind::EnumView) + ) => + push_child( + children, + Some("quantifier"), + parse_fun_type_parameter_list(state), + ) + Some((AMPER, _, _)) if phase is AwaitingName && + effective_kind is Decl(DeclKind::Function) => + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + Some((LPAREN, _, _)) if effective_kind is Decl(DeclKind::Struct) => { + push_child(children, Some("body"), parse_tuple_struct_type_list(state)) + saw_body = true + accepts_params = false + phase = Complete + } + Some((LPAREN, _, _)) if accepts_params && + effective_kind is Decl(DeclKind::Function) => { + let params = parse_parameter_list(state) + params_complete = !node_has_missing_token(params, TokenKind::TK_RPAREN) + push_child(children, Some("params"), params) + accepts_params = false + phase = if is_extern { + AwaitingEqual + } else if is_declare { + Complete + } else { + AwaitingBody + } + } + Some((DOT_UIDENT(_), _, _)) | Some((DOT_LIDENT(_), _, _)) if accepts_params && + ( + effective_kind is Decl(DeclKind::Type) || + effective_kind is Decl(DeclKind::Struct) || + effective_kind is Decl(DeclKind::Enum) || + effective_kind is Decl(DeclKind::EnumView) || + effective_kind is Decl(DeclKind::Impl) + ) => { + match state.peek() { + Some((DOT_LIDENT(_), start, end)) => + state.report(mk_loc(start, end), "expected uppercase type name") + _ => () + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + } + Some((DOT_UIDENT(_), _, _)) | Some((DOT_LIDENT(_), _, _)) if accepts_params && + effective_kind is Decl(DeclKind::Function) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + Some((COLONCOLON, _, _)) if accepts_params && + effective_kind is Decl(DeclKind::Function) => { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) + | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(method_name) => + push_child(children, Some("method"), method_name) + None => () + } + _ => + push_child( + children, + Some("method"), + state.missing_token_node(TokenKind::TK_LIDENT), + ) + } + } + Some((LBRACKET, _, _)) if accepts_params && + decl_accepts_type_params(effective_kind) => { + if is_suberror { + suberror_had_type_params = true + } + push_child( + children, + Some("params"), + parse_type_decl_parameter_list(state), + ) + accepts_params = false + } + Some(_) if phase is AwaitingLetPattern => { + let pat = parse_pattern_until(state, [ + TokenKind::TK_COLON, + TokenKind::TK_EQUAL, + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } else { + push_child( + children, + Some("pattern"), + state.missing_token_node_with_expected(TokenKind::TK_LIDENT, [ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + ]), + ) + } + phase = AwaitingEqual + } + Some((LET, _, _)) => { + saw_core_keyword = true + match state.consume_any() { + Some(let_token) => + push_child(children, decl_token_label(LET), let_token) + None => () + } + if decl_allows_kind_update(kind) { + effective_kind = Decl(DeclKind::Let) + } + phase = AwaitingLetPattern + } + Some((TEST, _, _)) if effective_kind is Decl(DeclKind::Test) || + kind is Decl(DeclKind::BalancedDecl) => { + saw_core_keyword = true + match state.consume_any() { + Some(test_token) => + push_child(children, decl_token_label(TEST), test_token) + None => () + } + if decl_allows_kind_update(kind) { + effective_kind = Decl(DeclKind::Test) + } + match state.peek_significant() { + Some((STRING(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_parameter_list(state)) + } + phase = AwaitingBody + } + Some((PUB, _, _)) | Some((PRIV, _, _)) => + push_child(children, Some("vis"), parse_visibility_clause(state)) + Some((EXTERN, _, _)) => { + is_extern = true + phase = AwaitingExternLanguage + match state.consume_any() { + Some(consumed) => + push_child(children, decl_token_label(EXTERN), consumed) + None => () + } + push_children(children, state.consume_trivia()) + match state.peek() { + Some((STRING(_), _, _)) => + match state.consume_any() { + Some(language) => { + saw_language = true + phase = AwaitingExternFunction + push_child(children, Some("language"), language) + } + None => () + } + _ => state.report_expected_tokens([TokenKind::TK_STRING]) + } + } + Some((tok, _, _)) if decl_prefix_token(tok) => { + if tok is DECLARE { + is_declare = true + } + match state.consume_any() { + Some(consumed) => + push_child(children, decl_token_label(tok), consumed) + None => () + } + } + Some((tok, _, _)) if decl_keyword_expects_name(tok) => { + if is_extern && saw_language && !(tok is FN) { + match state.peek() { + Some((found, start, end)) => + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_FN, + ]) + None => () + } + while !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(children, Some("error"), bad) + None => () + } + } + done = true + continue + } + saw_core_keyword = true + if tok is SUBERROR { + is_suberror = true + } + if tok is EXTENUM { + is_extenum = true + } + match state.consume_any() { + Some(keyword) => push_child(children, decl_token_label(tok), keyword) + None => () + } + if decl_allows_kind_update(kind) { + effective_kind = decl_keyword_kind(tok) + } + phase = AwaitingName + } + Some((USING, _, _)) if effective_kind is Impl(ImplKind::Using) || + kind is Decl(DeclKind::BalancedDecl) => { + if is_declare { + match state.peek() { + Some((found, start, end)) => + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_PUB, + TokenKind::TK_PRIV, + TokenKind::TK_TYPE, + TokenKind::TK_SUBERROR, + TokenKind::TK_ASYNC, + TokenKind::TK_FN, + TokenKind::TK_STRUCT, + TokenKind::TK_ENUM, + TokenKind::TK_EXTENUM, + TokenKind::TK_LET, + TokenKind::TK_CONST, + TokenKind::TK_EXTERN, + TokenKind::TK_TEST, + TokenKind::TK_IMPL, + TokenKind::TK_TRAIT, + TokenKind::TK_ENUMVIEW, + TokenKind::TK_ATTRIBUTE, + TokenKind::TK_USING, + ]) + None => () + } + while !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(children, Some("error"), bad) + None => () + } + } + done = true + continue + } + saw_core_keyword = true + phase = AwaitingUsingName + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + effective_kind = Impl(ImplKind::Using) + match state.peek() { + Some((PACKAGE_NAME(_), _, _)) + | Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => { + phase = AwaitingBody + push_child(children, Some("name"), name) + } + None => () + } + _ => () + } + } + Some((tok, _, _)) if paren_depth > 0 && + pattern_can_start(tok) && + state.pattern_has_top_level_annotation() => { + let pat = parse_pattern_until(state, [ + TokenKind::TK_COLON, + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + } + Some((COLON, _, _)) if effective_kind is Decl(DeclKind::Trait) && + depth == 0 => { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let stops = [ + TokenKind::TK_PLUS, + TokenKind::TK_LBRACE, + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ] + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, stops) + let ty = if ty.children_length() > 0 { + ty + } else { + if state.reports.length() == reports_before_type { + state.report_expected_tokens([ + TokenKind::TK_UIDENT, + TokenKind::TK_PACKAGE_NAME, + ]) + } + state.missing_type_node() + } + push_child(children, Some("type"), ty) + while state.current_is(TokenKind::TK_PLUS) { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let ty = parse_type_arrow(state, stops) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + } + } + Some((COLON, start, end)) if effective_kind is Decl(DeclKind::Function) && + depth == 0 => { + state.report_expected_tokens_at(COLON, mk_loc(start, end), [ + TokenKind::TK_LBRACE, + ]) + while !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(children, Some("error"), bad) + None => () + } + } + saw_body = true + phase = Complete + done = true + } + Some((COLON, _, _)) => { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_RBRACKET, + TokenKind::TK_RBRACE, + TokenKind::TK_EQUAL, + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + push_child( + children, + Some("type"), + state.required_type_node(ty, reports_before_type), + ) + } + Some((THIN_ARROW, _, _)) if depth == 0 => { + phase = AwaitingType + push_child(children, Some("clause"), parse_func_return_type(state)) + phase = if is_declare { Complete } else { AwaitingBody } + } + Some((RAISE, _, _)) | Some((NORAISE, _, _)) if depth == 0 => { + push_child(children, Some("clause"), parse_func_return_type(state)) + phase = if is_declare { Complete } else { AwaitingBody } + } + Some((EQUAL, _, _)) if effective_kind is Decl(DeclKind::Type) => { + phase = AwaitingType + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [TokenKind::TK_SEMI, TokenKind::TK_EOF]) + let ty = state.required_type_node(ty, reports_before_type) + push_child(children, Some("type"), ty) + if !(ty.kind() is Type(TypeKind::MissingType)) { + phase = Complete + } + } + Some((EQUAL, _, _)) if effective_kind is Decl(DeclKind::Let) || + effective_kind is Decl(DeclKind::Const) => { + phase = AwaitingValue + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + phase = Complete + } else { + state.report_expected_description("simple expression") + push_child(children, Some("expr"), state.missing_expr_node()) + } + } + Some((EQUAL, _, _)) if effective_kind is Decl(DeclKind::Function) => { + phase = AwaitingStub + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + match state.peek() { + Some((STRING(_), _, _)) | Some((MULTILINE_STRING(_), _, _)) => { + saw_stub = true + phase = Complete + parse_function_stub_tokens(state, children) + } + _ => state.report_expected_tokens([TokenKind::TK_STRING]) + } + done = true + } + Some((FOR, _, _)) if effective_kind is Decl(DeclKind::Impl) => { + phase = AwaitingType + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [ + TokenKind::TK_WITH, + TokenKind::TK_LBRACE, + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + let ty = state.required_type_node(ty, reports_before_type) + push_child(children, Some("type"), ty) + if !(ty.kind() is Type(TypeKind::MissingType)) { + phase = Complete + } + } + Some((FOR, _, _)) if effective_kind is Decl(DeclKind::EnumView) => { + phase = AwaitingType + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let reports_before_type = state.reports.length() + let ty = parse_type_arrow(state, [ + TokenKind::TK_WITH, + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + let ty = state.required_type_node(ty, reports_before_type) + push_child(children, Some("type"), ty) + if !(ty.kind() is Type(TypeKind::MissingType)) { + phase = AwaitingEnumViewWith + } + } + Some((WITH, _, _)) if effective_kind is Decl(DeclKind::Impl) => { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + push_child(children, Some("method"), parse_impl_method_item(state)) + phase = Complete + } + Some((WITH, _, _)) if effective_kind is Decl(DeclKind::EnumView) => { + phase = AwaitingMethodName + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => { + push_child(children, Some("method"), name) + phase = AwaitingParameters + } + None => () + } + _ => { + state.report_expected_tokens([ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + ]) + push_child( + children, + Some("method"), + state.missing_token_node_silent(TokenKind::TK_LIDENT), + ) + } + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_parameter_list(state)) + phase = AwaitingBody + } + } + Some((PLUS_EQUAL, _, _)) if is_extenum => + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + Some((DERIVE, _, _)) if depth == 0 => { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("deriving"), parse_deriving_list(state)) + } + } + Some((WHERE, _, _)) if effective_kind is Decl(DeclKind::Function) => + push_child(children, Some("where"), parse_where_clause(state)) + Some((EXCLAMATION, _, _)) if effective_kind is Decl(DeclKind::Function) => + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + Some((LBRACKET, _, _)) if accepts_params && + effective_kind is Decl(DeclKind::Function) => + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + Some((QUESTION, _, _)) if depth == 0 && + (kind is Error || kind is Decl(DeclKind::BalancedDecl)) => + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + Some((RBRACE, start, end)) if effective_kind is Decl(DeclKind::Function) && + !saw_body => { + state.report_expected_tokens_at(RBRACE, mk_loc(start, end), [ + TokenKind::TK_LBRACE, + ]) + match state.consume_any() { + Some(bad) => push_child(children, Some("error"), bad) + None => () + } + saw_body = true + phase = Complete + done = true + } + Some((LBRACE, _, _)) => + if effective_kind is Impl(ImplKind::Using) { + saw_body = true + push_child(children, Some("body"), parse_using_name_list(state)) + phase = Complete + } else if effective_kind is Decl(DeclKind::Struct) { + saw_body = true + let body = parse_struct_member_list(state) + recovered_body_without_close = node_has_missing_token( + body, + TokenKind::TK_RBRACE, + ) + push_child(children, Some("body"), body) + phase = Complete + } else if effective_kind is Decl(DeclKind::Enum) { + saw_body = true + push_child(children, Some("body"), parse_enum_constructor_list(state)) + phase = Complete + } else if effective_kind is Decl(DeclKind::EnumView) && + !enumview_has_constructors { + saw_body = true + push_child(children, Some("body"), parse_enum_constructor_list(state)) + enumview_has_constructors = true + phase = AwaitingEnumViewFor + } else if effective_kind is Decl(DeclKind::EnumView) { + saw_body = true + push_child(children, Some("expr"), parse_block_expr(state)) + phase = Complete + } else if effective_kind is Decl(DeclKind::Trait) { + saw_body = true + push_child(children, Some("body"), parse_trait_member_list(state)) + phase = Complete + } else if decl_body_uses_balanced_group(effective_kind) { + saw_body = true + push_child( + children, + Some("body"), + parse_balanced_brace_group( + state, + decl_body_list_kind(effective_kind), + ), + ) + phase = Complete + } else { + saw_body = true + push_child(children, Some("body"), parse_block_expr(state)) + phase = Complete + } + Some((found, start, end)) if phase is AwaitingName => { + let expected = decl_name_expectation(effective_kind) + if effective_kind is Decl(DeclKind::Function) { + let mut recovery_end = end + while !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + match state.peek() { + Some((next, _, _)) if top_level_decl_can_start(next) => break + _ => () + } + match state.consume_any() { + Some(bad) => { + recovery_end = bad.loc().end + push_child(children, Some("error"), bad) + } + None => () + } + } + state.report_expected_tokens_at( + found, + mk_loc(start, recovery_end), + expected, + ) + phase = Complete + done = true + } else { + state.report_expected_tokens_at(found, mk_loc(start, end), expected) + match state.consume_any() { + Some(bad) => push_child(children, Some("error"), bad) + None => done = true + } + } + } + Some((tok, _, _)) => { + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + depth = next_depth(depth, tok) + paren_depth = next_paren_depth(paren_depth, tok) + } + None => done = true + } + } + let should_report_completion = state.reports.length() == diagnostics_before || + ( + effective_kind is Decl(DeclKind::Function) && + !is_extern && + phase is AwaitingBody && + params_complete + ) + if should_report_completion { + if effective_kind is Decl(DeclKind::Function) && + is_declare && + (saw_body || saw_stub) { + state.report( + empty_loc(state.current_position()), + "declare function cannot have a body", + ) + } else if effective_kind is Decl(DeclKind::Enum) && suberror_had_type_params { + state.report( + empty_loc(state.current_position()), + "Unexpected token here.", + ) + } else { + state.report_decl_phase(phase, effective_kind) + } + } + node_from_children(effective_kind, children) +} diff --git a/untyped_cst/parse_expr.mbt b/untyped_cst/parse_expr.mbt new file mode 100644 index 00000000..c52e7518 --- /dev/null +++ b/untyped_cst/parse_expr.mbt @@ -0,0 +1,3626 @@ +///| +fn statement_only_expression_token(tok : Token) -> Bool { + match tok { + RAISE + | BREAK + | CONTINUE + | RETURN + | LET + | LETREC + | GUARD + | DEFER + | PROOF_ASSERT + | PROOF_LET + | ELLIPSIS => true + _ => false + } +} + +///| +fn report_statement_only_expression( + state : ParserState, + context : String, +) -> Unit { + match state.peek_significant() { + Some((tok, start, end)) if statement_only_expression_token(tok) => + state.report( + mk_loc(start, end), + "statement expression is not allowed in \{context}", + ) + _ => () + } +} + +///| +fn report_removed_loop_else(state : ParserState) -> Unit { + match state.peek() { + Some((ELSE, start, end)) => + state.report_expected_tokens_at(ELSE, mk_loc(start, end), [ + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + _ => () + } +} + +///| +fn parse_expr_fragment(state : ParserState) -> CstNode { + let expr = parse_expr_until(state, [TokenKind::TK_EOF]) + let children : Array[(String?, CstNode)] = [] + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + push_children(children, state.consume_trivia()) + while state.current_is(TokenKind::TK_SEMI) { + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + push_children(children, state.consume_trivia()) + if !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + let next = parse_expr_until(state, [TokenKind::TK_EOF]) + if next.children_length() > 0 { + push_child(children, Some("expr"), next) + } + push_children(children, state.consume_trivia()) + } + } + if !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + state.report_expected_tokens([TokenKind::TK_SEMI, TokenKind::TK_EOF]) + while !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + match state.consume_any() { + Some(bad) => + push_child( + children, + Some("error"), + error_node(bad.loc(), bad.source_span(), [(Some("item"), bad)]), + ) + None => () + } + push_children(children, state.consume_trivia()) + } + } + match state.peek() { + Some((EOF, _, _)) => + match state.consume_any() { + Some(eof) => push_child(children, Some("eof"), eof) + None => () + } + None => push_child(children, Some("eof"), state.missing_eof_node()) + _ => () + } + node(Fragment(Expression), state.root_loc(), state.root_span(), children) +} + +///| +fn parse_expr_until( + state : ParserState, + stop_kinds : Array[TokenKind], + allow_argument_label_question? : Bool = false, +) -> CstNode { + let _ = allow_argument_label_question + let lhs = parse_expr_pipe(state, stop_kinds) + parse_expr_catch_suffix(state, stop_kinds, lhs) +} + +///| +fn parse_expr_catch_suffix( + state : ParserState, + stop_kinds : Array[TokenKind], + lhs : CstNode, +) -> CstNode { + if kind_in(TokenKind::TK_CATCH, stop_kinds) || + !state.current_is(TokenKind::TK_CATCH) { + return lhs + } + let children : Array[(String?, CstNode)] = [(Some("body"), lhs)] + match state.consume_any() { + Some(catch_kw) => push_child(children, Some("kw"), catch_kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("case"), parse_case_block(state)) + } + if state.current_is(TokenKind::TK_NORAISE) { + match state.consume_any() { + Some(else_kw) => push_child(children, Some("else_kw"), else_kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("else"), parse_case_block(state)) + } + } + node_from_children(Expr(ExprKind::Try), children) +} + +///| +fn parse_expr_pipe( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut lhs = parse_expr_infix(state, stop_kinds) + let mut done = false + while !done { + discard_expr_trivia(state) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((COLON, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("expr"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let ty = parse_type_arrow(state, expr_constraint_type_stops(stop_kinds)) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + lhs = node_from_children(Expr(ExprKind::Constraint), children) + } + Some((EQUAL, _, _)) + | Some((PLUS_EQUAL, _, _)) + | Some((AUGMENTED_ASSIGNMENT(_), _, _)) => { + let kind = match state.peek() { + Some((EQUAL, _, _)) => ExprKind::Assign + _ => ExprKind::AugmentedAssign + } + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let rhs = parse_expr_until(state, stop_kinds) + if rhs.children_length() > 0 { + push_child(children, Some("rhs"), rhs) + } + lhs = node_from_children(Expr(kind), children) + } + Some((PIPE, _, _)) | Some((PIPE_LEFT, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let rhs = parse_pipe_rhs(state, stop_kinds) + if rhs.children_length() > 0 { + push_child(children, Some("rhs"), rhs) + } + lhs = node_from_children(Expr(ExprKind::Infix), children) + } + Some((FAT_ARROW, _, _)) => { + if lhs.kind() is Expr(ExprKind::Constraint) && + lhs.first_child_with_label("open") is None { + match lhs.first_child_with_label("op") { + Some(op) => + state.report_expected_tokens_at(COLON, op.loc(), [ + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + None => () + } + } + let children : Array[(String?, CstNode)] = [ + (Some("params"), parse_arrow_params_from_expr(lhs)), + ] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let body = parse_expr_until(state, stop_kinds) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } + lhs = node_from_children(Expr(ExprKind::Function), children) + } + _ => done = true + } + } + lhs +} + +///| +fn expr_constraint_type_stops( + stop_kinds : Array[TokenKind], +) -> Array[TokenKind] { + let stops = stop_kinds.copy() + stops.push(TokenKind::TK_FAT_ARROW) + stops.push(TokenKind::TK_EQUAL) + stops.push(TokenKind::TK_SEMI) + stops.push(TokenKind::TK_RBRACE) + stops +} + +///| +fn stop_kinds_with_pipe(stop_kinds : Array[TokenKind]) -> Array[TokenKind] { + let stops : Array[TokenKind] = [TokenKind::TK_PIPE, TokenKind::TK_PIPE_LEFT] + for kind in stop_kinds { + stops.push(kind) + } + stops +} + +///| +fn parse_pipe_rhs( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let head = parse_expr_infix(state, stop_kinds) + discard_expr_trivia(state) + match state.peek() { + Some((FAT_ARROW, _, _)) => { + let children : Array[(String?, CstNode)] = [ + (Some("params"), parse_arrow_params_from_expr(head)), + ] + match state.consume_any() { + Some(op) => { + push_child(children, Some("op"), op) + push_child(children, Some("pipe"), op) + } + None => () + } + match state.peek_significant() { + Some((LBRACE, _, _)) => () + Some((found, start, end)) => + state.report_expected_description_at( + found, + mk_loc(start, end), + "`{ ... }`", + ) + None => state.report_expected_description("`{ ... }`") + } + let body = parse_expr_until(state, stop_kinds_with_pipe(stop_kinds)) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } + node_from_children(Expr(ExprKind::Function), children) + } + _ => head + } +} + +///| +fn parse_arrow_params_from_expr(expr : CstNode) -> CstNode { + match expr.kind() { + Expr(ExprKind::Tuple) => + match expr.first_child_with_label("body") { + Some(body) => { + let children : Array[(String?, CstNode)] = [] + for item in lower_item_children(body) { + push_child( + children, + Some("item"), + parse_arrow_param_from_expr(item), + ) + } + node( + List(ListKind::ParameterList), + body.loc(), + body.source_span(), + children, + ) + } + None => parse_single_arrow_param_list(expr) + } + _ => parse_single_arrow_param_list(expr) + } +} + +///| +fn parse_single_arrow_param_list(expr : CstNode) -> CstNode { + node_from_children(List(ListKind::ParameterList), [ + (Some("item"), parse_arrow_param_from_expr(expr)), + ]) +} + +///| +fn parse_arrow_param_from_expr(expr : CstNode) -> CstNode { + match expr.kind() { + Expr(ExprKind::Constraint) => { + let children : Array[(String?, CstNode)] = [] + match expr.first_child_with_label("expr") { + Some(pattern) => push_child(children, Some("pattern"), pattern) + None => () + } + match expr.first_child_with_label("type") { + Some(ty) => push_child(children, Some("type"), ty) + None => () + } + node_from_children(Clause(ClauseKind::Parameter), children) + } + _ => + node_from_children(Clause(ClauseKind::Parameter), [ + (Some("pattern"), expr), + ]) + } +} + +///| +fn parse_expr_infix( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + parse_expr_binary_prec(state, stop_kinds, 1) +} + +///| +fn infix_precedence(tok : Token) -> (Int, Bool)? { + match tok { + IMPLIES => Some((1, true)) + BARBAR => Some((2, true)) + AMPERAMPER => Some((3, true)) + BAR => Some((4, false)) + CARET => Some((5, false)) + AMPER => Some((6, false)) + INFIX1(_) => Some((7, false)) + INFIX2(_) => Some((8, false)) + PLUS | MINUS => Some((9, false)) + INFIX3(_) => Some((10, false)) + INFIX4(_) => Some((11, false)) + _ => None + } +} + +///| +fn parse_expr_binary_prec( + state : ParserState, + stop_kinds : Array[TokenKind], + min_prec : Int, +) -> CstNode { + let mut lhs = parse_expr_postfix(state, stop_kinds) + let mut done = false + while !done { + discard_expr_trivia(state) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((tok, _, _)) => + match infix_precedence(tok) { + Some((prec, right_assoc)) if prec >= min_prec => { + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let next_min = if right_assoc { prec } else { prec + 1 } + let rhs = parse_expr_binary_prec(state, stop_kinds, next_min) + if rhs.children_length() > 0 { + push_child(children, Some("rhs"), rhs) + } + lhs = node_from_children(Expr(ExprKind::Infix), children) + } + _ => done = true + } + None => done = true + } + } + lhs +} + +///| +fn parse_expr_postfix( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let lhs = parse_expr_range(state, stop_kinds) + discard_expr_trivia(state) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => lhs + Some((AS, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("expr"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let ty = parse_type_name_node(state) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + node_from_children(Expr(ExprKind::As), children) + } + Some((IS, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("expr"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let pat = parse_pattern_range(state, stop_kinds) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + node_from_children(Expr(ExprKind::Is), children) + } + Some((EQ_TILDE, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("expr"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let pat = parse_regex_pattern_until(state, stop_kinds) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + node_from_children(Expr(ExprKind::RegexMatch), children) + } + Some((LT_PLUS, _, _)) | Some((LT_QUESTION, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("expr"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let template = parse_expr_until(state, template_operand_stops(stop_kinds)) + if template.children_length() > 0 { + push_child(children, Some("template"), template) + } + node_from_children(Expr(ExprKind::TemplateWriting), children) + } + _ => lhs + } +} + +///| +fn parse_expr_simple_postfix( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut lhs = parse_expr_simple(state, stop_kinds) + let mut done = false + while !done { + discard_expr_trivia(state) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((_, _, _)) => + match parse_expr_postfix_suffix(state, lhs) { + Some(next) => lhs = next + None => done = true + } + None => done = true + } + } + lhs +} + +///| +fn template_operand_stops(stop_kinds : Array[TokenKind]) -> Array[TokenKind] { + let stops = stop_kinds.copy() + stops.push(TokenKind::TK_IMPLIES) + stops.push(TokenKind::TK_BARBAR) + stops.push(TokenKind::TK_AMPERAMPER) + stops.push(TokenKind::TK_BAR) + stops.push(TokenKind::TK_CARET) + stops.push(TokenKind::TK_AMPER) + stops.push(TokenKind::TK_INFIX1) + stops.push(TokenKind::TK_INFIX2) + stops.push(TokenKind::TK_PLUS) + stops.push(TokenKind::TK_MINUS) + stops.push(TokenKind::TK_INFIX3) + stops.push(TokenKind::TK_INFIX4) + stops +} + +///| +fn parse_expr_postfix_suffix(state : ParserState, lhs : CstNode) -> CstNode? { + match state.peek() { + Some((LPAREN, _, _)) => { + let args = parse_argument_list(state) + Some( + node_from_children(Expr(ExprKind::Apply), [ + (Some("func"), lhs), + (Some("arg"), args), + ]), + ) + } + Some((LBRACKET, _, _)) => Some(parse_array_get_suffix(state, lhs)) + Some((DOT_LIDENT(_), _, _)) | Some((DOT_INT(_), _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("self"), lhs)] + match state.consume_any() { + Some(field) => push_child(children, Some("field"), field) + None => () + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("arg"), parse_argument_list(state)) + Some(node_from_children(Expr(ExprKind::DotApply), children)) + } else { + Some(node_from_children(Expr(ExprKind::Field), children)) + } + } + Some((COLONCOLON, _, _)) => Some(parse_coloncolon_suffix(state, lhs)) + Some((DOTDOT, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("self"), lhs)] + let mut valid_field = true + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(field) => push_child(children, Some("field"), field) + None => () + } + Some((found, start, end)) => { + valid_field = false + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + ]) + while !state.current_is(TokenKind::TK_EOF) && !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(children, Some("error"), bad) + None => () + } + } + } + None => valid_field = false + } + if valid_field && state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("arg"), parse_argument_list(state)) + } else if valid_field { + push_child( + children, + Some("arg"), + state.missing_token_node(TokenKind::TK_LPAREN), + ) + } + Some(node_from_children(Expr(ExprKind::DotApply), children)) + } + _ => None + } +} + +///| +fn parse_coloncolon_suffix(state : ParserState, lhs : CstNode) -> CstNode { + let children : Array[(String?, CstNode)] = [(Some("type"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + push_children(children, state.consume_trivia()) + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_record_field_list(state)) + node_from_children(Expr(ExprKind::Record), children) + } else { + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("field"), name) + None => () + } + Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("field"), name) + None => () + } + Some((PACKAGE_NAME(_), _, _)) => { + match state.consume_any() { + Some(pkg) => push_child(children, Some("pkg"), pkg) + None => () + } + match state.peek() { + Some((DOT_UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("field"), name) + None => () + } + _ => + push_child( + children, + Some("field"), + state.missing_token_node(TokenKind::TK_UIDENT), + ) + } + } + _ => + push_child( + children, + Some("field"), + state.missing_token_node(TokenKind::TK_LIDENT), + ) + } + match children.last() { + Some((Some("field"), field)) => + match field.first_token() { + Some(UIDENT(_)) | Some(DOT_UIDENT(_)) => + node_from_children(Expr(ExprKind::Constructor), children) + _ => node_from_children(Expr(ExprKind::Method), children) + } + _ => node_from_children(Expr(ExprKind::Method), children) + } + } +} + +///| +fn brace_starts_record_expr(state : ParserState) -> Bool { + let mut offset = 1 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((DOTDOT, _, _)) => return true + Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) + | Some((EXTEND, _, _)) => { + offset += 1 + let mut after_name = false + while !after_name { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((COLON, _, _)) | Some((COMMA, _, _)) => return true + _ => after_name = true + } + } + done = true + } + _ => done = true + } + } + false +} + +///| +fn brace_starts_map_expr(state : ParserState) -> Bool { + let mut offset = 1 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((RBRACE, _, _)) => return true + Some((MINUS, _, _)) => { + offset += 1 + match state.peek(offset~) { + Some((INT(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) => { + offset += 1 + let mut after_key = false + while !after_key { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((COLON, _, _)) => return true + _ => after_key = true + } + } + } + _ => () + } + done = true + } + Some((STRING(_), _, _)) + | Some((BYTES(_), _, _)) + | Some((REGEX_LITERAL(_), _, _)) + | Some((CHAR(_), _, _)) + | Some((INT(_), _, _)) + | Some((BYTE(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) + | Some((TRUE, _, _)) + | Some((FALSE, _, _)) => { + offset += 1 + let mut after_key = false + while !after_key { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((COLON, _, _)) => return true + _ => after_key = true + } + } + done = true + } + _ => done = true + } + } + false +} + +///| +fn brace_expr_is_followed_by_range_operator(state : ParserState) -> Bool { + let mut offset = 0 + let mut depth = 0 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((LBRACE, _, _)) => { + depth += 1 + offset += 1 + } + Some((RBRACE, _, _)) => { + depth -= 1 + offset += 1 + if depth == 0 { + while true { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((tok, _, _)) => return range_operator(tok) + None => return false + } + } + } + } + Some((EOF, _, _)) | None => done = true + Some((_, _, _)) => offset += 1 + } + } + false +} + +///| +fn parse_record_expr(state : ParserState) -> CstNode { + node_from_children(Expr(ExprKind::Record), [ + (Some("body"), parse_record_field_list(state)), + ]) +} + +///| +fn parse_map_expr(state : ParserState) -> CstNode { + node_from_children(Expr(ExprKind::Map), [ + (Some("body"), parse_map_field_list(state)), + ]) +} + +///| +fn parse_record_field_list(state : ParserState) -> CstNode { + let list = parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_record_field_item, + FieldList, + sep_kind=TokenKind::TK_COMMA, + ) + let mut item_index = 0 + for i in 0.. { + match item.first_child_with_label("op") { + Some(op) if op.first_token() is Some(DOTDOT) && item_index > 0 => + state.report_expected_tokens_at(DOTDOT, op.loc(), [ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + ]) + _ => () + } + item_index += 1 + } + _ => () + } + } + list +} + +///| +fn parse_map_field_list(state : ParserState) -> CstNode { + let list = parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_map_field_item, + FieldList, + sep_kind=TokenKind::TK_COMMA, + ) + for i in 0.. { + match item.first_child_with_label("op") { + Some(op) if op.first_token() is Some(MINUS) => + for entry in item.children_view() { + let (_, child) = entry + if child.first_token() is Some(COLON) { + state.report_expected_tokens_at(COLON, child.loc(), [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + ]) + break + } + } + _ => () + } + break + } + _ => () + } + } + list +} + +///| +fn parse_map_field_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((MINUS, _, _)) => { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.peek() { + Some((INT(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) => + match state.consume_any() { + Some(key) => push_child(children, Some("key"), key) + None => () + } + _ => () + } + } + Some((STRING(_), _, _)) + | Some((BYTES(_), _, _)) + | Some((REGEX_LITERAL(_), _, _)) + | Some((CHAR(_), _, _)) + | Some((INT(_), _, _)) + | Some((BYTE(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) + | Some((TRUE, _, _)) + | Some((FALSE, _, _)) => + match state.consume_any() { + Some(key) => push_child(children, Some("key"), key) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + } + let expr = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Argument(ArgumentKind::Labelled), children) +} + +///| +fn parse_record_field_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + if state.current_is(TokenKind::TK_DOTDOT) { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } else { + match state.peek() { + Some((LIDENT(_), _, _)) + | Some((EXTEND, _, _)) + | Some((UIDENT(_), _, _)) + | Some((STRING(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } else if children.length() == 0 { + let expr = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + } + node_from_children(Argument(ArgumentKind::Labelled), children) +} + +///| +fn parse_array_get_suffix(state : ParserState, lhs : CstNode) -> CstNode { + let children : Array[(String?, CstNode)] = [(Some("self"), lhs)] + push_child(children, Some("open"), state.expect_token(TokenKind::TK_LBRACKET)) + let mut start_index : CstNode? = None + if !state.current_is(TokenKind::TK_RBRACKET) && + !state.current_is(TokenKind::TK_EOF) { + let index = parse_expr_until(state, [ + TokenKind::TK_RBRACKET, + TokenKind::TK_COLON, + TokenKind::TK_EOF, + ]) + if index.children_length() > 0 { + start_index = Some(index) + } + } + if state.current_is(TokenKind::TK_COLON) { + match start_index { + Some(index) => push_child(children, Some("start"), index) + None => () + } + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + if !state.current_is(TokenKind::TK_RBRACKET) && + !state.current_is(TokenKind::TK_EOF) { + let end_index = parse_expr_until(state, [ + TokenKind::TK_RBRACKET, + TokenKind::TK_EOF, + ]) + if end_index.children_length() > 0 { + push_child(children, Some("end"), end_index) + } + } + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RBRACKET), + ) + node_from_children(Expr(ExprKind::ArrayGetSlice), children) + } else { + match start_index { + Some(index) => push_child(children, Some("index"), index) + None => () + } + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RBRACKET), + ) + node_from_children(Expr(ExprKind::ArrayGet), children) + } +} + +///| +fn range_operator(tok : Token) -> Bool { + match tok { + RANGE_EXCLUSIVE + | RANGE_INCLUSIVE + | RANGE_LT_INCLUSIVE + | RANGE_INCLUSIVE_REV + | RANGE_EXCLUSIVE_REV => true + _ => false + } +} + +///| +fn parse_expr_range( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut lhs = parse_expr_prefix(state, stop_kinds) + match state.peek() { + Some((tok, _, _)) if !kind_in(tok.kind(), stop_kinds) && range_operator(tok) => { + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let rhs = parse_expr_prefix(state, stop_kinds) + if rhs.children_length() > 0 { + push_child(children, Some("rhs"), rhs) + } + lhs = node_from_children(Expr(ExprKind::Range), children) + } + _ => () + } + lhs +} + +///| +fn parse_expr_prefix( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + discard_expr_trivia(state) + match state.peek() { + Some((PLUS, _, _)) | Some((MINUS, _, _)) | Some((EXCLAMATION, _, _)) => { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let expr = parse_expr_prefix(state, stop_kinds) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Expr(ExprKind::Prefix), children) + } + _ => parse_expr_simple_postfix(state, stop_kinds) + } +} + +///| +fn parse_invalid_simple_expr( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let invalid_top_level_start = match state.peek() { + Some((found, start, end)) => { + state.report_expected_description_at( + found, + mk_loc(start, end), + "simple expression", + ) + top_level_decl_can_start(found) + } + None => false + } + while !state.current_in(stop_kinds) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(bad) => + push_child( + children, + Some("error"), + error_node(bad.loc(), bad.source_span(), [(Some("item"), bad)]), + ) + None => () + } + } + if invalid_top_level_start && state.current_is(TokenKind::TK_RBRACE) { + match state.peek() { + Some((found, start, end)) => + state.report_expected_tokens_at( + found, + mk_loc(start, end), + top_level_decl_expected_tokens(), + ) + None => () + } + } + node_from_children(Expr(ExprKind::MissingExpr), children) +} + +///| +fn parse_expr_simple( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => + node_from_children(Expr(ExprKind::MissingExpr), []) + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => { + let children = state.consume_trivia() + let expr = parse_expr_simple(state, stop_kinds) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Expr(ExprKind::Group), children) + } + Some((LOOP, _, _)) | Some((THROW, _, _)) | Some((QUESTION, _, _)) => + parse_invalid_simple_expr(state, stop_kinds) + Some((IF, _, _)) => parse_if_expr(state) + Some((MATCH, _, _)) => parse_match_expr(state) + Some((FORALL, _, _)) | Some((EXISTS, _, _)) => + parse_quantifier_expr(state, stop_kinds) + Some((LET, _, _)) => parse_let_statement(state) + Some((LETREC, _, _)) => parse_letrec_statement(state) + Some((GUARD, _, _)) => parse_guard_statement(state) + Some((DEFER, _, _)) => parse_defer_statement(state) + Some((PROOF_ASSERT, _, _)) => parse_proof_assert_statement(state) + Some((PROOF_LET, _, _)) => parse_proof_let_statement(state) + Some((TRY, _, _)) + | Some((TRY_QUESTION, _, _)) + | Some((TRY_EXCLAMATION, _, _)) => parse_try_expr(state) + Some((RETURN, _, _)) => + parse_keyword_value_expr(state, ExprKind::Return, stop_kinds) + Some((RAISE, _, _)) => + parse_keyword_value_expr(state, ExprKind::Raise, stop_kinds) + Some((BREAK, _, _)) => parse_break_expr(state, stop_kinds) + Some((CONTINUE, _, _)) => parse_continue_expr(state, stop_kinds) + Some((POST_LABEL(_), _, _)) if labelled_loop_prefix_starts(state) => + parse_labelled_loop_expr(state) + Some((WHILE, _, _)) => parse_keyword_body_expr(state, ExprKind::While) + Some((FOR, _, _)) => parse_for_expr(state) + Some((LEXMATCH, _, _)) => parse_lex_block_expr(state, ExprKind::LexMatch) + Some((LEXSCAN, _, _)) => parse_lex_block_expr(state, ExprKind::LexScan) + Some((FN, _, _)) | Some((ASYNC, _, _)) => + parse_function_expr(state, stop_kinds) + Some((LBRACE, _, _)) => + if brace_starts_record_expr(state) { + parse_record_expr(state) + } else if brace_starts_map_expr(state) { + parse_map_expr(state) + } else { + parse_block_expr(state) + } + Some((LPAREN, _, _)) => parse_paren_expr(state) + Some((LBRACKET, _, _)) | Some((LBRACKET_BAR, _, _)) => + parse_array_expr(state) + Some((AMPER, _, _)) => + match state.peek(offset=1) { + Some((UIDENT(_), _, _)) | Some((PACKAGE_NAME(_), _, _)) => + parse_type_name_node(state) + _ => + match state.consume_any() { + Some(token) => + node_from_children(Expr(ExprKind::Identifier), [ + (token_label_from_node(token), token), + ]) + None => node_from_children(Expr(ExprKind::MissingExpr), []) + } + } + Some((ELLIPSIS, _, _)) | Some((UNDERSCORE, _, _)) => + parse_single_token_expr(state, ExprKind::Hole, "token") + Some((STRING(_), _, _)) + | Some((BYTES(_), _, _)) + | Some((REGEX_LITERAL(_), _, _)) + | Some((INTERP(_), _, _)) + | Some((BYTES_INTERP(_), _, _)) + | Some((MULTILINE_STRING(_), _, _)) + | Some((MULTILINE_INTERP(_), _, _)) => parse_string_literal_node(state) + Some((REGEX_INTERP(_), _, _)) => parse_regex_interp_lex_pattern_node(state) + Some((TRUE, _, _)) + | Some((FALSE, _, _)) + | Some((CHAR(_), _, _)) + | Some((INT(_), _, _)) + | Some((BYTE(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) => + parse_single_token_expr(state, ExprKind::Literal, "token") + Some((LIDENT(_), _, _)) + | Some((EXTEND, _, _)) + | Some((PACKAGE_NAME(_), _, _)) => parse_ident_or_qualified_expr(state) + Some((UIDENT(_), _, _)) => + parse_single_token_expr(state, ExprKind::Constructor, "name") + Some(_) => parse_invalid_simple_expr(state, stop_kinds) + None => node_from_children(Expr(ExprKind::MissingExpr), []) + } +} + +///| +fn parse_keyword_value_expr( + state : ParserState, + kind : ExprKind, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if !state.current_in(stop_kinds) && + !state.current_is(TokenKind::TK_SEMI) && + !state.current_is(TokenKind::TK_RBRACE) && + !state.current_is(TokenKind::TK_EOF) { + let expr = parse_expr_until(state, stop_kinds) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + node_from_children(Expr(kind), children) +} + +///| +fn parse_break_expr( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + match state.peek() { + Some((POST_LABEL(_), _, _)) => + match state.consume_any() { + Some(label) => push_child(children, Some("label"), label) + None => () + } + _ => () + } + if !state.current_in(stop_kinds) && + !state.current_is(TokenKind::TK_SEMI) && + !state.current_is(TokenKind::TK_RBRACE) && + !state.current_is(TokenKind::TK_EOF) { + let expr = parse_expr_until(state, stop_kinds) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + node_from_children(Expr(ExprKind::Break), children) +} + +///| +fn parse_continue_expr( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut anchor : CstNode? = None + match state.consume_any() { + Some(kw) => { + anchor = Some(kw) + push_child(children, Some("kw"), kw) + } + None => () + } + match state.peek() { + Some((POST_LABEL(_), _, _)) => + match state.consume_any() { + Some(label) => { + anchor = Some(label) + push_child(children, Some("label"), label) + } + None => () + } + _ => () + } + let has_same_line_arg = match anchor { + Some(item) => + !state.current_starts_after_node_line(item) && + !state.current_is(TokenKind::TK_NEWLINE) + None => false + } + if has_same_line_arg && + !state.current_in(stop_kinds) && + !state.current_is(TokenKind::TK_SEMI) && + !state.current_is(TokenKind::TK_RBRACE) && + !state.current_is(TokenKind::TK_EOF) { + let args = parse_continue_argument_list(state, stop_kinds) + if args.children_length() > 0 { + push_child(children, Some("arg"), args) + } + } + node_from_children(Expr(ExprKind::Continue), children) +} + +///| +fn labelled_loop_prefix_starts(state : ParserState) -> Bool { + match (state.peek(), state.peek(offset=1)) { + (Some((POST_LABEL(_), _, _)), Some((COLON, _, _))) => { + let mut offset = 2 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((FOR, _, _)) | Some((WHILE, _, _)) => return true + _ => done = true + } + } + false + } + _ => false + } +} + +///| +fn parse_labelled_loop_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(label) => push_child(children, Some("label"), label) + None => () + } + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let loop_node = match state.peek() { + Some((FOR, _, _)) => parse_for_expr(state) + Some((WHILE, _, _)) => parse_keyword_body_expr(state, ExprKind::While) + _ => node_from_children(Expr(ExprKind::MissingExpr), []) + } + for i in 0.. children.push(child) + None => () + } + } + node_from_children(loop_node.kind(), children) +} + +///| +fn parse_continue_argument_list( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let item_stops = stop_kinds.copy() + item_stops.push(TokenKind::TK_COMMA) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((SEMI(_), _, _)) | Some((RBRACE, _, _)) | Some((EOF, _, _)) | None => + done = true + Some((COMMA, _, _)) => + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => done = true + } + Some(_) => { + let expr = parse_expr_until(state, item_stops) + if expr.children_length() > 0 { + push_child(children, Some("item"), expr) + } + push_children(children, state.consume_trivia()) + if state.current_is(TokenKind::TK_COMMA) { + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + } + } + } + } + node_from_children(List(ListKind::StatementList), children) +} + +///| +fn discard_expr_trivia(state : ParserState) -> Unit { + match state.peek() { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => { + let _ = state.consume_trivia() + } + _ => () + } +} + +///| +fn parse_single_token_expr( + state : ParserState, + kind : ExprKind, + label : String, +) -> CstNode { + match state.consume_any() { + Some(token) => node_from_children(Expr(kind), [(Some(label), token)]) + None => node_from_children(Expr(ExprKind::MissingExpr), []) + } +} + +///| +fn parse_ident_or_qualified_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let allow_qualified = match state.peek() { + Some((PACKAGE_NAME(_), _, _)) => true + _ => false + } + match state.consume_any() { + Some(first) => push_child(children, Some("name"), first) + None => () + } + match state.peek() { + Some((DOT_LIDENT(_), _, _)) if allow_qualified => { + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + node_from_children(Expr(ExprKind::Identifier), children) + } + Some((DOT_UIDENT(_), _, _)) if allow_qualified => { + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + node_from_children(Expr(ExprKind::Constructor), children) + } + _ => node_from_children(Expr(ExprKind::Identifier), children) + } +} + +///| +fn parse_type_name_node(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + if state.current_is(TokenKind::TK_AMPER) { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => + push_child( + children, + Some("name"), + state.missing_token_node(TokenKind::TK_UIDENT), + ) + } + match state.peek() { + Some((DOT_LIDENT(_), _, _)) | Some((DOT_UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + node_from_children(Type(TypeKind::Name), children) +} + +///| +fn parse_paren_expr(state : ParserState) -> CstNode { + if paren_expr_has_top_level_constraint(state) { + return parse_constraint_paren_expr(state) + } + let list = parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_argument_item, + ArgumentList, + sep_kind=TokenKind::TK_COMMA, + ) + node_from_children(Expr(ExprKind::Tuple), [(Some("body"), list)]) +} + +///| +fn paren_expr_has_top_level_constraint(state : ParserState) -> Bool { + let mut offset = 1 + let mut depth = 0 + let mut saw_colon = false + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((LPAREN, _, _)) + | Some((LBRACKET, _, _)) + | Some((LBRACE, _, _)) + | Some((LBRACKET_BAR, _, _)) => { + depth += 1 + offset += 1 + } + Some((RPAREN, _, _)) => + if depth == 0 { + return saw_colon + } else { + depth -= 1 + offset += 1 + } + Some((RBRACKET, _, _)) + | Some((BAR_RBRACKET, _, _)) + | Some((RBRACE, _, _)) => { + if depth > 0 { + depth -= 1 + } + offset += 1 + } + Some((COMMA, _, _)) if depth == 0 => return false + Some((COLON, _, _)) if depth == 0 => { + saw_colon = true + offset += 1 + } + Some((EOF, _, _)) | None => done = true + Some(_) => offset += 1 + } + } + false +} + +///| +fn parse_constraint_paren_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(open) => push_child(children, Some("open"), open) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_COLON, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + } + let ty = parse_type_until(state, [TokenKind::TK_RPAREN, TokenKind::TK_EOF]) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + push_child(children, Some("close"), state.expect_token(TokenKind::TK_RPAREN)) + node_from_children(Expr(ExprKind::Constraint), children) +} + +///| +fn parse_array_expr(state : ParserState) -> CstNode { + let open_kind = match state.peek() { + Some((LBRACKET_BAR, _, _)) => TokenKind::TK_LBRACKET_BAR + _ => TokenKind::TK_LBRACKET + } + let close_kind = match open_kind { + TokenKind::TK_LBRACKET_BAR => TokenKind::TK_BAR_RBRACKET + _ => TokenKind::TK_RBRACKET + } + if array_starts_list_comprehension(state) { + return parse_list_comprehension_expr(state, open_kind, close_kind) + } + let list = parse_delimited( + state, + open_kind, + close_kind, + parse_array_expr_item, + ArgumentList, + sep_kind=TokenKind::TK_COMMA, + ) + node_from_children(Expr(ExprKind::Array), [(Some("body"), list)]) +} + +///| +fn array_starts_list_comprehension(state : ParserState) -> Bool { + let mut offset = 1 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((FOR, _, _)) => return true + _ => done = true + } + } + false +} + +///| +fn parse_list_comprehension_expr( + state : ParserState, + open_kind : TokenKind, + close_kind : TokenKind, +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + push_children(children, state.consume_trivia()) + push_child(children, Some("open"), state.expect_token(open_kind)) + push_children(children, state.consume_trivia()) + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if list_comprehension_is_foreach(state) { + parse_list_comprehension_foreach_header(state, children, close_kind) + } else { + parse_list_comprehension_for_header(state, children, close_kind) + } + if state.current_is(TokenKind::TK_IF) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let guard_expr = parse_expr_until(state, [ + TokenKind::TK_FAT_ARROW, + close_kind, + TokenKind::TK_EOF, + ]) + if guard_expr.children_length() > 0 { + push_child(children, Some("guard"), guard_expr) + } + } + if state.current_is(TokenKind::TK_FAT_ARROW) { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + } + let body = parse_expr_until(state, [close_kind, TokenKind::TK_EOF]) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } + push_children(children, state.consume_trivia()) + push_child(children, Some("close"), state.expect_token(close_kind)) + node_from_children(Expr(ExprKind::ListComprehension), children) +} + +///| +fn list_comprehension_is_foreach(state : ParserState) -> Bool { + let mut offset = 0 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((IN, _, _)) => return true + Some((COMMA, _, _)) => { + offset += 1 + let mut skipping = true + while skipping { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + _ => skipping = false + } + } + match state.peek(offset~) { + Some((LIDENT(_), _, _)) + | Some((EXTEND, _, _)) + | Some((UNDERSCORE, _, _)) => offset += 1 + _ => return false + } + } + Some((EQUAL, _, _)) + | Some((SEMI(_), _, _)) + | Some((FAT_ARROW, _, _)) + | Some((IF, _, _)) + | Some((EOF, _, _)) => done = true + Some(_) => offset += 1 + None => done = true + } + } + false +} + +///| +fn list_comprehension_expr_stops(close_kind : TokenKind) -> Array[TokenKind] { + [ + TokenKind::TK_SEMI, + TokenKind::TK_NEWLINE, + TokenKind::TK_IF, + TokenKind::TK_FAT_ARROW, + close_kind, + TokenKind::TK_EOF, + ] +} + +///| +fn list_comprehension_binding_stops(close_kind : TokenKind) -> Array[TokenKind] { + [ + TokenKind::TK_COMMA, + TokenKind::TK_SEMI, + TokenKind::TK_NEWLINE, + TokenKind::TK_IF, + TokenKind::TK_FAT_ARROW, + close_kind, + TokenKind::TK_EOF, + ] +} + +///| +fn ParserState::current_starts_after_node_line( + self : ParserState, + item : CstNode, +) -> Bool { + match self.peek() { + Some((_, start, _)) => start.lnum > item.loc().end.lnum + None => false + } +} + +///| +fn ParserState::current_token_line(self : ParserState) -> Int? { + match self.peek() { + Some((_, start, _)) => Some(start.lnum) + None => None + } +} + +///| +fn ParserState::current_starts_after_line( + self : ParserState, + line : Int, +) -> Bool { + match self.current_token_line() { + Some(current_line) => current_line > line + None => false + } +} + +///| +fn parse_list_comprehension_foreach_header( + state : ParserState, + children : Array[(String?, CstNode)], + close_kind : TokenKind, +) -> Unit { + let pat = parse_foreach_pattern(state) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + if state.current_is(TokenKind::TK_IN) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + } + let expr = parse_expr_until(state, list_comprehension_expr_stops(close_kind)) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + if !state.current_is(TokenKind::TK_IF) && + !state.current_is(TokenKind::TK_FAT_ARROW) && + !state.current_is(close_kind) && + !state.current_is(TokenKind::TK_EOF) { + if state.current_is(TokenKind::TK_SEMI) || + state.current_is(TokenKind::TK_NEWLINE) { + consume_for_header_separator(state) + } + push_child( + children, + Some("init"), + parse_list_comprehension_binding_list(state, close_kind), + ) + if !state.current_is(TokenKind::TK_IF) && + !state.current_is(TokenKind::TK_FAT_ARROW) && + !state.current_is(close_kind) && + !state.current_is(TokenKind::TK_EOF) { + if state.current_is(TokenKind::TK_SEMI) || + state.current_is(TokenKind::TK_NEWLINE) { + consume_for_header_separator(state) + } + push_child( + children, + Some("continue"), + parse_list_comprehension_binding_list(state, close_kind), + ) + } + } +} + +///| +fn parse_list_comprehension_for_header( + state : ParserState, + children : Array[(String?, CstNode)], + close_kind : TokenKind, +) -> Unit { + let init = parse_list_comprehension_binding_list(state, close_kind) + push_child(children, Some("init"), init) + if !state.current_is(TokenKind::TK_IF) && + !state.current_is(TokenKind::TK_FAT_ARROW) && + !state.current_is(close_kind) && + !state.current_is(TokenKind::TK_EOF) { + if state.current_is(TokenKind::TK_SEMI) || + state.current_is(TokenKind::TK_NEWLINE) { + consume_for_header_separator(state) + } + if !state.current_is(TokenKind::TK_SEMI) && + !state.current_is(TokenKind::TK_NEWLINE) && + !state.current_is(TokenKind::TK_IF) && + !state.current_is(TokenKind::TK_FAT_ARROW) && + !state.current_is(close_kind) && + !state.current_is(TokenKind::TK_EOF) { + let cond_expr = parse_expr_until( + state, + list_comprehension_expr_stops(close_kind), + ) + if cond_expr.children_length() > 0 { + push_child(children, Some("cond"), cond_expr) + } + } + if !state.current_is(TokenKind::TK_IF) && + !state.current_is(TokenKind::TK_FAT_ARROW) && + !state.current_is(close_kind) && + !state.current_is(TokenKind::TK_EOF) { + if state.current_is(TokenKind::TK_SEMI) || + state.current_is(TokenKind::TK_NEWLINE) { + consume_for_header_separator(state) + } + push_child( + children, + Some("continue"), + parse_list_comprehension_binding_list(state, close_kind), + ) + } + } +} + +///| +fn parse_list_comprehension_binding_list( + state : ParserState, + close_kind : TokenKind, +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + push_children(children, state.consume_trivia()) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), [ + TokenKind::TK_SEMI, + TokenKind::TK_NEWLINE, + TokenKind::TK_IF, + TokenKind::TK_FAT_ARROW, + close_kind, + TokenKind::TK_EOF, + ]) => done = true + Some((COMMA, _, _)) => + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => done = true + } + Some(_) => { + let item = parse_list_comprehension_binding(state, close_kind) + push_child(children, Some("item"), item) + push_children(children, state.consume_trivia()) + if state.current_is(TokenKind::TK_COMMA) { + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + } else { + done = true + } + } + None => done = true + } + } + node_from_children(List(ListKind::StatementList), children) +} + +///| +fn parse_list_comprehension_binding( + state : ParserState, + close_kind : TokenKind, +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => + push_child( + children, + Some("name"), + state.missing_token_node(TokenKind::TK_LIDENT), + ) + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + } + let expr = parse_expr_until( + state, + list_comprehension_binding_stops(close_kind), + ) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_array_expr_item(state : ParserState) -> CstNode { + let stops = [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACKET, + TokenKind::TK_BAR_RBRACKET, + TokenKind::TK_EOF, + ] + match state.peek() { + Some((ELLIPSIS, _, _)) => parse_invalid_simple_expr(state, stops) + Some((DOTDOT, _, _)) => { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let expr = parse_expr_until(state, stops) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Argument(ArgumentKind::Positional), children) + } + _ => { + report_statement_only_expression(state, "array element") + parse_expr_until(state, stops) + } + } +} + +///| +fn parse_if_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let cond = parse_expr_until(state, [TokenKind::TK_LBRACE, TokenKind::TK_EOF]) + if cond.children_length() > 0 { + push_child(children, Some("cond"), cond) + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + } else { + state.report_expected_tokens([TokenKind::TK_LBRACE]) + push_child(children, Some("body"), state.missing_expr_node()) + } + if state.current_is(TokenKind::TK_ELSE) { + match state.consume_any() { + Some(else_kw) => push_child(children, Some("else"), else_kw) + None => () + } + let else_body = match state.peek() { + Some((IF, _, _)) => parse_if_expr(state) + Some((LBRACE, _, _)) => parse_block_expr(state) + Some((found, start, end)) => { + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_LBRACE, + ]) + parse_expr_until(state, [TokenKind::TK_SEMI, TokenKind::TK_EOF]) + } + None => parse_expr_until(state, [TokenKind::TK_SEMI, TokenKind::TK_EOF]) + } + if else_body.children_length() > 0 { + push_child(children, Some("else"), else_body) + } + } + node_from_children(Expr(ExprKind::If), children) +} + +///| +fn parse_try_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut kind = ExprKind::Try + match state.peek() { + Some((TRY_QUESTION, _, _)) | Some((TRY_EXCLAMATION, _, _)) => + kind = ExprKind::TryOperator + _ => () + } + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let body = parse_expr_until(state, [ + TokenKind::TK_CATCH, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } else { + state.report_expected_description("simple expression") + push_child(children, Some("body"), state.missing_expr_node()) + } + let mut saw_catch = false + if state.current_is(TokenKind::TK_CATCH) { + saw_catch = true + match state.consume_any() { + Some(catch_kw) => push_child(children, Some("kw"), catch_kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("case"), parse_case_block(state)) + } else { + state.report_expected_tokens([TokenKind::TK_LBRACE]) + push_child(children, Some("case"), state.missing_expr_node()) + } + } + if state.current_is(TokenKind::TK_NORAISE) { + match state.consume_any() { + Some(else_kw) => push_child(children, Some("else_kw"), else_kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("else"), parse_case_block(state)) + } + } + if kind is ExprKind::Try && !saw_catch { + state.report_expected_tokens([TokenKind::TK_CATCH]) + } + node_from_children(Expr(kind), children) +} + +///| +fn parse_keyword_body_expr(state : ParserState, kind : ExprKind) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let cond = parse_expr_until(state, [TokenKind::TK_LBRACE, TokenKind::TK_EOF]) + if cond.children_length() > 0 { + push_child(children, Some("cond"), cond) + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + } else { + state.report_expected_tokens([TokenKind::TK_LBRACE]) + push_child(children, Some("body"), state.missing_expr_node()) + } + if state.current_is(TokenKind::TK_NOBREAK) || + state.current_is(TokenKind::TK_ELSE) { + report_removed_loop_else(state) + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("else"), parse_block_expr(state)) + } + } + node_from_children(Expr(kind), children) +} + +///| +fn for_header_is_foreach(state : ParserState) -> Bool { + let mut offset = 1 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((IN, _, _)) => return true + Some((EQUAL, _, _)) + | Some((SEMI(_), _, _)) + | Some((LBRACE, _, _)) + | Some((EOF, _, _)) => done = true + Some(_) => offset += 1 + None => done = true + } + } + false +} + +///| +fn parse_for_expr(state : ParserState) -> CstNode { + if for_header_is_foreach(state) { + parse_foreach_expr(state) + } else { + parse_c_style_for_expr(state) + } +} + +///| +fn parse_foreach_pattern(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut first_pattern : CstNode? = None + let mut item_count = 0 + let mut saw_sep = false + push_children(children, state.consume_trivia()) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), [ + TokenKind::TK_IN, + TokenKind::TK_LBRACE, + TokenKind::TK_EOF, + ]) => done = true + Some((COMMA, _, _)) => { + saw_sep = true + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => done = true + } + } + Some(_) => { + match state.peek() { + Some((LIDENT(_), _, _)) + | Some((EXTEND, _, _)) + | Some((UNDERSCORE, _, _)) => () + Some((found, start, end)) => + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + TokenKind::TK_SEMI, + TokenKind::TK_LBRACE, + ]) + None => () + } + let pat = parse_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_IN, + TokenKind::TK_LBRACE, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + if first_pattern is None { + first_pattern = Some(pat) + } + item_count += 1 + } + push_child(children, Some("item"), pat) + push_children(children, state.consume_trivia()) + if state.current_is(TokenKind::TK_COMMA) { + saw_sep = true + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + } + } + None => done = true + } + } + if item_count == 0 { + state.report( + empty_loc(state.current_position()), + "foreach loop requires a binder", + ) + } + if item_count == 1 && !saw_sep { + match first_pattern { + Some(pat) => pat + None => node_from_children(Pattern(PatternKind::MissingPattern), []) + } + } else { + node_from_children(Pattern(PatternKind::Tuple), [ + (Some("body"), node_from_children(List(ListKind::PatternList), children)), + ]) + } +} + +///| +fn parse_foreach_iter_expr(state : ParserState) -> CstNode { + if state.current_is(TokenKind::TK_LBRACE) && + !brace_starts_record_expr(state) && + !brace_starts_map_expr(state) && + !brace_expr_is_followed_by_range_operator(state) { + node_from_children(Expr(ExprKind::MissingExpr), []) + } else { + parse_expr_until(state, foreach_iter_expr_stops()) + } +} + +///| +fn foreach_iter_expr_stops() -> Array[TokenKind] { + [ + TokenKind::TK_SEMI, + TokenKind::TK_NOBREAK, + TokenKind::TK_ELSE, + TokenKind::TK_EOF, + ] +} + +///| +fn parse_foreach_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let pat = parse_foreach_pattern(state) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + if state.current_is(TokenKind::TK_IN) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let expr = parse_foreach_iter_expr(state) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + if state.current_is(TokenKind::TK_SEMI) { + consume_for_header_separator(state) + push_child(children, Some("init"), parse_for_binding_list(state)) + if state.current_is(TokenKind::TK_SEMI) { + consume_for_header_separator(state) + push_child(children, Some("continue"), parse_for_binding_list(state)) + } + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + } + if state.current_is(TokenKind::TK_NOBREAK) || + state.current_is(TokenKind::TK_ELSE) { + report_removed_loop_else(state) + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("else"), parse_block_expr(state)) + } + } + if state.current_is(TokenKind::TK_WHERE) { + push_child(children, Some("where"), parse_where_clause(state)) + } + node_from_children(Expr(ExprKind::ForEach), children) +} + +///| +fn for_clause_stops() -> Array[TokenKind] { + [ + TokenKind::TK_COMMA, + TokenKind::TK_SEMI, + TokenKind::TK_NEWLINE, + TokenKind::TK_LBRACE, + TokenKind::TK_EOF, + ] +} + +///| +fn for_header_stops() -> Array[TokenKind] { + [ + TokenKind::TK_SEMI, + TokenKind::TK_NEWLINE, + TokenKind::TK_LBRACE, + TokenKind::TK_EOF, + ] +} + +///| +fn parse_for_binding(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => + push_child( + children, + Some("name"), + state.missing_token_node(TokenKind::TK_LIDENT), + ) + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + } + let expr = parse_expr_until(state, for_clause_stops()) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_for_binding_list(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + push_children(children, state.consume_trivia()) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), for_header_stops()) => + done = true + Some((COMMA, _, _)) => + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => done = true + } + Some(_) => { + push_child(children, Some("item"), parse_for_binding(state)) + push_children(children, state.consume_trivia()) + if state.current_is(TokenKind::TK_COMMA) { + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + } + } + None => done = true + } + } + node_from_children(List(ListKind::StatementList), children) +} + +///| +fn consume_for_header_separator(state : ParserState) -> Unit { + match state.peek() { + Some((SEMI(_), _, _)) => { + let _ = state.consume_any() + let _ = state.consume_trivia() + } + Some((NEWLINE, _, _)) => { + let _ = state.consume_trivia() + } + _ => () + } +} + +///| +fn consume_for_header_separators(state : ParserState) -> Unit { + let mut done = false + while !done { + match state.peek() { + Some((SEMI(_), _, _)) => { + let _ = state.consume_any() + } + Some((NEWLINE, _, _)) => { + let _ = state.consume_trivia() + } + _ => done = true + } + } +} + +///| +fn parse_c_style_for_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + push_child(children, Some("init"), parse_for_binding_list(state)) + consume_for_header_separator(state) + if !state.current_is(TokenKind::TK_SEMI) && + !state.current_is(TokenKind::TK_LBRACE) && + !state.current_is(TokenKind::TK_EOF) { + let cond = parse_expr_until(state, for_header_stops()) + if cond.children_length() > 0 { + push_child(children, Some("cond"), cond) + } + } + consume_for_header_separator(state) + push_child(children, Some("continue"), parse_for_binding_list(state)) + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + } + if state.current_is(TokenKind::TK_NOBREAK) || + state.current_is(TokenKind::TK_ELSE) { + report_removed_loop_else(state) + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("else"), parse_block_expr(state)) + } + } + if state.current_is(TokenKind::TK_WHERE) { + push_child(children, Some("where"), parse_where_clause(state)) + } + node_from_children(Expr(ExprKind::For), children) +} + +///| +fn parse_where_clause(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child( + children, + Some("body"), + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_where_field_item, + FieldList, + sep_kind=TokenKind::TK_COMMA, + ), + ) + } + node_from_children(Clause(ClauseKind::Where), children) +} + +///| +fn parse_where_field_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + } + let expr = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_quantifier_expr( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let ty = parse_type_arrow(state, [TokenKind::TK_COMMA, TokenKind::TK_EOF]) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + } + if state.current_is(TokenKind::TK_COMMA) { + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + } + let body = parse_expr_until(state, stop_kinds) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } + node_from_children(Expr(ExprKind::Quantifier), children) +} + +///| +fn parse_lex_block_expr(state : ParserState, kind : ExprKind) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_LBRACE, + TokenKind::TK_WITH, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + if state.current_is(TokenKind::TK_WITH) { + match state.consume_any() { + Some(with_kw) => push_child(children, Some("kw"), with_kw) + None => () + } + match state.consume_any() { + Some(strategy) => push_child(children, Some("name"), strategy) + None => () + } + } + if state.current_is(TokenKind::TK_LBRACE) { + if kind is ExprKind::LexScan { + push_child(children, Some("case"), parse_lexscan_case_block(state)) + } else { + push_child(children, Some("case"), parse_lexmatch_case_block(state)) + } + } + node_from_children(Expr(kind), children) +} + +///| +fn parse_lexmatch_case_block(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + push_child(children, Some("open"), state.expect_token(TokenKind::TK_LBRACE)) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((RBRACE, _, _)) => { + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RBRACE), + ) + done = true + } + Some((EOF, _, _)) | None => { + push_child( + children, + Some("close"), + state.missing_token_node(TokenKind::TK_RBRACE), + ) + done = true + } + Some((SEMI(_), _, _)) => + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + Some((COMMA, _, _)) => + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + _ => push_child(children, Some("case"), parse_lexmatch_case(state)) + } + } + node_from_children(List(ListKind::StatementList), children) +} + +///| +fn parse_lex_top_pattern_item(state : ParserState) -> CstNode { + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + parse_single_token_pattern(state, PatternKind::Binder) + Some((UNDERSCORE, _, _)) => + parse_single_token_pattern(state, PatternKind::Underscore) + _ => + parse_lex_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + } +} + +///| +fn parse_lexmatch_case_pattern(state : ParserState) -> CstNode { + match state.peek() { + Some((LPAREN, _, _)) => { + let list = parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_lex_top_pattern_item, + PatternList, + sep_kind=TokenKind::TK_COMMA, + require_nonempty=true, + ) + node_from_children(Pattern(PatternKind::Tuple), [(Some("body"), list)]) + } + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + parse_single_token_pattern(state, PatternKind::Binder) + Some((UNDERSCORE, _, _)) => + parse_single_token_pattern(state, PatternKind::Underscore) + _ => + parse_lex_pattern_until( + state, + [ + TokenKind::TK_IF, + TokenKind::TK_FAT_ARROW, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ], + allow_alias=false, + ) + } +} + +///| +fn parse_lexmatch_case(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + if state.current_is(TokenKind::TK_ELLIPSIS) { + match state.consume_any() { + Some(todo) => { + push_child( + children, + Some("pattern"), + node_from_children(Pattern(PatternKind::MissingPattern), [ + (Some("token"), todo), + ]), + ) + push_child( + children, + Some("body"), + node_from_children(Expr(ExprKind::Hole), [(Some("token"), todo)]), + ) + } + None => () + } + return node_from_children(Case(CaseKind::LexCase), children) + } + let pat = parse_lexmatch_case_pattern(state) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + if state.current_is(TokenKind::TK_IF) { + match state.consume_any() { + Some(if_token) => push_child(children, Some("kw"), if_token) + None => () + } + let guard_expr = parse_expr_until(state, [ + TokenKind::TK_FAT_ARROW, + TokenKind::TK_EOF, + ]) + if guard_expr.children_length() > 0 { + push_child(children, Some("cond"), guard_expr) + } + } + let reports_before_arrow = state.reports.length() + push_child(children, Some("op"), state.expect_token(TokenKind::TK_FAT_ARROW)) + if state.reports.length() > reports_before_arrow { + while !state.current_is(TokenKind::TK_SEMI) && + !state.current_is(TokenKind::TK_RBRACE) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(bad) => + push_child( + children, + Some("error"), + error_node(bad.loc(), bad.source_span(), [(Some("item"), bad)]), + ) + None => () + } + } + } else { + let body = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } + } + node_from_children(Case(CaseKind::LexCase), children) +} + +///| +fn parse_lexscan_case_block(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + push_child(children, Some("open"), state.expect_token(TokenKind::TK_LBRACE)) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((RBRACE, _, _)) => { + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RBRACE), + ) + done = true + } + Some((EOF, _, _)) | None => { + push_child( + children, + Some("close"), + state.missing_token_node(TokenKind::TK_RBRACE), + ) + done = true + } + Some((SEMI(_), _, _)) => + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + Some((_, _, _)) => + push_child(children, Some("case"), parse_lexscan_case(state)) + } + } + node_from_children(List(ListKind::StatementList), children) +} + +///| +fn parse_lexscan_case_pattern(state : ParserState) -> CstNode { + let stops = [ + TokenKind::TK_IF, + TokenKind::TK_FAT_ARROW, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ] + match state.peek() { + Some((UNDERSCORE, _, _)) | Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + parse_pattern_until(state, stops) + _ => parse_regex_as_pattern(state, stops) + } +} + +///| +fn parse_lexscan_case(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((ELLIPSIS, _, _)) => { + match state.consume_any() { + Some(todo) => { + push_child( + children, + Some("pattern"), + node_from_children(Pattern(PatternKind::MissingPattern), [ + (Some("token"), todo), + ]), + ) + push_child( + children, + Some("body"), + node_from_children(Expr(ExprKind::Hole), [(Some("token"), todo)]), + ) + } + None => () + } + return node_from_children(Case(CaseKind::MatchCase), children) + } + _ => () + } + let pat = parse_lexscan_case_pattern(state) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + if state.current_is(TokenKind::TK_IF) { + match state.consume_any() { + Some(if_token) => push_child(children, Some("kw"), if_token) + None => () + } + let guard_expr = parse_expr_until(state, [ + TokenKind::TK_FAT_ARROW, + TokenKind::TK_EOF, + ]) + if guard_expr.children_length() > 0 { + push_child(children, Some("cond"), guard_expr) + } + } + if state.current_is(TokenKind::TK_FAT_ARROW) { + match state.consume_any() { + Some(arrow) => push_child(children, Some("op"), arrow) + None => () + } + } + let body = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } + node_from_children(Case(CaseKind::MatchCase), children) +} + +///| +fn parse_function_expr( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + if state.current_is(TokenKind::TK_ASYNC) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + } + if state.current_is(TokenKind::TK_FN) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + } + if state.current_is(TokenKind::TK_EXCLAMATION) { + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => () + } + } + match state.peek() { + Some((tok, start, end)) if !(tok is LPAREN || tok is LBRACE) => { + state.report( + mk_loc(start, end), + "Failed to parse func: expected '(' or '{', got \{@debug.render(tok.to_repr())}", + ) + while !state.current_in(stop_kinds) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(bad) => + push_child( + children, + Some("error"), + error_node(bad.loc(), bad.source_span(), [(Some("item"), bad)]), + ) + None => () + } + } + return node_from_children(Expr(ExprKind::Function), children) + } + _ => () + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_parameter_list(state)) + } + if state.current_is(TokenKind::TK_THIN_ARROW) || + state.current_is(TokenKind::TK_RAISE) || + state.current_is(TokenKind::TK_NORAISE) { + push_child(children, Some("type"), parse_func_return_type(state)) + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + } + node_from_children(Expr(ExprKind::Function), children) +} + +///| +fn parse_case_block(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + push_child(children, Some("open"), state.expect_token(TokenKind::TK_LBRACE)) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((RBRACE, _, _)) => { + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RBRACE), + ) + done = true + } + Some((EOF, _, _)) | None => { + push_child( + children, + Some("close"), + state.missing_token_node(TokenKind::TK_RBRACE), + ) + done = true + } + Some((SEMI(_), _, _)) => + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + Some((_, _, _)) => + push_child(children, Some("case"), parse_match_case(state)) + } + } + node_from_children(List(ListKind::StatementList), children) +} + +///| +fn parse_match_case(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((ELLIPSIS, _, _)) => { + match state.consume_any() { + Some(todo) => { + push_child( + children, + Some("pattern"), + node_from_children(Pattern(PatternKind::MissingPattern), [ + (Some("token"), todo), + ]), + ) + push_child( + children, + Some("body"), + node_from_children(Expr(ExprKind::Hole), [(Some("token"), todo)]), + ) + } + None => () + } + return node_from_children(Case(CaseKind::MatchCase), children) + } + _ => () + } + let pat = parse_pattern_until(state, [ + TokenKind::TK_IF, + TokenKind::TK_FAT_ARROW, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + if pat.kind() is Pattern(PatternKind::Constraint) { + match pat.first_child_with_label("op") { + Some(op) => + match op.first_token() { + Some(found) => + state.report_expected_tokens_at(found, op.loc(), [ + TokenKind::TK_FAT_ARROW, + ]) + None => () + } + None => () + } + } + push_child(children, Some("pattern"), pat) + } + if state.current_is(TokenKind::TK_IF) { + match state.consume_any() { + Some(if_token) => push_child(children, Some("kw"), if_token) + None => () + } + let guard_expr = parse_expr_until(state, [ + TokenKind::TK_FAT_ARROW, + TokenKind::TK_EOF, + ]) + if guard_expr.children_length() > 0 { + push_child(children, Some("cond"), guard_expr) + } + } + if state.current_is(TokenKind::TK_FAT_ARROW) { + match state.consume_any() { + Some(arrow) => push_child(children, Some("op"), arrow) + None => () + } + } + let body = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if body.children_length() > 0 { + push_child(children, Some("body"), body) + } + node_from_children(Case(CaseKind::MatchCase), children) +} + +///| +fn parse_match_expr(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(match_token) => push_child(children, Some("kw"), match_token) + None => () + } + let scrutinee = parse_expr_until(state, [ + TokenKind::TK_LBRACE, + TokenKind::TK_EOF, + ]) + if scrutinee.children_length() > 0 { + push_child(children, Some("expr"), scrutinee) + } + match state.peek() { + Some((LBRACE, _, _)) => + match state.consume_any() { + Some(open) => push_child(children, Some("open"), open) + None => () + } + _ => () + } + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((RBRACE, _, _)) => { + match state.consume_any() { + Some(close) => push_child(children, Some("close"), close) + None => () + } + done = true + } + Some((EOF, _, _)) | None => done = true + Some((SEMI(_), _, _)) => + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + _ => push_child(children, Some("case"), parse_match_case(state)) + } + } + node_from_children(Expr(ExprKind::Match), children) +} + +///| +fn argument_label_question_can_follow_token(tok : Token) -> Bool { + match tok { + LIDENT(_) | EXTEND => true + _ => false + } +} + +///| +fn argument_label_question_has_valid_prefix(state : ParserState) -> Bool { + let mut offset = -1 + while state.pos + offset >= 0 { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset -= 1 + Some((tok, _, _)) => return argument_label_question_can_follow_token(tok) + None => return false + } + } + false +} + +///| +fn argument_label_question_has_valid_suffix(state : ParserState) -> Bool { + let mut offset = 1 + while true { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((COMMA, _, _)) | Some((RPAREN, _, _)) | Some((EQUAL, _, _)) => + return true + Some(_) | None => return false + } + } + false +} + +///| +fn argument_label_question_has_valid_context(state : ParserState) -> Bool { + argument_label_question_has_valid_prefix(state) && + argument_label_question_has_valid_suffix(state) +} + +///| +fn token_label_from_node(node : CstNode) -> String? { + match node.kind() { + Token(SEMI(_)) => Some("sep") + Token(tok) => expr_token_label(tok) + _ => Some("item") + } +} + +///| +fn expr_token_label(tok : Token) -> String? { + match tok { + TRY_QUESTION + | TRY_EXCLAMATION + | NORAISE + | IS + | IN + | FOR + | DEFER + | GUARD + | RETURN + | WHILE + | THIN_ARROW + | FAT_ARROW + | PROOF_ASSERT + | PROOF_LET + | IF + | ELSE + | PIPE + | PIPE_LEFT + | LT_PLUS + | LT_QUESTION + | AS + | BARBAR + | CARET + | AMPER + | AMPERAMPER + | BAR + | PLUS + | EXCLAMATION + | QUESTION + | MINUS + | EQUAL + | EQ_TILDE + | CATCH + | TRY + | RAISE + | WITH + | CONTINUE + | BREAK + | AUGMENTED_ASSIGNMENT(_) + | PLUS_EQUAL + | INFIX4(_) + | INFIX3(_) + | INFIX2(_) + | INFIX1(_) + | LEXMATCH + | LEXMATCH_QUESTION + | LEXSCAN + | RANGE_EXCLUSIVE + | RANGE_INCLUSIVE + | RANGE_EXCLUSIVE_REV + | RANGE_INCLUSIVE_REV + | RANGE_LT_INCLUSIVE + | WHERE + | NOBREAK + | FORALL + | EXISTS + | IMPLIES => Some("op") + _ => Some("token") + } +} + +///| +fn parse_argument_item(state : ParserState) -> CstNode { + let item_stops = [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ] + if state.current_argument_starts_optional_forwarding() { + parse_optional_forwarding_argument(state) + } else if state.current_argument_starts_labelled() { + parse_labelled_argument(state) + } else { + match state.peek_significant() { + Some((tok, _, _)) if statement_only_expression_token(tok) => + parse_invalid_simple_expr(state, item_stops) + _ => + parse_expr_until(state, item_stops, allow_argument_label_question=true) + } + } +} + +///| +fn ParserState::current_argument_starts_labelled(self : ParserState) -> Bool { + match (self.peek(), self.peek(offset=1)) { + (Some((LIDENT(_), _, _)), Some((EQUAL, _, _))) + | (Some((EXTEND, _, _)), Some((EQUAL, _, _))) + | (Some((UIDENT(_), _, _)), Some((EQUAL, _, _))) + | (Some((POST_LABEL(_), _, _)), Some((EQUAL, _, _))) + | (Some((POST_LABEL(_), _, _)), Some((COMMA, _, _))) + | (Some((POST_LABEL(_), _, _)), Some((RPAREN, _, _))) + | (Some((POST_LABEL(_), _, _)), Some((EOF, _, _))) => true + (Some((POST_LABEL(_), _, _)), Some((COLON, _, _))) => + !labelled_loop_prefix_starts(self) + _ => false + } +} + +///| +fn ParserState::current_argument_starts_optional_forwarding( + self : ParserState, +) -> Bool { + match (self.peek(), self.peek(offset=1)) { + (Some((LIDENT(_), _, _)), Some((QUESTION, _, _))) + | (Some((EXTEND, _, _)), Some((QUESTION, _, _))) + | (Some((UIDENT(_), _, _)), Some((QUESTION, _, _))) => true + _ => false + } +} + +///| +fn parse_labelled_argument(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((UIDENT(name), start, end)) => + state.report( + mk_loc(start, end), + "Unexpected uppercase identifier \{name}, expected lowercase identifier in .", + ) + _ => () + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + let mut kind = ArgumentKind::Labelled + match state.peek() { + Some((COMMA, _, _)) | Some((RPAREN, _, _)) | Some((EOF, _, _)) => + kind = ArgumentKind::LabelledPun + _ => { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + report_statement_only_expression(state, "labelled call argument") + let expr = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + } + node_from_children(Argument(kind), children) +} + +///| +fn parse_optional_forwarding_argument(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((UIDENT(name), start, end)) => + state.report( + mk_loc(start, end), + "Unexpected uppercase identifier \{name}, expected lowercase identifier in .", + ) + _ => () + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + match state.consume_any() { + Some(question) => push_child(children, Some("op"), question) + None => () + } + let mut kind = ArgumentKind::LabelledOptionPun + if state.current_is(TokenKind::TK_EQUAL) { + kind = ArgumentKind::LabelledOption + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + report_statement_only_expression(state, "optional call argument") + let expr = parse_expr_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + node_from_children(Argument(kind), children) +} + +///| +fn parse_statement_item(state : ParserState) -> CstNode { + parse_statement(state) +} + +///| +fn parse_let_statement(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut expr_kind = ExprKind::Let + match state.peek() { + Some((LET, _, _)) => + match state.consume_any() { + Some(let_token) => push_child(children, Some("kw"), let_token) + None => () + } + _ => () + } + match state.peek() { + Some((MUTABLE, _, _)) => + match state.consume_any() { + Some(mut_token) => { + push_child(children, Some("kw"), mut_token) + expr_kind = ExprKind::LetMut + } + None => () + } + _ => () + } + let pat = parse_pattern_until(state, [ + TokenKind::TK_COLON, + TokenKind::TK_EQUAL, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let ty = parse_type_arrow(state, [ + TokenKind::TK_EQUAL, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + node_from_children(Expr(expr_kind), children) +} + +///| +fn local_function_statement_can_start(state : ParserState) -> Bool { + match state.peek_significant() { + Some((FN, _, _)) => + match state.peek_nth_significant(1) { + Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) + | Some((EXTEND, _, _)) => true + _ => false + } + Some((ASYNC, _, _)) => + match (state.peek_nth_significant(1), state.peek_nth_significant(2)) { + (Some((FN, _, _)), Some((LIDENT(_), _, _))) + | (Some((FN, _, _)), Some((UIDENT(_), _, _))) + | (Some((FN, _, _)), Some((EXTEND, _, _))) => true + _ => false + } + _ => false + } +} + +///| +fn parse_statement(state : ParserState) -> CstNode { + match state.peek() { + Some((LET, _, _)) => parse_let_statement(state) + Some((LETREC, _, _)) => parse_letrec_statement(state) + Some((FN, _, _)) | Some((ASYNC, _, _)) if local_function_statement_can_start( + state, + ) => parse_local_function_statement(state) + Some((GUARD, _, _)) => parse_guard_statement(state) + Some((DEFER, _, _)) => parse_defer_statement(state) + Some((PROOF_ASSERT, _, _)) => parse_proof_assert_statement(state) + Some((PROOF_LET, _, _)) => parse_proof_let_statement(state) + Some((TYPE | STRUCT | ENUM | TRAIT | SUBERROR | EXTENUM, start, end)) => { + state.report( + mk_loc(start, end), + "local type declarations are not supported", + ) + parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + } + _ => + parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + } +} + +///| +fn parse_guard_statement(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let cond = parse_expr_until(state, [ + TokenKind::TK_ELSE, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if cond.children_length() > 0 { + push_child(children, Some("cond"), cond) + } + if state.current_is(TokenKind::TK_ELSE) { + match state.consume_any() { + Some(else_kw) => push_child(children, Some("kw"), else_kw) + None => () + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("else"), parse_block_expr(state)) + } + } + node_from_children(Expr(ExprKind::Guard), children) +} + +///| +fn parse_defer_statement(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Expr(ExprKind::Defer), children) +} + +///| +fn parse_proof_assert_statement(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + node_from_children(Stmt(StmtKind::ProofAssert), children) +} + +///| +fn parse_proof_let_statement(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + let expr = parse_expr_until(state, [ + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + node_from_children(Stmt(StmtKind::ProofLet), children) +} + +///| +fn parse_letrec_statement(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + push_child(children, Some("binding"), parse_letrec_binding(state)) + let mut done = false + while !done { + push_children(children, state.consume_trivia()) + match state.peek() { + Some((AND, _, _)) => { + match state.consume_any() { + Some(and_kw) => push_child(children, Some("kw"), and_kw) + None => () + } + push_child(children, Some("binding"), parse_letrec_binding(state)) + } + _ => done = true + } + } + node_from_children(Expr(ExprKind::LetRec), children) +} + +///| +fn parse_letrec_binding(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let ty = parse_type_arrow(state, [ + TokenKind::TK_EQUAL, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if ty.children_length() > 0 { + push_child(children, Some("binding_type"), ty) + } + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + } + let mut saw_fn = false + if state.current_is(TokenKind::TK_ASYNC) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + } + if state.current_is(TokenKind::TK_FN) { + match state.consume_any() { + Some(kw) => { + push_child(children, Some("kw"), kw) + saw_fn = true + } + None => () + } + } + if saw_fn { + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_parameter_list(state)) + } + if state.current_is(TokenKind::TK_THIN_ARROW) || + state.current_is(TokenKind::TK_RAISE) || + state.current_is(TokenKind::TK_NORAISE) { + push_child(children, Some("type"), parse_func_return_type(state)) + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + } + } else if !state.current_in([ + TokenKind::TK_AND, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) { + let expr = parse_expr_until(state, [ + TokenKind::TK_AND, + TokenKind::TK_SEMI, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + } + node_from_children(Expr(ExprKind::LetFn), children) +} + +///| +fn parse_local_function_statement(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + if state.current_is(TokenKind::TK_ASYNC) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + } + if state.current_is(TokenKind::TK_FN) { + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_EXCLAMATION) { + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => () + } + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child(children, Some("params"), parse_parameter_list(state)) + } + if state.current_is(TokenKind::TK_THIN_ARROW) || + state.current_is(TokenKind::TK_RAISE) || + state.current_is(TokenKind::TK_NORAISE) { + push_child(children, Some("type"), parse_func_return_type(state)) + } + if state.current_is(TokenKind::TK_LBRACE) { + push_child(children, Some("body"), parse_block_expr(state)) + } + node_from_children(Expr(ExprKind::LetFn), children) +} + +///| +fn parse_argument_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_argument_item, + ArgumentList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_statement_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_statement_item, + StatementList, + sep_kind=TokenKind::TK_SEMI, + ) +} + +///| +fn parse_block_expr(state : ParserState) -> CstNode { + let list = parse_statement_list(state) + node(Expr(ExprKind::Block), list.loc(), list.source_span(), [ + (Some("body"), list), + ]) +} diff --git a/untyped_cst/parse_list.mbt b/untyped_cst/parse_list.mbt new file mode 100644 index 00000000..a71b6dc8 --- /dev/null +++ b/untyped_cst/parse_list.mbt @@ -0,0 +1,200 @@ +///| +fn kind_in(kind : TokenKind, kinds : Array[TokenKind]) -> Bool { + for candidate in kinds { + if kind == candidate { + return true + } + } + false +} + +///| +fn ParserState::current_is(self : ParserState, kind : TokenKind) -> Bool { + match self.peek() { + Some((tok, _, _)) => tok.kind() == kind + None => false + } +} + +///| +fn ParserState::current_in( + self : ParserState, + kinds : Array[TokenKind], +) -> Bool { + match self.peek() { + Some((tok, _, _)) => kind_in(tok.kind(), kinds) + None => false + } +} + +///| +fn ParserState::consume_labeled( + self : ParserState, + label : String, + kind : TokenKind, +) -> (String?, CstNode) { + (Some(label), self.expect_token(kind)) +} + +///| +fn ParserState::consume_error_for_progress(self : ParserState) -> CstNode? { + match self.consume_any() { + Some(bad) => { + self.reports.push(Report::{ loc: bad.loc(), msg: "unexpected token" }) + Some(error_node(bad.loc(), bad.source_span(), [(Some("item"), bad)])) + } + None => None + } +} + +///| +fn parse_delimited( + state : ParserState, + open_kind : TokenKind, + close_kind : TokenKind, + item_parser : (ParserState) -> CstNode, + list_kind : ListKind, + sep_kind? : TokenKind = TokenKind::TK_COMMA, + follow_set? : Array[TokenKind] = [], + invalid_delims? : Array[TokenKind] = [], + require_nonempty? : Bool = false, + nonempty_expected? : String = "", + item_expected? : String = "", +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + push_children(children, state.consume_trivia()) + push_child(children, Some("open"), state.expect_token(open_kind)) + let mut done = false + let mut saw_item = false + let mut expect_item = true + let mut suppress_missing_close = false + while !done { + push_children(children, state.consume_trivia()) + if state.current_is(close_kind) { + if require_nonempty && !saw_item { + if nonempty_expected == "" { + state.report_expected_tokens([open_kind]) + } else { + state.report_expected_description(nonempty_expected) + } + } + push_child(children, Some("close"), state.expect_token(close_kind)) + done = true + } else if state.current_is(TokenKind::TK_EOF) || + state.current_in(follow_set) || + state.peek() is None { + if expect_item { + let item = item_parser(state) + push_child(children, Some("item"), item) + push_child( + children, + Some("close"), + state.missing_token_node_silent(close_kind), + ) + } else { + let close = if suppress_missing_close { + state.missing_token_node_silent(close_kind) + } else { + state.missing_token_node_with_expected(close_kind, [ + sep_kind, close_kind, + ]) + } + push_child(children, Some("close"), close) + } + done = true + } else if expect_item { + if state.current_is(sep_kind) { + if item_expected != "" { + state.report_expected_description(item_expected) + match state.consume_any() { + Some(bad) => + push_child( + children, + Some("error"), + error_node(bad.loc(), bad.source_span(), [(Some("item"), bad)]), + ) + None => done = true + } + } else { + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + } + } else if state.current_in(invalid_delims) { + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + } else { + let entry_pos = state.pos + let reports_before_item = state.reports.length() + let item = item_parser(state) + push_child(children, Some("item"), item) + if item.children_length() > 0 { + saw_item = true + expect_item = false + } + if state.reports.length() > reports_before_item && + ( + state.current_is(TokenKind::TK_EOF) || + state.current_in(follow_set) || + state.peek() is None + ) { + suppress_missing_close = true + } + if state.pos == entry_pos && + state.reports.length() > reports_before_item { + push_child( + children, + Some("close"), + state.missing_token_node_silent(close_kind), + ) + done = true + } + if state.pos == entry_pos && + !done && + !state.current_is(close_kind) && + !state.current_is(sep_kind) && + !state.current_is(TokenKind::TK_EOF) && + !state.current_in(follow_set) { + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + } + } + } else if state.current_is(sep_kind) { + push_child(children, Some("sep"), state.expect_token(sep_kind)) + expect_item = true + } else if state.current_in(invalid_delims) { + match state.consume_error_for_progress() { + Some(error) => push_child(children, Some("error"), error) + None => done = true + } + } else { + state.report_expected_tokens([sep_kind, close_kind, ..follow_set]) + let mut recovering = true + while recovering { + if state.current_is(sep_kind) || + state.current_is(close_kind) || + state.current_is(TokenKind::TK_EOF) || + state.current_in(follow_set) || + state.peek() is None { + recovering = false + } else { + match state.consume_any() { + Some(bad) => + push_child( + children, + Some("error"), + error_node(bad.loc(), bad.source_span(), [(Some("item"), bad)]), + ) + None => recovering = false + } + } + } + } + } + node_from_children(List(list_kind), children) +} diff --git a/untyped_cst/parse_pattern.mbt b/untyped_cst/parse_pattern.mbt new file mode 100644 index 00000000..32b3a771 --- /dev/null +++ b/untyped_cst/parse_pattern.mbt @@ -0,0 +1,1147 @@ +///| +fn parse_pattern_until( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + parse_pattern_alias(state, stop_kinds) +} + +///| +fn pattern_can_start(tok : Token) -> Bool { + match tok { + TRUE + | FALSE + | CHAR(_) + | INT(_) + | BYTE(_) + | DOUBLE(_) + | FLOAT(_) + | STRING(_) + | MULTILINE_STRING(_) + | BYTES(_) + | UNDERSCORE + | LIDENT(_) + | EXTEND + | UIDENT(_) + | PACKAGE_NAME(_) + | LPAREN + | LBRACKET + | LBRACE + | MINUS => true + _ => false + } +} + +///| +fn ParserState::pattern_has_top_level_annotation(self : ParserState) -> Bool { + let mut offset = 0 + let mut depth = 0 + while true { + match self.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((tok, _, _)) if depth == 0 => + match tok { + COLON => return true + COMMA | RPAREN | RBRACKET | RBRACE | EQUAL | SEMI(_) | EOF => + return false + _ => { + depth = next_depth(depth, tok) + offset += 1 + } + } + Some((tok, _, _)) => { + depth = next_depth(depth, tok) + offset += 1 + } + None => return false + } + } + false +} + +///| +fn parse_pattern_atom( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => + node_from_children(Pattern(PatternKind::MissingPattern), []) + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => { + let children = state.consume_trivia() + let pat = parse_pattern_atom(state, stop_kinds) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + node_from_children(Pattern(PatternKind::Constraint), children) + } + Some((UNDERSCORE, _, _)) => + parse_single_token_pattern(state, PatternKind::Underscore) + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => { + let mut node = parse_single_token_pattern(state, PatternKind::Binder) + if state.current_is(TokenKind::TK_LPAREN) { + let args = parse_constructor_pattern_args(state) + node = node_from_children(Pattern(PatternKind::SpecialConstructor), [ + (Some("name"), node), + (Some("params"), args), + ]) + } + node + } + Some((UIDENT(_), _, _)) | Some((PACKAGE_NAME(_), _, _)) => { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + match state.peek() { + Some((DOT_UIDENT(_), _, _)) | Some((DOT_LIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLONCOLON) { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + match state.peek() { + Some((DOT_UIDENT(_), _, _)) | Some((DOT_LIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + } + if state.current_is(TokenKind::TK_LPAREN) { + push_child( + children, + Some("params"), + parse_constructor_pattern_args(state), + ) + } + node_from_children(Pattern(PatternKind::Constructor), children) + } + Some((LPAREN, _, _)) => { + let list = parse_constructor_pattern_args(state) + node_from_children(Pattern(PatternKind::Tuple), [(Some("body"), list)]) + } + Some((LBRACKET, _, _)) => { + let list = parse_array_pattern_list(state) + node_from_children(Pattern(PatternKind::Array), [(Some("body"), list)]) + } + Some((LBRACE, _, _)) => + if brace_starts_map_pattern(state) { + let list = parse_map_pattern_list(state) + node_from_children(Pattern(PatternKind::Map), [(Some("body"), list)]) + } else { + let list = parse_record_pattern_list(state) + node_from_children(Pattern(PatternKind::Record), [(Some("body"), list)]) + } + Some((found, start, end)) if found is (REGEX_INTERP(_) | INTERP(_)) => { + let pat = parse_regex_interp_lex_pattern_node(state) + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_TRUE, + TokenKind::TK_FALSE, + TokenKind::TK_CHAR, + TokenKind::TK_INT, + TokenKind::TK_BYTE, + TokenKind::TK_FLOAT, + TokenKind::TK_DOUBLE, + TokenKind::TK_MINUS, + TokenKind::TK_BYTES, + TokenKind::TK_STRING, + TokenKind::TK_UNDERSCORE, + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + TokenKind::TK_PACKAGE_NAME, + TokenKind::TK_UIDENT, + TokenKind::TK_LPAREN, + TokenKind::TK_LBRACKET, + TokenKind::TK_LBRACE, + ]) + pat + } + Some((found, start, end)) if found is REGEX_LITERAL(_) => { + let pat = parse_literal_pattern(state, stop_kinds) + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_TRUE, + TokenKind::TK_FALSE, + TokenKind::TK_CHAR, + TokenKind::TK_INT, + TokenKind::TK_BYTE, + TokenKind::TK_FLOAT, + TokenKind::TK_DOUBLE, + TokenKind::TK_MINUS, + TokenKind::TK_BYTES, + TokenKind::TK_STRING, + TokenKind::TK_UNDERSCORE, + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + TokenKind::TK_PACKAGE_NAME, + TokenKind::TK_UIDENT, + TokenKind::TK_LPAREN, + TokenKind::TK_LBRACKET, + TokenKind::TK_LBRACE, + ]) + pat + } + Some((TRUE, _, _)) + | Some((FALSE, _, _)) + | Some((CHAR(_), _, _)) + | Some((INT(_), _, _)) + | Some((BYTE(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) + | Some((STRING(_), _, _)) + | Some((BYTES(_), _, _)) + | Some((MINUS, _, _)) => parse_literal_pattern(state, stop_kinds) + Some((_, _, _)) => + match state.consume_any() { + Some(token) => + node_from_children(Pattern(PatternKind::Literal), [ + (token_label_from_node(token), token), + ]) + None => node_from_children(Pattern(PatternKind::MissingPattern), []) + } + None => node_from_children(Pattern(PatternKind::MissingPattern), []) + } +} + +///| +fn brace_starts_map_pattern(state : ParserState) -> Bool { + let mut offset = 1 + let mut done = false + while !done { + match state.peek(offset~) { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => offset += 1 + Some((STRING(_), _, _)) + | Some((BYTES(_), _, _)) + | Some((REGEX_LITERAL(_), _, _)) + | Some((INT(_), _, _)) + | Some((BYTE(_), _, _)) + | Some((CHAR(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) + | Some((TRUE, _, _)) + | Some((FALSE, _, _)) + | Some((MINUS, _, _)) => return true + _ => done = true + } + } + false +} + +///| +fn parse_single_token_pattern( + state : ParserState, + kind : PatternKind, +) -> CstNode { + match state.consume_any() { + Some(token) => node_from_children(Pattern(kind), [(Some("name"), token)]) + None => node_from_children(Pattern(PatternKind::MissingPattern), []) + } +} + +///| +fn parse_literal_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let _ = stop_kinds + match state.consume_any() { + Some(token) => push_child(children, token_label_from_node(token), token) + None => () + } + let first_is_minus = match children.get(0) { + Some((_, child)) => + match child.kind() { + Token(MINUS) => true + _ => false + } + None => false + } + match state.peek() { + Some((INT(_), _, _)) if first_is_minus => + match state.consume_any() { + Some(token) => push_child(children, Some("token"), token) + None => () + } + Some((DOUBLE(_), _, _)) if first_is_minus => + match state.consume_any() { + Some(token) => push_child(children, Some("token"), token) + None => () + } + Some((FLOAT(_), _, _)) if first_is_minus => + match state.consume_any() { + Some(token) => push_child(children, Some("token"), token) + None => () + } + _ => () + } + node_from_children(Pattern(PatternKind::Literal), children) +} + +///| +fn parse_constructor_pattern_args(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_constructor_pattern_arg, + PatternList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_constructor_pattern_arg(state : ParserState) -> CstNode { + match (state.peek_significant(), state.peek_nth_significant(1)) { + (Some((DOTDOT, _, _)), _) => parse_dotdot_pattern_rest(state) + (Some((POST_LABEL(_), _, _)), _) => + parse_labelled_pun_constructor_pattern_arg(state) + ( + Some((LIDENT(_), _, _)) + | Some((EXTEND, _, _)) + | Some((UIDENT(_), _, _)), + Some((EQUAL, _, _)), + ) => parse_labelled_constructor_pattern_arg(state) + _ => + parse_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + } +} + +///| +fn parse_labelled_pun_constructor_pattern_arg(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + node_from_children(Argument(ArgumentKind::LabelledPun), children) +} + +///| +fn parse_labelled_constructor_pattern_arg(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + } + let pat = parse_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + node_from_children(Argument(ArgumentKind::Labelled), children) +} + +///| +fn parse_array_pattern_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACKET, + TokenKind::TK_RBRACKET, + parse_array_pattern_item, + PatternList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_array_pattern_item(state : ParserState) -> CstNode { + match state.peek() { + Some((DOTDOT, _, _)) => parse_dotdot_pattern_rest(state) + _ => + parse_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACKET, + TokenKind::TK_EOF, + ]) + } +} + +///| +fn parse_dotdot_pattern_rest(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.peek() { + Some((STRING(_), _, _)) => + match state.consume_any() { + Some(str) => push_child(children, Some("str"), str) + None => () + } + Some((BYTES(_), _, _)) => + match state.consume_any() { + Some(bytes) => push_child(children, Some("bytes"), bytes) + None => () + } + Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("const"), name) + None => () + } + Some((PACKAGE_NAME(_), _, _)) => { + match state.consume_any() { + Some(pkg) => push_child(children, Some("pkg"), pkg) + None => () + } + match state.peek() { + Some((DOT_UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("const"), name) + None => () + } + _ => () + } + } + Some((AS, _, _)) => { + match state.consume_any() { + Some(as_tok) => push_child(children, Some("as"), as_tok) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + } + Some((UNDERSCORE, _, _)) | Some((LIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + node_from_children(Pattern(PatternKind::Rest), children) +} + +///| +fn parse_record_pattern_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_record_pattern_item, + PatternList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_record_pattern_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((DOTDOT, _, _)) => { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + return node_from_children(Pattern(PatternKind::Rest), children) + } + Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) + | Some((STRING(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let pat = parse_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + } else if children.length() == 0 { + let pat = parse_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_map_pattern_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_map_pattern_item, + PatternList, + sep_kind=TokenKind::TK_COMMA, + ) +} + +///| +fn parse_map_pattern_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((found, start, end)) if found is REGEX_LITERAL(_) => + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + TokenKind::TK_TRUE, + TokenKind::TK_FALSE, + TokenKind::TK_CHAR, + TokenKind::TK_INT, + TokenKind::TK_FLOAT, + TokenKind::TK_DOUBLE, + TokenKind::TK_MINUS, + TokenKind::TK_STRING, + TokenKind::TK_REGEX_LITERAL, + ]) + _ => () + } + match state.peek() { + Some((DOTDOT, _, _)) => { + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + return node_from_children(Pattern(PatternKind::Rest), children) + } + Some((MINUS, _, _)) => { + state.report( + empty_loc(state.current_position()), + "negative map keys are not supported in patterns", + ) + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.consume_any() { + Some(key) => push_child(children, Some("key"), key) + None => () + } + } + Some((STRING(_), _, _)) + | Some((BYTES(_), _, _)) + | Some((REGEX_LITERAL(_), _, _)) + | Some((INT(_), _, _)) + | Some((BYTE(_), _, _)) + | Some((CHAR(_), _, _)) + | Some((DOUBLE(_), _, _)) + | Some((FLOAT(_), _, _)) + | Some((TRUE, _, _)) + | Some((FALSE, _, _)) => + match state.consume_any() { + Some(key) => push_child(children, Some("key"), key) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_QUESTION) { + match state.consume_any() { + Some(question) => push_child(children, Some("question"), question) + None => () + } + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + } + let pat = parse_pattern_until(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if pat.children_length() > 0 { + push_child(children, Some("pattern"), pat) + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_array_pattern_rest( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + parse_pattern_atom(state, stop_kinds) +} + +///| +fn parse_map_pattern_entry( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + parse_pattern_atom(state, stop_kinds) +} + +///| +fn parse_pattern_postfix( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut pat = parse_pattern_atom(state, stop_kinds) + let mut done = false + while !done { + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((QUESTION, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("pattern"), pat)] + match state.consume_any() { + Some(question) => push_child(children, Some("op"), question) + None => () + } + pat = node_from_children(Pattern(PatternKind::Binder), children) + } + Some((COLON, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("pattern"), pat)] + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let ty = parse_type_until(state, stop_kinds) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + pat = node_from_children(Pattern(PatternKind::Constraint), children) + } + _ => done = true + } + } + pat +} + +///| +fn parse_pattern_or( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut lhs = parse_pattern_range(state, stop_kinds) + let trivia = state.consume_trivia() + if state.current_is(TokenKind::TK_BAR) { + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + push_children(children, trivia) + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let rhs = parse_pattern_or(state, stop_kinds) + if rhs.children_length() > 0 { + push_child(children, Some("rhs"), rhs) + } + lhs = node_from_children(Pattern(PatternKind::Or), children) + } + lhs +} + +///| +fn parse_pattern_range( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut lhs = parse_pattern_postfix(state, stop_kinds) + match state.peek() { + Some((RANGE_EXCLUSIVE, _, _)) | Some((RANGE_INCLUSIVE, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let rhs = parse_pattern_postfix(state, stop_kinds) + if rhs.children_length() > 0 { + push_child(children, Some("rhs"), rhs) + } + lhs = node_from_children(Pattern(PatternKind::Range), children) + } + _ => () + } + lhs +} + +///| +fn parse_pattern_alias( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut lhs = parse_pattern_or(state, stop_kinds) + if state.current_is(TokenKind::TK_AS) { + let children : Array[(String?, CstNode)] = [(Some("pattern"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + lhs = node_from_children(Pattern(PatternKind::Alias), children) + } + lhs +} + +///| +fn parse_lex_pattern_until( + state : ParserState, + stop_kinds : Array[TokenKind], + allow_alias? : Bool = true, +) -> CstNode { + parse_lex_sequence_pattern(state, stop_kinds, allow_alias~) +} + +///| +fn lex_pattern_atom_can_start(tok : Token) -> Bool { + match tok { + REGEX_LITERAL(_) + | REGEX_INTERP(_) + | STRING(_) + | INTERP(_) + | UIDENT(_) + | PACKAGE_NAME(_) + | LPAREN => true + _ => false + } +} + +///| +fn parse_lex_sequence_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], + allow_alias? : Bool = true, +) -> CstNode { + let items : Array[(String?, CstNode)] = [] + let mut done = false + while !done { + push_children(items, state.consume_trivia()) + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((tok, _, _)) if lex_pattern_atom_can_start(tok) => { + let item = if allow_alias { + parse_lex_alias_pattern(state, stop_kinds) + } else { + parse_lex_atom_pattern(state, stop_kinds) + } + push_child(items, Some("item"), item) + push_children(items, state.consume_trivia()) + if state.current_is(TokenKind::TK_SEMI) { + match state.consume_any() { + Some(sep) => push_child(items, Some("sep"), sep) + None => () + } + } + } + Some((_, _, _)) => done = true + None => done = true + } + } + let body = node_from_children(List(ListKind::PatternList), items) + node_from_children(LexPattern(LexPatternKind::Sequence), [ + (Some("body"), body), + ]) +} + +///| +fn parse_lex_alias_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let atom = parse_lex_atom_pattern(state, stop_kinds) + if state.current_is(TokenKind::TK_AS) { + let children : Array[(String?, CstNode)] = [(Some("pattern"), atom)] + match state.consume_any() { + Some(as_token) => push_child(children, Some("kw"), as_token) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + Some((UIDENT(name), start, end)) => { + state.report( + mk_loc(start, end), + "Unexpected uppercase identifier \{name}, expected lowercase identifier in binder in `as` pattern.", + ) + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + } + _ => { + state.report_expected_tokens([ + TokenKind::TK_LIDENT, + TokenKind::TK_EXTEND, + ]) + push_child( + children, + Some("name"), + state.missing_token_node_silent(TokenKind::TK_LIDENT), + ) + } + } + node_from_children(LexPattern(LexPatternKind::Alias), children) + } else { + atom + } +} + +///| +fn parse_lex_atom_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LPAREN, _, _)) => { + match state.consume_any() { + Some(open) => push_child(children, Some("open"), open) + None => () + } + let inner = parse_lex_sequence_pattern(state, [TokenKind::TK_RPAREN]) + push_child( + children, + Some("body"), + inner.first_child_with_label("body").unwrap_or(inner), + ) + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RPAREN), + ) + node_from_children(LexPattern(LexPatternKind::Sequence), children) + } + Some((REGEX_INTERP(_), _, _)) | Some((INTERP(_), _, _)) => + parse_regex_interp_lex_pattern_node(state) + Some((REGEX_LITERAL(_), _, _)) | Some((STRING(_), _, _)) => + match state.consume_any() { + Some(token) => + node_from_children(LexPattern(LexPatternKind::Literal), [ + (token_label_from_node(token), token), + ]) + None => node_from_children(LexPattern(LexPatternKind::Literal), []) + } + Some((UIDENT(_), _, _)) | Some((PACKAGE_NAME(_), _, _)) => { + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + match state.peek() { + Some((DOT_UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + node_from_children(LexPattern(LexPatternKind::ConstantRef), children) + } + Some((tok, _, _)) if !kind_in(tok.kind(), stop_kinds) => + match state.consume_any() { + Some(token) => + node_from_children(LexPattern(LexPatternKind::Literal), [ + (token_label_from_node(token), token), + ]) + None => node_from_children(LexPattern(LexPatternKind::Literal), []) + } + _ => node_from_children(LexPattern(LexPatternKind::Literal), []) + } +} + +///| +fn parse_legacy_lex_literal_pattern_until( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut done = false + while !done { + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => + push_children(children, state.consume_trivia()) + Some((_, _, _)) => + match state.consume_any() { + Some(token) => + push_child(children, token_label_from_node(token), token) + None => done = true + } + None => done = true + } + } + node_from_children(LexPattern(LexPatternKind::Literal), children) +} + +///| +fn parse_regex_pattern_until( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + parse_regex_match_rhs(state, stop_kinds) +} + +///| +fn parse_regex_match_rhs( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let _ = state.consume_trivia() + if state.current_is(TokenKind::TK_LPAREN) { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(open) => push_child(children, Some("open"), open) + None => () + } + let _ = state.consume_trivia() + let pat = parse_regex_as_pattern(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) + push_child(children, Some("pattern"), pat) + let _ = state.consume_trivia() + if state.current_is(TokenKind::TK_COMMA) { + push_child( + children, + Some("bindings"), + parse_regex_match_binding_list(state), + ) + } + let _ = state.consume_trivia() + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RPAREN), + ) + node_from_children(RegexPattern(RegexPatternKind::Placeholder), children) + } else { + parse_regex_as_pattern(state, stop_kinds) + } +} + +///| +fn parse_regex_as_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let pat = parse_regex_or_pattern(state, stop_kinds) + if state.current_is(TokenKind::TK_AS) { + let children : Array[(String?, CstNode)] = [(Some("pattern"), pat)] + match state.consume_any() { + Some(as_token) => push_child(children, Some("kw"), as_token) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + node_from_children(RegexPattern(RegexPatternKind::Alias), children) + } else { + pat + } +} + +///| +fn parse_regex_or_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let _ = state.consume_trivia() + let mut lhs = parse_regex_sequence_pattern(state, stop_kinds) + let _ = state.consume_trivia() + while state.current_is(TokenKind::TK_BAR) { + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + push_child( + children, + Some("rhs"), + parse_regex_sequence_pattern(state, stop_kinds), + ) + lhs = node_from_children( + RegexPattern(RegexPatternKind::Alternation), + children, + ) + let _ = state.consume_trivia() + } + lhs +} + +///| +fn parse_regex_sequence_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let _ = state.consume_trivia() + let mut lhs = parse_regex_atom_pattern(state, stop_kinds) + let _ = state.consume_trivia() + while state.current_is(TokenKind::TK_PLUS) { + let children : Array[(String?, CstNode)] = [(Some("lhs"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + push_child( + children, + Some("rhs"), + parse_regex_atom_pattern(state, stop_kinds), + ) + lhs = node_from_children(RegexPattern(RegexPatternKind::Sequence), children) + let _ = state.consume_trivia() + } + lhs +} + +///| +fn parse_regex_atom_pattern( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let _ = state.consume_trivia() + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LPAREN, _, _)) => { + match state.consume_any() { + Some(open) => push_child(children, Some("open"), open) + None => () + } + let _ = state.consume_trivia() + let pat = parse_regex_as_pattern(state, [TokenKind::TK_RPAREN]) + push_child(children, Some("pattern"), pat) + let _ = state.consume_trivia() + push_child( + children, + Some("close"), + state.expect_token(TokenKind::TK_RPAREN), + ) + node_from_children(RegexPattern(RegexPatternKind::Placeholder), children) + } + Some((REGEX_LITERAL(_), _, _)) => + match state.consume_any() { + Some(token) => + node_from_children(RegexPattern(RegexPatternKind::Literal), [ + (token_label_from_node(token), token), + ]) + None => node_from_children(RegexPattern(RegexPatternKind::Literal), []) + } + Some((UIDENT(_), _, _)) | Some((PACKAGE_NAME(_), _, _)) => { + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + match state.peek() { + Some((DOT_UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + node_from_children(RegexPattern(RegexPatternKind::Reference), children) + } + Some((tok, _, _)) if !kind_in(tok.kind(), stop_kinds) => + match state.consume_any() { + Some(token) => + node_from_children(RegexPattern(RegexPatternKind::Literal), [ + (token_label_from_node(token), token), + ]) + None => node_from_children(RegexPattern(RegexPatternKind::Literal), []) + } + _ => node_from_children(RegexPattern(RegexPatternKind::Literal), []) + } +} + +///| +fn parse_regex_match_binding_list(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + while state.current_is(TokenKind::TK_COMMA) { + match state.consume_any() { + Some(sep) => push_child(children, Some("sep"), sep) + None => () + } + push_children(children, state.consume_trivia()) + if state.current_is(TokenKind::TK_RPAREN) || + state.current_is(TokenKind::TK_EOF) { + break + } + push_child(children, Some("item"), parse_regex_match_binding(state)) + push_children(children, state.consume_trivia()) + } + node_from_children(List(ListKind::PatternList), children) +} + +///| +fn parse_regex_match_binding(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((POST_LABEL(_), _, _)) => + match state.consume_any() { + Some(label) => push_child(children, Some("name"), label) + None => () + } + Some((LIDENT(_), _, _)) | Some((EXTEND, _, _)) => { + match state.consume_any() { + Some(label) => push_child(children, Some("name"), label) + None => () + } + if state.current_is(TokenKind::TK_EQUAL) { + match state.consume_any() { + Some(equal) => push_child(children, Some("op"), equal) + None => () + } + match state.peek() { + Some((LIDENT(_), _, _)) + | Some((UIDENT(_), _, _)) + | Some((EXTEND, _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("binder"), name) + None => () + } + _ => () + } + } + } + _ => () + } + node_from_children(Argument(ArgumentKind::Labelled), children) +} diff --git a/untyped_cst/parse_string_literal.mbt b/untyped_cst/parse_string_literal.mbt new file mode 100644 index 00000000..aa73fff2 --- /dev/null +++ b/untyped_cst/parse_string_literal.mbt @@ -0,0 +1,200 @@ +///| +fn loc_span(state : ParserState, loc : Location) -> SourceSpan { + mk_span( + state.offset_for_position(loc.start), + state.offset_for_position(loc.end), + ) +} + +///| +fn parse_string_literal_node(state : ParserState) -> CstNode { + match state.peek() { + Some((MULTILINE_STRING(_), _, _)) | Some((MULTILINE_INTERP(_), _, _)) => + parse_multiline_string_literal(state) + Some((tok, _, _)) => + match state.consume_any() { + Some(token_node) => { + let children : Array[(String?, CstNode)] = [ + (Some("token"), token_node), + ] + push_interp_segments_for_token(state, tok, children) + node( + Expr(ExprKind::Literal), + token_node.loc(), + token_node.source_span(), + children, + ) + } + None => node_from_children(Expr(ExprKind::MissingExpr), []) + } + None => node_from_children(Expr(ExprKind::MissingExpr), []) + } +} + +///| +fn parse_multiline_string_literal(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut done = false + while !done { + match state.peek() { + Some((tok, _, _)) => + match tok { + MULTILINE_STRING(_) | MULTILINE_INTERP(_) => + match state.consume_any() { + Some(token_node) => { + push_child(children, Some("token"), token_node) + push_interp_segments_for_token(state, tok, children) + if state.next_significant_is_multiline_string() { + push_children(children, state.consume_trivia()) + } + } + None => done = true + } + _ => done = true + } + None => done = true + } + } + node_from_children(Expr(ExprKind::MultilineString), children) +} + +///| +fn ParserState::next_significant_is_multiline_string( + self : ParserState, +) -> Bool { + match self.peek_significant() { + Some((MULTILINE_STRING(_), _, _)) | Some((MULTILINE_INTERP(_), _, _)) => + true + _ => false + } +} + +///| +fn parse_regex_interp_lex_pattern_node(state : ParserState) -> CstNode { + match state.peek() { + Some((tok, _, _)) => + match tok { + REGEX_INTERP(_) | INTERP(_) => + match state.consume_any() { + Some(token_node) => { + let children : Array[(String?, CstNode)] = [ + (Some("token"), token_node), + ] + push_interp_segments_for_token(state, tok, children) + node( + LexPattern(LexPatternKind::RegexInterp), + token_node.loc(), + token_node.source_span(), + children, + ) + } + None => + node_from_children(LexPattern(LexPatternKind::RegexInterp), []) + } + _ => node_from_children(LexPattern(LexPatternKind::RegexInterp), []) + } + _ => node_from_children(LexPattern(LexPatternKind::RegexInterp), []) + } +} + +///| +fn push_interp_segments_for_token( + state : ParserState, + tok : Token, + children : Array[(String?, CstNode)], +) -> Unit { + match tok { + INTERP(elems) + | BYTES_INTERP(elems) + | MULTILINE_INTERP(elems) + | REGEX_INTERP(elems) => + for elem in elems { + push_child(children, Some("segment"), parse_interp_segment(state, elem)) + } + _ => () + } +} + +///| +fn parse_interp_segment( + state : ParserState, + elem : @tokens.InterpElem, +) -> CstNode { + match elem { + InterpLit(repr~, loc~) => + node( + InterpSegment(InterpSegmentKind::InterpLiteralSegment), + loc, + loc_span(state, loc), + [ + ( + Some("repr"), + node(Token(STRING(repr)), loc, loc_span(state, loc), []), + ), + ], + ) + InterpSource(interp_source) => + parse_interp_source_segment(state, interp_source) + } +} + +///| +fn parse_interp_source_segment( + state : ParserState, + interp_source : InterpSource, +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let expr = parse_interp_source_expr(state, interp_source) + if expr.children_length() > 0 { + push_child(children, Some("expr"), expr) + } + push_child( + children, + Some("source"), + node( + Token(STRING(interp_source.source)), + interp_source.loc, + loc_span(state, interp_source.loc), + [], + ), + ) + node( + InterpSegment(InterpSegmentKind::InterpSourceSegment), + interp_source.loc, + loc_span(state, interp_source.loc), + children, + ) +} + +///| +fn parse_interp_source_expr( + state : ParserState, + interp_source : InterpSource, +) -> CstNode { + let lex = match state.location_mode { + DefaultLocation => + @lexer.tokens_from_string( + interp_source.source, + start_pos=interp_source.loc.start, + is_interpolation=true, + comment=true, + ) + Utf16Location | SourceOffsetLocation => + @lexer.tokens_from_string_with_utf16_location( + interp_source.source, + start_pos=interp_source.loc.start, + is_interpolation=true, + comment=true, + ) + } + for report in reports_from_lex_errors(lex.errors) { + state.reports.push(report) + } + let inner_state = new_state( + lex.tokens, + state.source, + state.location_mode, + state.reports, + ) + parse_expr_until(inner_state, [TokenKind::TK_EOF]) +} diff --git a/untyped_cst/parse_test.mbt b/untyped_cst/parse_test.mbt new file mode 100644 index 00000000..67507b77 --- /dev/null +++ b/untyped_cst/parse_test.mbt @@ -0,0 +1,962 @@ +///| +fn root_is_source_file(root : @untyped_cst.CstNode) -> Bool { + match root.kind() { + @untyped_cst.SourceFile => true + _ => false + } +} + +///| +fn root_is_expression_fragment(root : @untyped_cst.CstNode) -> Bool { + match root.kind() { + @untyped_cst.Fragment(@untyped_cst.Expression) => true + _ => false + } +} + +///| +fn is_top_item(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Impl(_) | @untyped_cst.Decl(_) => true + _ => false + } +} + +///| +fn is_expr(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Stmt(_) | @untyped_cst.Expr(_) => true + _ => false + } +} + +///| +fn has_balanced_decl_shape(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Decl(@untyped_cst.BalancedDecl) + | @untyped_cst.Decl(_) + | @untyped_cst.Impl(_) => true + _ => false + } +} + +///| +fn count_root_label(root : @untyped_cst.CstNode, label : String) -> Int { + let mut count = 0 + for i in 0.. count += 1 + _ => () + } + } + count +} + +///| +fn count_root_items(root : @untyped_cst.CstNode) -> Int { + let mut count = 0 + for i in 0.. count += 1 + _ => () + } + } + count +} + +///| +fn root_contains_expr(root : @untyped_cst.CstNode) -> Bool { + for i in 0.. return true + _ => () + } + } + false +} + +///| +fn find_node( + root : @untyped_cst.CstNode, + pred : (@untyped_cst.CstNode) -> Bool, +) -> @untyped_cst.CstNode? { + if pred(root) { + return Some(root) + } + for i in 0.. + match find_node(child, pred) { + Some(found) => return Some(found) + None => () + } + None => () + } + } + None +} + +///| +fn is_argument_list(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.List(@untyped_cst.ArgumentList) => true + _ => false + } +} + +///| +fn is_statement_list(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.List(@untyped_cst.StatementList) => true + _ => false + } +} + +///| +fn has_missing_close(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Missing(@tokens.TokenKind::TK_RPAREN) => true + _ => false + } +} + +///| +fn is_type_node(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Type(_) => true + _ => false + } +} + +///| +fn is_pattern_node(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Pattern(_) => true + _ => false + } +} + +///| +fn is_parameter_clause(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Clause(@untyped_cst.Parameter) => true + _ => false + } +} + +///| +fn is_error_node(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Error => true + _ => false + } +} + +///| +fn is_multiline_string_expr(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Expr(@untyped_cst.MultilineString) => true + _ => false + } +} + +///| +fn is_interp_literal_segment(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.InterpSegment(@untyped_cst.InterpLiteralSegment) => true + _ => false + } +} + +///| +fn is_interp_source_segment(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.InterpSegment(@untyped_cst.InterpSourceSegment) => true + _ => false + } +} + +///| +fn is_regex_interp_lex_pattern(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.LexPattern(@untyped_cst.RegexInterp) => true + _ => false + } +} + +///| +fn is_interp_source_segment_with_expr(node : @untyped_cst.CstNode) -> Bool { + is_interp_source_segment(node) && find_node(node, is_expr) is Some(_) +} + +///| +fn is_error_annotation(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Clause(@untyped_cst.ErrorAnnotation) => true + _ => false + } +} + +///| +fn is_return_type_clause(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Clause(@untyped_cst.ReturnType) => true + _ => false + } +} + +///| +fn has_node_text( + root : @untyped_cst.CstNode, + source : String, + pred : (@untyped_cst.CstNode) -> Bool, + text : String, +) -> Bool { + match find_node(root, fn(node) { pred(node) && node.text(source) == text }) { + Some(_) => true + None => false + } +} + +///| +fn count_label(node : @untyped_cst.CstNode, label : String) -> Int { + let mut count = 0 + for i in 0.. count += 1 + _ => () + } + } + count +} + +///| +fn has_label_recursive(node : @untyped_cst.CstNode, label : String) -> Bool { + for i in 0.. return true + Some((_, child)) if has_label_recursive(child, label) => return true + _ => () + } + } + false +} + +///| +fn has_decl_kind( + root : @untyped_cst.CstNode, + kind : @untyped_cst.DeclKind, +) -> Bool { + find_node(root, fn(node) { + match node.kind() { + @untyped_cst.Decl(found) => found == kind + _ => false + } + }) + is Some(_) +} + +///| +fn has_type_kind(root : @untyped_cst.CstNode, expected : String) -> Bool { + find_node(root, fn(node) { + match node.kind() { + @untyped_cst.Type(@untyped_cst.Name) => expected == "Name" + @untyped_cst.Type(@untyped_cst.Tuple) => expected == "Tuple" + @untyped_cst.Type(@untyped_cst.Function) => expected == "Function" + @untyped_cst.Type(@untyped_cst.Application) => expected == "Application" + @untyped_cst.Type(@untyped_cst.Option) => expected == "Option" + @untyped_cst.Type(@untyped_cst.Object) => expected == "Object" + @untyped_cst.Type(@untyped_cst.Group) => expected == "Group" + _ => false + } + }) + is Some(_) +} + +///| +fn has_impl_kind( + root : @untyped_cst.CstNode, + kind : @untyped_cst.ImplKind, +) -> Bool { + find_node(root, fn(node) { + match node.kind() { + @untyped_cst.Impl(found) => found == kind + _ => false + } + }) + is Some(_) +} + +///| +fn is_placeholder_node(node : @untyped_cst.CstNode) -> Bool { + match node.kind() { + @untyped_cst.Expr(@untyped_cst.Placeholder) + | @untyped_cst.Pattern(@untyped_cst.Placeholder) + | @untyped_cst.Type(@untyped_cst.Placeholder) + | @untyped_cst.LexPattern(@untyped_cst.Placeholder) + | @untyped_cst.RegexPattern(@untyped_cst.Placeholder) => true + _ => false + } +} + +///| +fn has_no_placeholder_nodes(root : @untyped_cst.CstNode) -> Bool { + find_node(root, is_placeholder_node) is None +} + +///| +test "empty structure returns source file" { + let source = "" + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(root_is_source_file(result.root()), true) + @test.assert_eq(result.root().text(source), source) +} + +///| +test "let structure returns one top-level item" { + let source = "let x = 1" + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(root_is_source_file(result.root()), true) + @test.assert_eq(count_root_items(result.root()), 1) +} + +///| +test "top-level separators are preserved without empty trailing item" { + let explicit = @untyped_cst.parse_string( + "let x = 1; let y = 2", + entry=Structure, + ) + @test.assert_eq(count_root_items(explicit.root()), 2) + @test.assert_eq(count_root_label(explicit.root(), "sep"), 1) + let asi = @untyped_cst.parse_string("let x = 1\nlet y = 2", entry=Structure) + @test.assert_eq(count_root_items(asi.root()), 2) + @test.assert_eq(count_root_label(asi.root(), "sep"), 1) + let trailing = @untyped_cst.parse_string("let x = 1;", entry=Structure) + @test.assert_eq(count_root_items(trailing.root()), 1) + @test.assert_eq(count_root_label(trailing.root(), "sep"), 1) +} + +///| +test "expression entry returns expression fragment" { + let source = "1 + 2" + let result = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(root_is_expression_fragment(result.root()), true) + @test.assert_eq(root_contains_expr(result.root()), true) + @test.assert_eq(result.root().text(source), source) +} + +///| +test "successful expressions are grammar shaped without placeholders" { + let cases = [ + "a + b * c", "object.method(1).field[0]", "if ok { yes } else { no }", "match value { Some(x) => x; _ => y }", + "{ let x = 1; x }", + ] + for source in cases { + let result = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(has_no_placeholder_nodes(result.root()), true) + } +} + +///| +test "grammar shaped expression labels are semantic not positional" { + let infix = @untyped_cst.parse_string("a + b * c", entry=Expression) + @test.assert_eq(infix.diagnostics_view().length(), 0) + @test.assert_eq(has_label_recursive(infix.root(), "lhs"), true) + @test.assert_eq(has_label_recursive(infix.root(), "rhs"), true) + @test.assert_eq(has_label_recursive(infix.root(), "op"), true) + let control = @untyped_cst.parse_string( + "if ok { yes } else { no }", + entry=Expression, + ) + @test.assert_eq(control.diagnostics_view().length(), 0) + @test.assert_eq(has_label_recursive(control.root(), "cond"), true) + @test.assert_eq(has_label_recursive(control.root(), "body"), true) + @test.assert_eq(has_label_recursive(control.root(), "else"), true) + let call = @untyped_cst.parse_string( + "object.method(1).field[0]", + entry=Expression, + ) + @test.assert_eq(call.diagnostics_view().length(), 0) + @test.assert_eq(has_label_recursive(call.root(), "self"), true) + @test.assert_eq(has_label_recursive(call.root(), "arg"), true) + @test.assert_eq(has_label_recursive(call.root(), "field"), true) + @test.assert_eq(has_label_recursive(call.root(), "index"), true) +} + +///| +test "expression entries return expression fragment" { + let source = "1 + 2" + let bare = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(root_is_expression_fragment(bare.root()), true) +} + +///| +test "argument list preserves open items separators and close" { + let source = "f(1, 2,)" + let result = @untyped_cst.parse_string(source, entry=Expression) + guard find_node(result.root(), is_argument_list) is Some(args) else { + fail("expected argument list") + } + @test.assert_eq(count_label(args, "open"), 1) + @test.assert_eq(count_label(args, "item"), 2) + @test.assert_eq(count_label(args, "sep"), 2) + @test.assert_eq(count_label(args, "close"), 1) + @test.assert_eq(args.text(source), "(1, 2,)") +} + +///| +test "block statement list preserves braces and semicolon" { + let source = "{ let x = 1; x }" + let result = @untyped_cst.parse_string(source, entry=Expression) + guard find_node(result.root(), is_statement_list) is Some(block) else { + fail("expected statement list") + } + @test.assert_eq(count_label(block, "open"), 1) + @test.assert_eq(count_label(block, "sep"), 1) + @test.assert_eq(count_label(block, "close"), 1) + @test.assert_eq(block.text(source), source) +} + +///| +test "missing list close inserts missing node and diagnostic" { + let source = "f(1, 2" + let result = @untyped_cst.parse_string(source, entry=Expression) + guard find_node(result.root(), is_argument_list) is Some(args) else { + fail("expected argument list") + } + @test.assert_eq(find_node(args, has_missing_close) is Some(_), true) + @test.assert_eq(result.diagnostics_view().length(), 1) +} + +///| +test "top-level starts have balanced exterior nodes" { + let cases = [ + "fn f() {}", "async fn f() {}", "let x = 1", "const X = 1", "type T = Int", "struct S {}", + "enum E { A }", "suberror E { A }", "extenum E { A }", "enumview V { C } for A with view(x) { C }", + "trait T {}", "impl T with f(self) { self }", "test \"t\" {}", "using @pkg {func as func2}", + "extern \"js\" fn f() = \"f\"", "declare fn f()", "pub fn f() {}", "priv type T = Int", + "#deprecated\nfn f() {}", + ] + for source in cases { + let result = @untyped_cst.parse_string(source, entry=Structure) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + @test.assert_eq( + find_node(result.root(), has_balanced_decl_shape) is Some(_), + true, + ) + @test.assert_eq(count_root_items(result.root()), 1) + } +} + +///| +test "balanced declaration body keeps nested semicolon inside item" { + let source = "fn f() { let x = 1; x }\nlet y = 2" + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(count_root_label(result.root(), "item"), 2) + @test.assert_eq(count_root_items(result.root()), 2) + @test.assert_eq(count_root_label(result.root(), "sep"), 1) +} + +///| +test "top-level splitter ignores fn tokens inside let bodies" { + let source = "let f = fn(x) { let z = 1; z }\nlet y = 1" + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(count_root_label(result.root(), "item"), 2) + @test.assert_eq(count_root_items(result.root()), 2) + @test.assert_eq(count_root_label(result.root(), "sep"), 1) +} + +///| +test "type grammar cases produce type nodes" { + let cases = [ + ("fn f(x : Int) {}", "Int"), + ("fn f(x : @pkg.T[A]) {}", "@pkg.T[A]"), + ("fn f(x : (Int, String)) {}", "(Int, String)"), + ("fn f(x : (Int) -> String) {}", "(Int) -> String"), + ("fn f(x : Int?) {}", "Int?"), + ("fn f(x : Self) {}", "Self"), + ("fn f(x : _) {}", "_"), + ("fn f(x : &Name) {}", "&Name"), + ("type T = Array[Int]", "Array[Int]"), + ] + for case in cases { + let (source, expected_type) = case + let result = @untyped_cst.parse_string(source, entry=Structure) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq( + has_node_text(result.root(), source, is_type_node, expected_type), + true, + ) + } +} + +///| +test "successful type grammar cases are shape-specific without placeholders" { + let cases = [ + ("fn f(x : Int) {}", "Name"), + ("fn f(x : Array[Int]) {}", "Application"), + ("fn f(x : A?) {}", "Option"), + ("fn f(x : (A, B)) {}", "Tuple"), + ("fn f(x : (A) -> B) {}", "Function"), + ("fn f(x : &T) {}", "Object"), + ] + for case in cases { + let (source, expected_kind) = case + let result = @untyped_cst.parse_string(source, entry=Structure) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(has_type_kind(result.root(), expected_kind), true) + @test.assert_eq(has_no_placeholder_nodes(result.root()), true) + } +} + +///| +test "return type and error annotation cases" { + let cases = [ + ("fn f() -> T raise E { x }", "T"), + ("fn f() -> T noraise { x }", "T"), + ("fn f() -> T raise ? { x }", "T"), + ( + "fn f() -> async () -> async () -> Int { x }", "async () -> async () -> Int", + ), + ("fn f() -> &Name { x }", "&Name"), + ("fn f() -> @pkg.T[A] { x }", "@pkg.T[A]"), + ("fn[N : Name] f(x : Int) -> T { x }", "Name"), + ] + for case in cases { + let (source, expected_type) = case + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq( + has_node_text(result.root(), source, is_type_node, expected_type), + true, + ) + } + let with_raise = @untyped_cst.parse_string( + "fn f() -> T raise E { x }", + entry=Structure, + ) + @test.assert_eq( + find_node(with_raise.root(), is_return_type_clause) is Some(_), + true, + ) + @test.assert_eq( + find_node(with_raise.root(), is_error_annotation) is Some(_), + true, + ) + let with_noraise = @untyped_cst.parse_string( + "fn f() -> T noraise { x }", + entry=Structure, + ) + @test.assert_eq( + find_node(with_noraise.root(), is_return_type_clause) is Some(_), + true, + ) + @test.assert_eq( + find_node(with_noraise.root(), is_error_annotation) is Some(_), + true, + ) +} + +///| +test "pattern grammar cases produce pattern nodes" { + let cases = [ + ("fn f() { let x = value; () }", "x"), + ("fn f() { let _ = value; () }", "_"), + ("fn f() { let 1 = value; () }", "1"), + ("fn f() { let (a, _) = value; () }", "(a, _)"), + ("fn f() { let [a, .. rest,] = value; () }", "[a, .. rest,]"), + ("fn f() { let Some(x) = value; () }", "Some(x)"), + ("fn f() { let { x: y, z, .. } = value; () }", "{ x: y, z, .. }"), + ("fn f() { let { \"x\"? : v, .. } = value; () }", "{ \"x\"? : v, .. }"), + ("fn f() { let A | B = value; () }", "A | B"), + ("fn f() { let A as a = value; () }", "A as a"), + ("fn f() { let (x : Int) = value; () }", "(x : Int)"), + ("fn f() { let 1..<3 = value; () }", "1..<3"), + ("fn f() { let W::A(bv, d) = value; () }", "W::A(bv, d)"), + ("fn f() { let @pkg.Constr = value; () }", "@pkg.Constr"), + ("fn f() { let Ty::Constr = value; () }", "Ty::Constr"), + ("fn f() { let Ty::@pkg.Constr = value; () }", "Ty::@pkg.Constr"), + ("fn f() { let C(x=pat, y~, ..) = value; () }", "C(x=pat, y~, ..)"), + ("fn f() { let ['c', .. b,] = value; () }", "['c', .. b,]"), + ("fn f() { let [.. as rest] = value; () }", "[.. as rest]"), + ("fn f() { let [.. _] = value; () }", "[.. _]"), + ("fn f() { let [.. \"lit\"] = value; () }", "[.. \"lit\"]"), + ("fn f() { let [.. b\"lit\"] = value; () }", "[.. b\"lit\"]"), + ("fn f() { let i6(x) = value; () }", "i6(x)"), + ] + for case in cases { + let (source, expected_pattern) = case + let result = @untyped_cst.parse_string(source, entry=Structure) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq( + has_node_text(result.root(), source, is_pattern_node, expected_pattern), + true, + ) + } +} + +///| +test "match expression cases produce pattern nodes" { + let source = "match value { Some(x) => x; _ => y }" + let result = @untyped_cst.parse_string(source, entry=Expression) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq( + has_node_text(result.root(), source, is_pattern_node, "Some(x)"), + true, + ) + @test.assert_eq( + has_node_text(result.root(), source, is_pattern_node, "_"), + true, + ) +} + +///| +test "function parameters reject destructuring patterns" { + let source = "fn f((a, _) : Pair, [x, .. rest] : Array[Int]) { x }" + assert_structure_acceptance( + "destructuring function parameters", source, false, + ) +} + +///| +test "local function optional parameters parse without diagnostics" { + let local_fn_source = + #|fn init { + #| fn question(a? : Int, b? : Int) -> Int? { Some(0) } + #| question() + #|} + let lambda_source = + #|fn init { + #| let question = fn(a? : Int, b? : Int) { Some(0) } + #| question() + #|} + let sources = [local_fn_source, lambda_source] + for source in sources { + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq( + has_node_text(result.root(), source, is_parameter_clause, "a? : Int"), + true, + ) + @test.assert_eq( + has_node_text(result.root(), source, is_parameter_clause, "b? : Int"), + true, + ) + } +} + +///| +test "parenthesized expression type annotation accepts optional type marker" { + let source = + #|fn init { + #| let g0 = (None : Int?) + #| () + #|} + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq( + has_node_text(result.root(), source, is_type_node, "Int?"), + true, + ) +} + +///| +test "optional argument forwarding in call list parses without diagnostics" { + let sources = [ + "f(label~)", "f(label?, other?=value)", "extend(extend?, extend?=value)", + ] + let argument_texts = [ + "(label~)", "(label?, other?=value)", "(extend?, extend?=value)", + ] + for i, source in sources { + let result = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq( + has_node_text(result.root(), source, is_argument_list, argument_texts[i]), + true, + ) + } +} + +///| +test "expression cases preserve source text" { + let cases = [ + "42", "name", "f(1, 2)", "object.method(1)", "object.field", "array[0]", "(a, b)", + "[1, 2]", "{ \"x\": 1 }", "T::{ x: 1 }", "x => x + 1", "if ok { yes } else { no }", + "match value { Some(x) => x; _ => y }", "try f() catch { _ => g }", "try? f()", + "try! f()", "raise err", "return value", "break value", "continue", "while ok { work() } nobreak { done }", + "for i in xs { i }", "defer cleanup()", "guard ok else { return }", "a |> b + c", + "a <| b + c", "-x", "a + b * c", "x = y", "x += y", "0.. 1 }", "lexscan s { re\"a\" => 1 }", + "s =~ re\"a\"", "∀ i : Int, i >= 0", "∃ i : Int, i >= 0", "a → b", "letrec f = fn () { 0 }", + "proof_assert a == b", "proof_let witness = mk_proof()", + ] + for source in cases { + assert_expression_acceptance(source, source, true) + let result = @untyped_cst.parse_string(source, entry=Expression) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + if !root_contains_expr(result.root()) { + fail(source) + } + } +} + +///| +test "unsupported expression tokens produce recovery diagnostics" { + let cases = [ + "loop 0 { break }", "throw err", "s lexmatch? \"a\"", "s lexmatch? \"a\" with longest", + "pub", "else", "catch", "with", + ] + for source in cases { + assert_expression_acceptance(source, source, false) + let result = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq(find_node(result.root(), is_error_node) is Some(_), true) + } + let structure_cases = [ + "fn f() { pub }", "fn f() { else }", "fn f() { catch }", "fn f() { with }", + ] + for source in structure_cases { + assert_structure_acceptance(source, source, false) + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(result.diagnostics_view().length() > 0, true) + @test.assert_eq(find_node(result.root(), is_error_node) is Some(_), true) + } + let where_source = "while ok { step() } where { proof_assert ok }" + assert_expression_acceptance(where_source, where_source, false) + let where_result = @untyped_cst.parse_string(where_source, entry=Expression) + @test.assert_eq(where_result.root().text(where_source), where_source) + @test.assert_eq(where_result.diagnostics_view().length() > 0, true) +} + +///| +test "string literal cases preserve source text" { + let cases = ["\"hello\"", "b\"hello\"", "re\"a+\""] + for source in cases { + let result = @untyped_cst.parse_string(source, entry=Expression) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(root_contains_expr(result.root()), true) + } +} + +///| +test "interpolation segments include parsed expression children" { + let cases = [ + "\"hello \\{name}\"", "b\"hello \\{name}\"", "$| hello \\{name}\n#| tail", + ] + for source in cases { + let result = @untyped_cst.parse_string(source, entry=Expression) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + @test.assert_eq( + find_node(result.root(), is_interp_literal_segment) is Some(_), + true, + ) + @test.assert_eq( + find_node(result.root(), is_interp_source_segment_with_expr) is Some(_), + true, + ) + } +} + +///| +test "consecutive multiline string tokens form one multiline expression" { + let source = "#| Hello\n#| World" + let result = @untyped_cst.parse_string(source, entry=Expression) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + @test.assert_eq( + find_node(result.root(), is_multiline_string_expr) is Some(_), + true, + ) +} + +///| +test "regex interpolation is represented as lex pattern segment" { + let source = "lexmatch s { re\"a\\{x}\" => 1 }" + let result = @untyped_cst.parse_string(source, entry=Expression) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + @test.assert_eq( + find_node(result.root(), is_regex_interp_lex_pattern) is Some(_), + true, + ) + @test.assert_eq( + find_node(result.root(), is_interp_source_segment_with_expr) is Some(_), + true, + ) +} + +///| +test "top-level declarations expose semantic children" { + let fn_source = "fn f(x : Int) -> Int { x }" + let fn_result = @untyped_cst.parse_string(fn_source, entry=Structure) + @test.assert_eq(fn_result.diagnostics_view().length(), 0) + @test.assert_eq(fn_result.root().text(fn_source), fn_source) + @test.assert_eq(has_decl_kind(fn_result.root(), @untyped_cst.Function), true) + @test.assert_eq(has_label_recursive(fn_result.root(), "name"), true) + @test.assert_eq(has_label_recursive(fn_result.root(), "type"), true) + @test.assert_eq(has_label_recursive(fn_result.root(), "clause"), true) + @test.assert_eq(has_label_recursive(fn_result.root(), "body"), true) + let let_source = "let x = 1" + let let_result = @untyped_cst.parse_string(let_source, entry=Structure) + @test.assert_eq(let_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(let_result.root(), @untyped_cst.Let), true) + @test.assert_eq(has_label_recursive(let_result.root(), "pattern"), true) + let const_source = "const X = 1" + let const_result = @untyped_cst.parse_string(const_source, entry=Structure) + @test.assert_eq(const_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(const_result.root(), @untyped_cst.Const), true) + @test.assert_eq(has_label_recursive(const_result.root(), "name"), true) + let type_source = "type T = Array[Int]" + let type_result = @untyped_cst.parse_string(type_source, entry=Structure) + @test.assert_eq(type_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(type_result.root(), @untyped_cst.Type), true) + @test.assert_eq(has_label_recursive(type_result.root(), "name"), true) + @test.assert_eq(has_label_recursive(type_result.root(), "type"), true) + let struct_source = "struct S { x : Int }" + let struct_result = @untyped_cst.parse_string(struct_source, entry=Structure) + @test.assert_eq(struct_result.diagnostics_view().length(), 0) + @test.assert_eq( + has_decl_kind(struct_result.root(), @untyped_cst.Struct), + true, + ) + @test.assert_eq(has_label_recursive(struct_result.root(), "name"), true) + @test.assert_eq(has_label_recursive(struct_result.root(), "body"), true) + let enum_source = "enum E { A }" + let enum_result = @untyped_cst.parse_string(enum_source, entry=Structure) + @test.assert_eq(enum_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(enum_result.root(), @untyped_cst.Enum), true) + @test.assert_eq(has_label_recursive(enum_result.root(), "name"), true) + @test.assert_eq(has_label_recursive(enum_result.root(), "body"), true) + let trait_source = "trait T { f(Self) -> Unit }" + let trait_result = @untyped_cst.parse_string(trait_source, entry=Structure) + @test.assert_eq(trait_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(trait_result.root(), @untyped_cst.Trait), true) + @test.assert_eq(has_label_recursive(trait_result.root(), "name"), true) + @test.assert_eq(has_label_recursive(trait_result.root(), "body"), true) + let impl_source = "impl T with f(self) { self }" + let impl_result = @untyped_cst.parse_string(impl_source, entry=Structure) + @test.assert_eq(impl_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(impl_result.root(), @untyped_cst.Impl), true) + @test.assert_eq(has_label_recursive(impl_result.root(), "name"), true) + @test.assert_eq(has_label_recursive(impl_result.root(), "body"), true) + let test_source = "test { 1 }" + let test_result = @untyped_cst.parse_string(test_source, entry=Structure) + @test.assert_eq(test_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(test_result.root(), @untyped_cst.Test), true) + @test.assert_eq(has_label_recursive(test_result.root(), "body"), true) + let using_source = "using @pkg {}" + let using_result = @untyped_cst.parse_string(using_source, entry=Structure) + @test.assert_eq(using_result.diagnostics_view().length(), 0) + @test.assert_eq(has_impl_kind(using_result.root(), @untyped_cst.Using), true) + let extern_source = "extern \"js\" fn f() = \"f\"" + let extern_result = @untyped_cst.parse_string(extern_source, entry=Structure) + @test.assert_eq(extern_result.diagnostics_view().length(), 0) + @test.assert_eq( + has_decl_kind(extern_result.root(), @untyped_cst.Function), + true, + ) + @test.assert_eq(has_label_recursive(extern_result.root(), "name"), true) + @test.assert_eq(has_label_recursive(extern_result.root(), "language"), true) + @test.assert_eq(has_label_recursive(extern_result.root(), "stub"), true) +} + +///| +test "successful top-level declarations are shaped without placeholders" { + let cases = [ + "fn f(x : Int) -> Int { x }", "let x : Int = 1", "const X : Int = 1", "type T = Array[Int]", + "struct S { x : Int }", "enum E { A(Int) }", "trait T { f(Self) -> Unit }", "impl T with f(self) { self }", + "test { inspect(1, content=\"1\") }", + ] + for source in cases { + let result = @untyped_cst.parse_string(source, entry=Structure) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(result.root().text(source), source) + @test.assert_eq(has_no_placeholder_nodes(result.root()), true) + } +} + +///| +test "prefixed declarations expose resolved declaration kind" { + let fn_cases = ["pub fn f() {}", "declare fn f()", "#deprecated\nfn f() {}"] + for source in fn_cases { + let result = @untyped_cst.parse_string(source, entry=Structure) + if result.diagnostics_view().length() != 0 { + fail(source) + } + @test.assert_eq(has_decl_kind(result.root(), @untyped_cst.Function), true) + @test.assert_eq( + has_decl_kind(result.root(), @untyped_cst.BalancedDecl), + false, + ) + } + let type_source = "priv type T = Int" + let type_result = @untyped_cst.parse_string(type_source, entry=Structure) + @test.assert_eq(type_result.diagnostics_view().length(), 0) + @test.assert_eq(has_decl_kind(type_result.root(), @untyped_cst.Type), true) + @test.assert_eq( + has_decl_kind(type_result.root(), @untyped_cst.BalancedDecl), + false, + ) +} + +///| +test "optional type markers in declaration bodies do not produce recovery diagnostics" { + let source = + #|struct S { + #| x : String? + #|} + #| + #|fn f(x : String??) -> String? { x } + let result = @untyped_cst.parse_string(source, entry=Structure) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) +} + +///| +test "optional type markers in block let annotations do not produce recovery diagnostics" { + let source = "{ let opt : _? = Some(42); opt }" + let result = @untyped_cst.parse_string(source, entry=Expression) + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(result.root().text(source), source) +} diff --git a/untyped_cst/parse_top.mbt b/untyped_cst/parse_top.mbt new file mode 100644 index 00000000..a95d5b45 --- /dev/null +++ b/untyped_cst/parse_top.mbt @@ -0,0 +1,137 @@ +///| +fn token_label(tok : Token) -> String? { + match tok { + SEMI(_) => Some("sep") + EOF => Some("eof") + _ => Some("item") + } +} + +///| +fn top_item_kind(tok : Token) -> NodeKind { + match tok { + TEST => Decl(DeclKind::Test) + USING => Impl(ImplKind::Using) + EXTERN => Decl(DeclKind::BalancedDecl) + FN | ASYNC => Decl(DeclKind::Function) + LET => Decl(DeclKind::Let) + CONST => Decl(DeclKind::Const) + TYPE => Decl(DeclKind::Type) + STRUCT => Decl(DeclKind::Struct) + ENUM | SUBERROR | EXTENUM => Decl(DeclKind::Enum) + ENUMVIEW => Decl(DeclKind::EnumView) + TRAIT => Decl(DeclKind::Trait) + IMPL => Decl(DeclKind::Impl) + DECLARE | PUB | PRIV | ATTRIBUTE(_) => Decl(DeclKind::BalancedDecl) + _ => Error + } +} + +///| +fn node_from_children( + kind : NodeKind, + children : Array[(String?, CstNode)], +) -> CstNode { + if children.length() == 0 { + let pos = initial_position("") + node(kind, empty_loc(pos), empty_span(0), []) + } else { + let (_, first) = children[0] + let (_, last) = children[children.length() - 1] + node( + kind, + first.loc().merge(last.loc()), + mk_span(first.source_span().start_offset, last.source_span().end_offset), + children, + ) + } +} + +///| +fn parse_token_list(state : ParserState, kind : NodeKind) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut saw_eof = false + let mut done = false + while !done { + match state.peek() { + Some((EOF, _, _)) => { + match state.consume_any() { + Some(eof) => children.push((Some("eof"), eof)) + None => () + } + saw_eof = true + done = true + } + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => + for child in state.consume_trivia() { + children.push(child) + } + Some((tok, _, _)) => + match state.consume_any() { + Some(consumed) => children.push((token_label(tok), consumed)) + None => () + } + None => done = true + } + } + if !saw_eof { + children.push((Some("eof"), state.missing_eof_node())) + } + node(kind, state.root_loc(), state.root_span(), children) +} + +///| +fn parse_top_level_item(state : ParserState) -> CstNode { + let first_kind = match state.peek_significant() { + Some((ASYNC, _, _)) => + match state.peek_nth_significant(1) { + Some((TEST, _, _)) => Decl(DeclKind::Test) + _ => Decl(DeclKind::Function) + } + Some((tok, _, _)) => top_item_kind(tok) + None => Error + } + parse_balanced_decl_body(state, first_kind) +} + +///| +fn parse_source_file(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut saw_eof = false + let mut done = false + while !done { + match state.peek() { + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => + for child in state.consume_trivia() { + children.push(child) + } + Some((SEMI(_), _, _)) => + match state.consume_any() { + Some(sep) => children.push((Some("sep"), sep)) + None => () + } + Some((EOF, _, _)) => { + match state.consume_any() { + Some(eof) => children.push((Some("eof"), eof)) + None => () + } + saw_eof = true + done = true + } + Some(_) => children.push((Some("item"), parse_top_level_item(state))) + None => done = true + } + } + if !saw_eof { + children.push((Some("eof"), state.missing_eof_node())) + } + node(SourceFile, state.root_loc(), state.root_span(), children) +} + +///| +fn parse_fragment(state : ParserState, entry : EntryPoint) -> CstNode { + match entry { + Structure => parse_source_file(state) + Expression => parse_expr_fragment(state) + } +} diff --git a/untyped_cst/parse_type.mbt b/untyped_cst/parse_type.mbt new file mode 100644 index 00000000..fcee1e9b --- /dev/null +++ b/untyped_cst/parse_type.mbt @@ -0,0 +1,511 @@ +///| +fn parse_type_until( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + parse_type_arrow(state, stop_kinds) +} + +///| +fn parse_type_atom( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => + node_from_children(Type(TypeKind::MissingType), []) + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => { + let children = state.consume_trivia() + let ty = parse_type_atom(state, stop_kinds) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + node_from_children(Type(TypeKind::Group), children) + } + Some((AMPER, _, _)) => parse_object_type_atom(state) + Some((UIDENT(_), _, _)) | Some((PACKAGE_NAME(_), _, _)) => + parse_type_name_atom(state) + Some((UNDERSCORE, _, _)) => parse_single_token_type(state, TypeKind::Hole) + Some((LPAREN, _, _)) => parse_paren_type(state, stop_kinds) + Some((LBRACE, start, end)) => { + let ty = parse_object_type_literal(state) + state.report_expected_description_at(LBRACE, mk_loc(start, end), "type") + ty + } + Some((ASYNC, _, _)) => parse_async_type_atom(state, stop_kinds) + Some((found, start, end)) => { + state.report_expected_description_at(found, mk_loc(start, end), "type") + let children : Array[(String?, CstNode)] = [] + while !state.current_in(stop_kinds) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(token) => push_child(children, Some("error"), token) + None => () + } + } + node_from_children(Type(TypeKind::MissingType), children) + } + None => node_from_children(Type(TypeKind::MissingType), []) + } +} + +///| +fn parse_single_token_type(state : ParserState, kind : TypeKind) -> CstNode { + match state.consume_any() { + Some(token) => node_from_children(Type(kind), [(Some("name"), token)]) + None => node_from_children(Type(TypeKind::MissingType), []) + } +} + +///| +fn parse_type_name_atom(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + match state.peek() { + Some((DOT_LIDENT(_), _, _)) | Some((DOT_UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + node_from_children(Type(TypeKind::Name), children) +} + +///| +fn parse_object_type_atom(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + match state.peek() { + Some((UIDENT(_), _, _)) | Some((PACKAGE_NAME(_), _, _)) => { + let name = parse_type_name_atom(state) + if name.children_length() > 0 { + push_child(children, Some("type"), name) + } + } + _ => + push_child( + children, + Some("type"), + state.missing_token_node(TokenKind::TK_UIDENT), + ) + } + node_from_children(Type(TypeKind::Object), children) +} + +///| +fn parse_async_type_atom( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(kw) => push_child(children, Some("kw"), kw) + None => () + } + match state.peek_significant() { + Some((LPAREN, _, _)) => () + Some((found, start, end)) => { + state.report_expected_tokens_at(found, mk_loc(start, end), [ + TokenKind::TK_LPAREN, + ]) + while !state.current_in(stop_kinds) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(children, Some("error"), bad) + None => () + } + } + return node_from_children(Type(TypeKind::Function), children) + } + None => state.report_expected_tokens([TokenKind::TK_LPAREN]) + } + let ty = parse_type_arrow(state, stop_kinds) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + node_from_children(Type(TypeKind::Function), children) +} + +///| +fn parse_paren_type_item(state : ParserState) -> CstNode { + parse_type_arrow(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RPAREN, + TokenKind::TK_EOF, + ]) +} + +///| +fn parse_paren_type( + state : ParserState, + _stop_kinds : Array[TokenKind], +) -> CstNode { + let list = parse_delimited( + state, + TokenKind::TK_LPAREN, + TokenKind::TK_RPAREN, + parse_paren_type_item, + TypeArgumentList, + sep_kind=TokenKind::TK_COMMA, + ) + let mut item_count = 0 + let mut sep_count = 0 + for i in 0.. item_count += 1 + Some((Some("sep"), _)) => sep_count += 1 + _ => () + } + } + let kind = if sep_count > 0 || item_count != 1 { + TypeKind::Tuple + } else { + TypeKind::Group + } + if item_count == 0 && !state.current_is(TokenKind::TK_THIN_ARROW) { + state.report_expected_tokens([TokenKind::TK_THIN_ARROW]) + } + node_from_children(Type(kind), [(Some("body"), list)]) +} + +///| +fn parse_object_field_type_item(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((LIDENT(_), _, _)) | Some((UIDENT(_), _, _)) => + match state.consume_any() { + Some(name) => push_child(children, Some("name"), name) + None => () + } + _ => () + } + if state.current_is(TokenKind::TK_COLON) { + match state.consume_any() { + Some(colon) => push_child(children, Some("op"), colon) + None => () + } + let ty = parse_type_arrow(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACE, + TokenKind::TK_EOF, + ]) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } + } + node_from_children(Clause(ClauseKind::Parameter), children) +} + +///| +fn parse_object_type_literal(state : ParserState) -> CstNode { + let list = parse_delimited( + state, + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + parse_object_field_type_item, + FieldList, + sep_kind=TokenKind::TK_COMMA, + ) + node_from_children(Type(TypeKind::Object), [(Some("body"), list)]) +} + +///| +fn parse_type_argument_item(state : ParserState) -> CstNode { + parse_type_arrow(state, [ + TokenKind::TK_COMMA, + TokenKind::TK_RBRACKET, + TokenKind::TK_EOF, + ]) +} + +///| +fn parse_type_argument_list(state : ParserState) -> CstNode { + parse_delimited( + state, + TokenKind::TK_LBRACKET, + TokenKind::TK_RBRACKET, + parse_type_argument_item, + TypeArgumentList, + sep_kind=TokenKind::TK_COMMA, + require_nonempty=true, + ) +} + +///| +fn parse_legacy_type_tail( + state : ParserState, + stop_kinds : Array[TokenKind], + head : CstNode, +) -> CstNode { + let children : Array[(String?, CstNode)] = [(Some("head"), head)] + let mut depth = 0 + let mut done = false + while !done { + match state.peek() { + Some((tok, _, _)) if depth == 0 && kind_in(tok.kind(), stop_kinds) => + done = true + Some((COMMENT(_), _, _)) | Some((NEWLINE, _, _)) => + push_children(children, state.consume_trivia()) + Some((tok, _, _)) => { + match state.consume_any() { + Some(consumed) => push_child(children, token_label(tok), consumed) + None => done = true + } + depth = next_depth(depth, tok) + } + None => done = true + } + } + node_from_children(Type(TypeKind::Application), children) +} + +///| +fn parse_simple_type( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + parse_type_atom(state, stop_kinds) +} + +///| +fn parse_type_postfix( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let mut ty = parse_simple_type(state, stop_kinds) + let mut done = false + while !done { + match state.peek() { + Some((tok, _, _)) if kind_in(tok.kind(), stop_kinds) => done = true + Some((LBRACKET, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("func"), ty)] + push_child(children, Some("arg"), parse_type_argument_list(state)) + ty = node_from_children(Type(TypeKind::Application), children) + } + Some((QUESTION, _, _)) => { + let children : Array[(String?, CstNode)] = [(Some("type"), ty)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + ty = node_from_children(Type(TypeKind::Option), children) + } + _ => done = true + } + } + match state.peek() { + Some((tok, _, _)) if !kind_in(tok.kind(), stop_kinds) => + match tok { + DOT_UIDENT(_) | DOT_LIDENT(_) | COLONCOLON => + parse_legacy_type_tail(state, stop_kinds, ty) + _ => ty + } + _ => ty + } +} + +///| +fn parse_type_arrow( + state : ParserState, + stop_kinds : Array[TokenKind], +) -> CstNode { + let lhs = parse_type_postfix(state, stop_kinds) + match state.peek() { + Some((THIN_ARROW, start, end)) if !kind_in( + TokenKind::TK_THIN_ARROW, + stop_kinds, + ) => { + let invalid_unparenthesized = match lhs.kind() { + Type(TypeKind::Group) | Type(TypeKind::Tuple) => false + _ => true + } + let _ = (start, end) + let children : Array[(String?, CstNode)] = [(Some("param"), lhs)] + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + let rhs = parse_type_arrow(state, stop_kinds) + if rhs.children_length() > 0 { + push_child(children, Some("result"), rhs) + } + match state.peek() { + Some((RAISE, _, _)) if !kind_in(TokenKind::TK_RAISE, stop_kinds) => + push_child(children, Some("clause"), parse_error_annotation(state)) + Some((NORAISE, _, _)) if !kind_in(TokenKind::TK_NORAISE, stop_kinds) => + push_child(children, Some("clause"), parse_error_annotation(state)) + _ => () + } + if invalid_unparenthesized { + state.report( + empty_loc(state.current_position()), + "Unexpected token here.", + ) + } + node_from_children(Type(TypeKind::Function), children) + } + _ => lhs + } +} + +///| +fn parse_return_type(state : ParserState) -> CstNode { + parse_type_arrow(state, [ + TokenKind::TK_RAISE, + TokenKind::TK_NORAISE, + TokenKind::TK_LBRACE, + TokenKind::TK_EQUAL, + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) +} + +///| +fn parse_func_return_type(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + let mut has_arrow = false + match state.peek() { + Some((THIN_ARROW, _, _)) => + match state.consume_any() { + Some(arrow) => { + has_arrow = true + push_child(children, Some("op"), arrow) + } + None => () + } + _ => () + } + if children.length() > 0 { + let ty = parse_return_type(state) + if ty.children_length() > 0 { + push_child(children, Some("type"), ty) + } else if has_arrow { + state.report_expected_description("return type annotation") + } + } + match state.peek() { + Some((RAISE, _, _)) | Some((NORAISE, _, _)) => + push_child(children, Some("clause"), parse_error_annotation(state)) + _ => () + } + node_from_children(Clause(ClauseKind::ReturnType), children) +} + +///| +fn parse_error_annotation(state : ParserState) -> CstNode { + let children : Array[(String?, CstNode)] = [] + match state.peek() { + Some((RAISE, _, _)) | Some((NORAISE, _, _)) => + match state.consume_any() { + Some(op) => push_child(children, Some("op"), op) + None => () + } + _ => () + } + match (state.peek(), state.peek(offset=1)) { + (Some((UIDENT(_), _, _)), Some((QUESTION, _, _))) => { + let type_children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(name) => push_child(type_children, Some("name"), name) + None => () + } + match state.consume_any() { + Some(question) => push_child(type_children, Some("question"), question) + None => () + } + push_child( + children, + Some("type"), + node_from_children(Type(TypeKind::Name), type_children), + ) + } + (Some((UIDENT(_), _, _)), Some((LBRACKET, start, end))) => { + let type_children : Array[(String?, CstNode)] = [] + match state.consume_any() { + Some(name) => push_child(type_children, Some("name"), name) + None => () + } + state.report_expected_tokens_at(LBRACKET, mk_loc(start, end), [ + TokenKind::TK_LBRACE, + ]) + while !state.current_is(TokenKind::TK_LBRACE) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(type_children, Some("error"), bad) + None => () + } + } + push_child( + children, + Some("type"), + node_from_children(Type(TypeKind::Name), type_children), + ) + } + (Some((AMPER, start, end)), _) => { + state.report_expected_tokens_at(AMPER, mk_loc(start, end), [ + TokenKind::TK_LBRACE, + ]) + let type_children : Array[(String?, CstNode)] = [] + while !state.current_is(TokenKind::TK_LBRACE) && + !state.current_is(TokenKind::TK_EOF) && + !(state.peek() is None) { + match state.consume_any() { + Some(bad) => push_child(type_children, Some("error"), bad) + None => () + } + } + push_child( + children, + Some("type"), + node_from_children(Type(TypeKind::MissingType), type_children), + ) + } + _ => + match state.peek() { + Some((QUESTION, _, _)) => + match state.consume_any() { + Some(question) => push_child(children, Some("type"), question) + None => () + } + Some((LBRACE, _, _)) + | Some((RBRACE, _, _)) + | Some((RPAREN, _, _)) + | Some((RBRACKET, _, _)) + | Some((COMMA, _, _)) + | Some((EQUAL, _, _)) + | Some((SEMI(_), _, _)) + | Some((EOF, _, _)) + | None => () + _ => { + let err_ty = parse_type_until(state, [ + TokenKind::TK_LBRACE, + TokenKind::TK_RBRACE, + TokenKind::TK_RPAREN, + TokenKind::TK_RBRACKET, + TokenKind::TK_COMMA, + TokenKind::TK_EQUAL, + TokenKind::TK_SEMI, + TokenKind::TK_EOF, + ]) + if err_ty.children_length() > 0 { + match err_ty.kind() { + Type(TypeKind::Name) | Type(TypeKind::Hole) => () + _ => state.report(err_ty.loc(), "invalid error type") + } + push_child(children, Some("type"), err_ty) + } + } + } + } + node_from_children(Clause(ClauseKind::ErrorAnnotation), children) +} diff --git a/untyped_cst/parser_state.mbt b/untyped_cst/parser_state.mbt new file mode 100644 index 00000000..e14c54af --- /dev/null +++ b/untyped_cst/parser_state.mbt @@ -0,0 +1,231 @@ +///| +priv struct ParserState { + tokens : Triples + mut pos : Int + reports : Array[Report] + source : String + location_mode : LocationMode + cnum_to_source_offset : (Int) -> Int +} + +///| +fn identity_source_offset(cnum : Int) -> Int { + cnum +} + +///| +fn default_location_mapper(source : String) -> (Int) -> Int { + let view = source.view() + let source_len = view.length() + let table = Array::make(source_len + 1, source_len) + let mut source_offset = 0 + let mut surrogate_pairs = 0 + while source_offset <= source_len { + let cnum = source_offset - surrogate_pairs + if cnum >= 0 && cnum < table.length() { + table[cnum] = source_offset + } + if source_offset < source_len && + view[source_offset].is_leading_surrogate() && + source_offset + 1 < source_len && + view[source_offset + 1].is_trailing_surrogate() { + source_offset += 2 + surrogate_pairs += 1 + } else { + source_offset += 1 + } + } + fn(cnum : Int) -> Int { + if cnum < 0 { + 0 + } else if cnum >= table.length() { + source_len + } else { + table[cnum] + } + } +} + +///| +fn source_offset_mapper( + source : String, + location_mode : LocationMode, +) -> (Int) -> Int { + match location_mode { + DefaultLocation => default_location_mapper(source) + Utf16Location | SourceOffsetLocation => identity_source_offset + } +} + +///| +fn ParserState::peek( + self : ParserState, + offset? : Int = 0, +) -> (Token, Position, Position)? { + self.tokens.get(self.pos + offset) +} + +///| +fn ParserState::peek_significant( + self : ParserState, +) -> (Token, Position, Position)? { + let mut offset = 0 + while self.pos + offset < self.tokens.length() { + match self.tokens[self.pos + offset] { + (COMMENT(_), _, _) | (NEWLINE, _, _) => { + offset += 1 + continue + } + triple => return Some(triple) + } + } + None +} + +///| +fn ParserState::peek_nth_significant( + self : ParserState, + nth : Int, +) -> (Token, Position, Position)? { + let mut offset = 0 + let mut seen = 0 + while self.pos + offset < self.tokens.length() { + match self.tokens[self.pos + offset] { + (COMMENT(_), _, _) | (NEWLINE, _, _) => { + offset += 1 + continue + } + triple => { + if seen == nth { + return Some(triple) + } + seen += 1 + offset += 1 + } + } + } + None +} + +///| +fn ParserState::token_source_span( + self : ParserState, + start : Position, + end : Position, +) -> SourceSpan { + mk_span( + (self.cnum_to_source_offset)(start.cnum), + (self.cnum_to_source_offset)(end.cnum), + ) +} + +///| +fn ParserState::consume_any(self : ParserState) -> CstNode? { + match self.peek() { + Some((tok, start, end)) => { + self.pos += 1 + Some(token_node(tok, start, end, self.token_source_span(start, end))) + } + None => None + } +} + +///| +fn ParserState::consume_trivia(self : ParserState) -> Array[(String?, CstNode)] { + let children : Array[(String?, CstNode)] = [] + let mut done = false + while !done { + match self.peek() { + Some((COMMENT(_), _, _)) => + match self.consume_any() { + Some(comment) => children.push((Some("trivia"), comment)) + None => done = true + } + Some((NEWLINE, _, _)) => self.pos += 1 + _ => done = true + } + } + children +} + +///| +fn ParserState::consume_if( + self : ParserState, + pred : (Token) -> Bool, + label? : String = "", +) -> CstNode? { + let _ = label + while true { + match self.peek() { + Some((NEWLINE, _, _)) => self.pos += 1 + Some((tok, _, _)) if pred(tok) => + match self.consume_any() { + Some(node) => return Some(node) + None => return None + } + _ => return None + } + } + None +} + +///| +fn ParserState::previous_end(self : ParserState) -> Position { + if self.pos > 0 { + let (_, _, end) = self.tokens[self.pos - 1] + end + } else if self.tokens.length() > 0 { + let (_, start, _) = self.tokens[0] + start + } else { + initial_position("") + } +} + +///| +fn ParserState::missing_eof_offset(self : ParserState) -> Int { + self.source.length() +} + +///| +fn ParserState::missing_eof_node(self : ParserState) -> CstNode { + let pos = self.previous_end() + let offset = self.missing_eof_offset() + let missing = missing_node(TokenKind::TK_EOF, pos, offset) + self.reports.push(Report::{ loc: empty_loc(pos), msg: "expected EOF" }) + missing +} + +///| +fn ParserState::root_loc(self : ParserState) -> Location { + if self.tokens.length() == 0 { + empty_loc(initial_position("")) + } else { + let (_, start, _) = self.tokens[0] + let (_, _, end) = self.tokens[self.tokens.length() - 1] + mk_loc(start, end) + } +} + +///| +fn ParserState::root_span(self : ParserState) -> SourceSpan { + mk_span(0, self.source.length()) +} + +///| +fn new_state( + tokens : Triples, + source : String, + location_mode : LocationMode, + reports : Array[Report], +) -> ParserState { + let mapper = source_offset_mapper(source, location_mode) + ParserState::{ + tokens, + pos: 0, + reports, + source, + location_mode, + cnum_to_source_offset: mapper, + } +} diff --git a/untyped_cst/parser_state_test.mbt b/untyped_cst/parser_state_test.mbt new file mode 100644 index 00000000..4d81167e --- /dev/null +++ b/untyped_cst/parser_state_test.mbt @@ -0,0 +1,160 @@ +///| +fn any_node( + node : @untyped_cst.CstNode, + pred : (@untyped_cst.NodeKind) -> Bool, +) -> Bool { + if pred(node.kind()) { + return true + } + for i in 0.. if any_node(child, pred) { return true } + None => () + } + } + false +} + +///| +fn count_nodes( + node : @untyped_cst.CstNode, + pred : (@untyped_cst.NodeKind) -> Bool, +) -> Int { + let mut count = if pred(node.kind()) { 1 } else { 0 } + for i in 0.. count += count_nodes(child, pred) + None => () + } + } + count +} + +///| +fn first_node( + node : @untyped_cst.CstNode, + pred : (@untyped_cst.NodeKind) -> Bool, +) -> @untyped_cst.CstNode? { + if pred(node.kind()) { + return Some(node) + } + for i in 0.. + match first_node(child, pred) { + Some(found) => return Some(found) + None => () + } + None => () + } + } + None +} + +///| +fn is_comment(kind : @untyped_cst.NodeKind) -> Bool { + match kind { + @untyped_cst.Token(@tokens.COMMENT(_)) => true + _ => false + } +} + +///| +fn is_newline(kind : @untyped_cst.NodeKind) -> Bool { + match kind { + @untyped_cst.Token(@tokens.NEWLINE) => true + _ => false + } +} + +///| +fn is_asi_semi(kind : @untyped_cst.NodeKind) -> Bool { + match kind { + @untyped_cst.Token(@tokens.SEMI(false)) => true + _ => false + } +} + +///| +fn is_any_semi(kind : @untyped_cst.NodeKind) -> Bool { + match kind { + @untyped_cst.Token(@tokens.SEMI(_)) => true + _ => false + } +} + +///| +fn is_eof(kind : @untyped_cst.NodeKind) -> Bool { + match kind { + @untyped_cst.Token(@tokens.EOF) => true + _ => false + } +} + +///| +fn is_missing_eof(kind : @untyped_cst.NodeKind) -> Bool { + match kind { + @untyped_cst.Missing(@tokens.TokenKind::TK_EOF) => true + _ => false + } +} + +///| +fn is_lident(name : String) -> (@untyped_cst.NodeKind) -> Bool { + fn(kind) { + match kind { + @untyped_cst.Token(@tokens.LIDENT(id)) => id == name + _ => false + } + } +} + +///| +test "comments are retained and newline tokens are skipped" { + let source = "// c\nlet x = 1" + let result = @untyped_cst.parse_string(source) + let root = result.root() + @test.assert_eq(result.diagnostics_view().length(), 0) + @test.assert_eq(root.text(source), source) + @test.assert_eq(any_node(root, is_comment), true) + @test.assert_eq(any_node(root, is_newline), false) +} + +///| +test "asi semicolon is zero width and explicit semicolon spans source" { + let asi_source = "let x = 1\nlet y = 2" + let asi = @untyped_cst.parse_string(asi_source).root() + guard first_node(asi, is_asi_semi) is Some(asi_semi) else { + fail("expected ASI semicolon") + } + @test.assert_eq( + asi_semi.source_span().start_offset, + asi_semi.source_span().end_offset, + ) + let explicit_source = "let x = 1; let y = 2" + let explicit = @untyped_cst.parse_string(explicit_source).root() + guard first_node(explicit, is_any_semi) is Some(explicit_semi) else { + fail("expected explicit semicolon") + } + @test.assert_eq(explicit_semi.text(explicit_source), ";") +} + +///| +test "default entry preserves text after emoji" { + let source = "let s = \"😀\"\nlet x = 1" + let result = @untyped_cst.parse_string(source) + @test.assert_eq(result.root().text(source), source) + guard first_node(result.root(), is_lident("x")) is Some(x) else { + fail("expected x token") + } + @test.assert_eq(x.text(source), "x") +} + +///| +test "source-backed roots cover leading and trailing layout" { + let source = " let x = 1 " + let result = @untyped_cst.parse_string(source) + @test.assert_eq(result.root().source_span().start_offset, 0) + @test.assert_eq(result.root().source_span().end_offset, source.length()) + @test.assert_eq(result.root().text(source), source) +} diff --git a/untyped_cst/pkg.generated.mbti b/untyped_cst/pkg.generated.mbti new file mode 100644 index 00000000..89223f87 --- /dev/null +++ b/untyped_cst/pkg.generated.mbti @@ -0,0 +1,275 @@ +// Generated using `moon info`, DON'T EDIT IT +package "moonbitlang/parser/untyped_cst" + +import { + "moonbitlang/core/debug", + "moonbitlang/core/list", + "moonbitlang/lexer/basic", + "moonbitlang/lexer/tokens", + "moonbitlang/parser/syntax", +} + +// Values +pub fn parse_string(String, name? : String, entry? : EntryPoint) -> ParseResult + +// Errors + +// Types and methods +pub(all) enum ArgumentKind { + Positional + Labelled + LabelledPun + LabelledOption + LabelledOptionPun +} derive(@debug.Debug) + +pub(all) enum CaseKind { + MatchCase + LexCase + LexScanCase +} derive(@debug.Debug) + +pub(all) enum ClauseKind { + Attribute + Visibility + Parameter + ReturnType + ErrorAnnotation + Where + Body +} derive(@debug.Debug) + +pub struct CstNode { + kind : NodeKind + loc : @basic.Location + source_span : SourceSpan + children : ReadOnlyArray[(String?, CstNode)] +} derive(@debug.Debug) +pub fn CstNode::child_at(Self, Int) -> (String?, Self)? +pub fn CstNode::children_length(Self) -> Int +pub fn CstNode::children_view(Self) -> ArrayView[(String?, Self)] +pub fn CstNode::kind(Self) -> NodeKind +pub fn CstNode::loc(Self) -> @basic.Location +pub fn CstNode::source_span(Self) -> SourceSpan +pub fn CstNode::text(Self, String) -> String + +pub(all) enum DeclKind { + BalancedDecl + Function + Let + Const + Type + Struct + Enum + EnumView + Trait + Impl + Test + Using + Extern + Declare +} derive(Eq, @debug.Debug) + +pub(all) enum EntryPoint { + Structure + Expression +} derive(@debug.Debug) + +pub(all) enum ExprKind { + Placeholder + Identifier + Literal + Block + Apply + DotApply + Field + ArrayGet + ArrayGetSlice + Array + ListComprehension + Tuple + Constraint + Record + Map + Infix + Prefix + Range + Assign + AugmentedAssign + If + Match + Try + TryOperator + For + ForEach + While + Function + Method + As + Is + LexMatch + LexScan + RegexMatch + TemplateWriting + Sequence + Let + LetMut + LetRec + LetFn + Return + Raise + Break + Continue + Defer + Guard + Quantifier + Hole + Group + Constructor + MultilineString + MissingExpr +} derive(@debug.Debug) + +pub(all) enum ImplKind { + Declaration + Test + Using + Extern + BalancedDecl +} derive(Eq, @debug.Debug) + +pub(all) enum InterpSegmentKind { + InterpLiteralSegment + InterpSourceSegment +} derive(@debug.Debug) + +pub(all) enum LexPatternKind { + Placeholder + Literal + RegexInterp + Sequence + Alias + Binder + Wildcard + ConstantRef +} derive(@debug.Debug) + +pub(all) enum ListKind { + ArgumentList + ParameterList + FieldList + StatementList + TypeArgumentList + PatternList + ConstructorList +} derive(@debug.Debug) + +pub(all) enum LocationMode { + DefaultLocation + Utf16Location + SourceOffsetLocation +} derive(@debug.Debug) + +pub(all) enum NameKind { + Identifier + Constructor + Field + Label + Qualified +} derive(@debug.Debug) + +pub(all) enum NodeKind { + SourceFile + Fragment(EntryPoint) + Token(@tokens.Token) + Missing(@tokens.TokenKind) + Error + Impl(ImplKind) + Expr(ExprKind) + Pattern(PatternKind) + Type(TypeKind) + LexPattern(LexPatternKind) + RegexPattern(RegexPatternKind) + Case(CaseKind) + Argument(ArgumentKind) + Stmt(StmtKind) + Decl(DeclKind) + Clause(ClauseKind) + Name(NameKind) + List(ListKind) + InterpSegment(InterpSegmentKind) +} derive(@debug.Debug) + +pub struct ParseResult { + root : CstNode + diagnostics : ReadOnlyArray[@basic.Report] + docstrings : ReadOnlyArray[@list.List[(@basic.Location, @tokens.Comment)]] +} derive(@debug.Debug) +pub fn ParseResult::diagnostics_view(Self) -> ArrayView[@basic.Report] +pub fn ParseResult::docstrings_view(Self) -> ArrayView[@list.List[(@basic.Location, @tokens.Comment)]] +pub fn ParseResult::root(Self) -> CstNode +pub fn ParseResult::to_expr(Self) -> (@syntax.Expr?, Array[@basic.Report]) +pub fn ParseResult::to_impls(Self) -> (@list.List[@syntax.Impl], Array[@basic.Report]) + +pub(all) enum PatternKind { + Placeholder + Binder + Underscore + Literal + Tuple + Array + Rest + Record + Map + Constructor + SpecialConstructor + Or + Alias + Range + Constraint + MissingPattern +} derive(@debug.Debug) + +pub(all) enum RegexPatternKind { + Placeholder + Literal + Reference + Sequence + Alternation + Alias +} derive(@debug.Debug) + +pub(all) struct SourceSpan { + start_offset : Int + end_offset : Int +} derive(Eq, @debug.Debug) + +pub(all) enum StmtKind { + Let + LetMut + LetRec + Function + Expr + Guard + Defer + ProofAssert + ProofLet +} derive(@debug.Debug) + +pub(all) enum TypeKind { + Placeholder + Name + Tuple + Function + Application + Option + Object + Hole + Group + MissingType +} derive(@debug.Debug) + +// Type aliases + +// Traits + diff --git a/untyped_cst/recovery.mbt b/untyped_cst/recovery.mbt new file mode 100644 index 00000000..39271df8 --- /dev/null +++ b/untyped_cst/recovery.mbt @@ -0,0 +1,208 @@ +///| +fn report_from_lex_error( + start : Position, + end : Position, + err : @lexer.LexicalError, +) -> Report { + Report::{ loc: mk_loc(start, end), msg: err.to_string() } +} + +///| +fn reports_from_lex_errors( + errors : Array[(Position, Position, @lexer.LexicalError)], +) -> Array[Report] { + let reports : Array[Report] = [] + for error in errors { + let (start, end, err) = error + match err { + Reserved_keyword(_) => () + _ => reports.push(report_from_lex_error(start, end, err)) + } + } + reports +} + +///| +fn ParserState::current_position(self : ParserState) -> Position { + match self.peek_significant() { + Some((_, start, _)) => start + None => self.previous_end() + } +} + +///| +fn expected_token_kinds_to_string(kinds : Array[TokenKind]) -> String { + match kinds { + [] => "" + [kind] => kind.to_expect_string() + _ => kinds.iter().map(kind => kind.to_expect_string()).join(", ") + } +} + +///| +fn ParserState::report_expected_description_at( + self : ParserState, + found : Token, + loc : Location, + expected : String, +) -> Unit { + let msg = match found { + SEMI(false) => + "Unexpected line break here, missing \{expected} at the end of this line." + EOF => "Unexpected end of file, missing \{expected} here." + _ => + "Unexpected token \{found.to_expect_string()}, you may expect \{expected}." + } + self.report(loc, msg) +} + +///| +fn ParserState::report_expected_tokens_at( + self : ParserState, + found : Token, + loc : Location, + expected : Array[TokenKind], +) -> Unit { + self.report_expected_description_at( + found, + loc, + expected_token_kinds_to_string(expected), + ) +} + +///| +fn ParserState::report_expected_description( + self : ParserState, + expected : String, +) -> Unit { + match self.peek_significant() { + Some((found, start, end)) => + self.report_expected_description_at(found, mk_loc(start, end), expected) + None => { + let pos = self.previous_end() + self.report( + empty_loc(pos), + "Unexpected end of file, missing \{expected} here.", + ) + } + } +} + +///| +fn ParserState::report_expected_tokens( + self : ParserState, + expected : Array[TokenKind], +) -> Unit { + self.report_expected_description(expected_token_kinds_to_string(expected)) +} + +///| +fn ParserState::offset_for_position(self : ParserState, pos : Position) -> Int { + if self.peek() is None { + self.source.length() + } else { + (self.cnum_to_source_offset)(pos.cnum) + } +} + +///| +fn ParserState::missing_token_node( + self : ParserState, + kind : TokenKind, +) -> CstNode { + let pos = self.current_position() + self.report_expected_tokens([kind]) + missing_node(kind, pos, self.offset_for_position(pos)) +} + +///| +fn ParserState::missing_token_node_with_expected( + self : ParserState, + kind : TokenKind, + expected : Array[TokenKind], +) -> CstNode { + let pos = self.current_position() + self.report_expected_tokens(expected) + missing_node(kind, pos, self.offset_for_position(pos)) +} + +///| +fn ParserState::missing_token_node_with_description( + self : ParserState, + kind : TokenKind, + expected : String, +) -> CstNode { + let pos = self.current_position() + self.report_expected_description(expected) + missing_node(kind, pos, self.offset_for_position(pos)) +} + +///| +fn ParserState::missing_token_node_silent( + self : ParserState, + kind : TokenKind, +) -> CstNode { + let pos = self.current_position() + missing_node(kind, pos, self.offset_for_position(pos)) +} + +///| +fn ParserState::missing_syntax_node( + self : ParserState, + kind : NodeKind, +) -> CstNode { + let pos = self.current_position() + node(kind, empty_loc(pos), empty_span(self.offset_for_position(pos)), []) +} + +///| +fn ParserState::missing_type_node(self : ParserState) -> CstNode { + self.missing_syntax_node(Type(TypeKind::MissingType)) +} + +///| +fn ParserState::missing_expr_node(self : ParserState) -> CstNode { + self.missing_syntax_node(Expr(ExprKind::MissingExpr)) +} + +///| +fn ParserState::missing_pattern_node(self : ParserState) -> CstNode { + self.missing_syntax_node(Pattern(PatternKind::MissingPattern)) +} + +///| +fn ParserState::required_type_node( + self : ParserState, + parsed : CstNode, + reports_before : Int, +) -> CstNode { + if parsed.children_length() > 0 { + parsed + } else { + if self.reports.length() == reports_before { + self.report_expected_description("type") + } + self.missing_type_node() + } +} + +///| +fn ParserState::report( + self : ParserState, + loc : Location, + msg : String, +) -> Unit { + self.reports.push(Report::{ loc, msg }) +} + +///| +fn ParserState::expect_token(self : ParserState, kind : TokenKind) -> CstNode { + match self.peek() { + Some((tok, _, _)) if tok.kind() == kind => + match self.consume_any() { + Some(node) => node + None => self.missing_token_node(kind) + } + _ => self.missing_token_node(kind) + } +} diff --git a/untyped_cst/text.mbt b/untyped_cst/text.mbt new file mode 100644 index 00000000..b1f850c6 --- /dev/null +++ b/untyped_cst/text.mbt @@ -0,0 +1,10 @@ +///| +pub fn CstNode::text(self : CstNode, source : String) -> String { + match self.kind { + Missing(_) => "" + _ => + source + .sub(start=self.source_span.start_offset, end=self.source_span.end_offset) + .to_owned() + } +} diff --git a/untyped_cst/to_ast.mbt b/untyped_cst/to_ast.mbt new file mode 100644 index 00000000..53d01da4 --- /dev/null +++ b/untyped_cst/to_ast.mbt @@ -0,0 +1,53 @@ +///| +fn ParseResult::diagnostics_array(self : ParseResult) -> Array[Report] { + let reports : Array[Report] = [] + for report in self.diagnostics_view() { + reports.push(report) + } + reports +} + +///| +fn conversion_report(root : CstNode, msg : String) -> Report { + Report::{ loc: root.loc(), msg } +} + +///| +pub fn ParseResult::to_impls( + self : ParseResult, +) -> (@syntax.Impls, Array[Report]) { + let diagnostics = self.diagnostics_array() + if diagnostics.length() > 0 { + return (@list.empty(), diagnostics) + } + match self.root().kind() { + SourceFile | Fragment(Structure) => () + _ => + return ( + @list.empty(), + [conversion_report(self.root(), "expected structure parse result")], + ) + } + let impls = lower_impls_from_cst(self.root()) + attach_docstrings_to_impls(self.docstrings_view(), impls) + (impls, []) +} + +///| +pub fn ParseResult::to_expr( + self : ParseResult, +) -> (@syntax.Expr?, Array[Report]) { + let diagnostics = self.diagnostics_array() + if diagnostics.length() > 0 { + return (None, diagnostics) + } + match self.root().kind() { + Expr(_) | Fragment(Expression) => () + _ => + return ( + None, + [conversion_report(self.root(), "expected expression parse result")], + ) + } + (Some(lower_expr(self.root())), []) +}