From 58d342725065ac2f201196a515dfef5b58bd7a11 Mon Sep 17 00:00:00 2001 From: Steven Smyth Date: Sun, 13 Sep 2026 23:10:50 +0200 Subject: [PATCH 1/4] Add railway diagrams --- cmd/fastbelt/generate.go | 24 ++ cmd/fastbelt/main.go | 4 +- internal/generator/railroad_generator.go | 51 ++++ internal/generator/railroad_generator_test.go | 83 ++++++ internal/grammar/server/main.go | 2 + internal/grammarhover/provider.go | 93 ++++++ internal/grammarhover/provider_test.go | 171 +++++++++++ internal/railroad/choice.go | 120 ++++++++ internal/railroad/diagram.go | 68 +++++ internal/railroad/diagram_test.go | 52 ++++ internal/railroad/grammar_railroad.go | 131 +++++++++ internal/railroad/grammar_railroad_test.go | 278 ++++++++++++++++++ internal/railroad/node.go | 46 +++ internal/railroad/node_test.go | 63 ++++ internal/railroad/path.go | 60 ++++ internal/railroad/repeat.go | 58 ++++ internal/railroad/sequence.go | 63 ++++ internal/railroad/style.go | 54 ++++ internal/railroad/svg.go | 127 ++++++++ internal/railroad/terminal.go | 109 +++++++ internal/vscode-extensions/fastbelt/README.md | 15 + server/hover_provider.go | 58 ++-- 22 files changed, 1710 insertions(+), 20 deletions(-) create mode 100644 internal/generator/railroad_generator.go create mode 100644 internal/generator/railroad_generator_test.go create mode 100644 internal/grammarhover/provider.go create mode 100644 internal/grammarhover/provider_test.go create mode 100644 internal/railroad/choice.go create mode 100644 internal/railroad/diagram.go create mode 100644 internal/railroad/diagram_test.go create mode 100644 internal/railroad/grammar_railroad.go create mode 100644 internal/railroad/grammar_railroad_test.go create mode 100644 internal/railroad/node.go create mode 100644 internal/railroad/node_test.go create mode 100644 internal/railroad/path.go create mode 100644 internal/railroad/repeat.go create mode 100644 internal/railroad/sequence.go create mode 100644 internal/railroad/style.go create mode 100644 internal/railroad/svg.go create mode 100644 internal/railroad/terminal.go diff --git a/cmd/fastbelt/generate.go b/cmd/fastbelt/generate.go index cf56d6ba..f3852174 100644 --- a/cmd/fastbelt/generate.go +++ b/cmd/fastbelt/generate.go @@ -26,6 +26,7 @@ type generateOptions struct { outputPath string packageName string atn bool + railroad bool verbose bool } @@ -165,10 +166,33 @@ func runGenerateCLI(opts generateOptions) error { return err } } + if opts.railroad { + if err := writeRailroadDiagrams(grammar, packageName, outputPath, writeFile); err != nil { + return err + } + } return nil } +func writeRailroadDiagrams(g grammarPkg.Grammar, packageName, outputPath string, writeFile func(name, path, content string) error) error { + railroadDir := filepath.Join(outputPath, "railroad") + if err := os.MkdirAll(railroadDir, 0755); err != nil { + return err + } + diagrams := generator.GenerateRailroadDiagrams(g) + names := make([]string, 0, len(diagrams)) + for name, svg := range diagrams { + names = append(names, name) + if err := writeFile("railroad-"+name, filepath.Join(railroadDir, name+".svg"), svg); err != nil { + return err + } + } + sort.Strings(names) + return writeFile("railroad-index", filepath.Join(railroadDir, "index.md"), + generator.GenerateRailroadIndexMarkdown(packageName, names)) +} + func validateEntryRule(g grammarPkg.Grammar) (grammarPkg.ParserRule, error) { var entries []grammarPkg.ParserRule for _, rule := range g.Rules() { diff --git a/cmd/fastbelt/main.go b/cmd/fastbelt/main.go index 08a517c7..825f1bd5 100644 --- a/cmd/fastbelt/main.go +++ b/cmd/fastbelt/main.go @@ -32,7 +32,8 @@ each step.` const generateExamples = ` fastbelt generate ./grammar.fb fastbelt generate ./lang.fb -o ./internal/lang -p lang - fastbelt generate ./mylanguage.fb --atn -v` + fastbelt generate ./mylanguage.fb --atn -v + fastbelt generate ./mylanguage.fb --railroad` const scaffoldLongHelp = `Scaffold a new language project from templates. @@ -83,6 +84,7 @@ func newGenerateCmd() *cobra.Command { cmd.Flags().StringVarP(&opts.outputPath, "output", "o", "./", "Path to the output directory") cmd.Flags().StringVarP(&opts.packageName, "package", "p", "", "Package name for generated code (defaults to the last segment of the output path)") cmd.Flags().BoolVar(&opts.atn, "atn", false, "Enable markdown output about ATN construction") + cmd.Flags().BoolVar(&opts.railroad, "railroad", false, "Generate railroad syntax diagrams (SVG per rule)") cmd.Flags().BoolVarP(&opts.verbose, "verbose", "v", false, "Enable verbose output about written files") return cmd } diff --git a/internal/generator/railroad_generator.go b/internal/generator/railroad_generator.go new file mode 100644 index 00000000..3cbce3f8 --- /dev/null +++ b/internal/generator/railroad_generator.go @@ -0,0 +1,51 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package generator + +import ( + "context" + "fmt" + "strings" + + "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/internal/railroad" +) + +// GenerateRailroadDiagrams renders a standalone SVG railroad diagram for +// every parser, composite, and infix rule of grammr, keyed by rule name. +// Rule kinds with no meaningful concrete-syntax diagram (terminal/token +// rules) are omitted. +func GenerateRailroadDiagrams(grammr grammar.Grammar) map[string]string { + ctx := context.Background() + out := make(map[string]string) + for _, rule := range grammr.Rules() { + if diagram, ok := railroad.BuildRuleDiagram(ctx, rule); ok { + out[rule.Name()] = diagram.SVG() + } + } + for _, rule := range grammr.Composites() { + if diagram, ok := railroad.BuildRuleDiagram(ctx, rule); ok { + out[rule.Name()] = diagram.SVG() + } + } + for _, rule := range grammr.InfixRules() { + if diagram, ok := railroad.BuildRuleDiagram(ctx, rule); ok { + out[rule.Name()] = diagram.SVG() + } + } + return out +} + +// GenerateRailroadIndexMarkdown renders a Markdown page linking every named +// diagram (by relative .svg file path) for convenient one-page +// browsing. +func GenerateRailroadIndexMarkdown(packageName string, ruleNames []string) string { + var b strings.Builder + fmt.Fprintf(&b, "# Railroad diagrams for %s\n\n", packageName) + for _, name := range ruleNames { + fmt.Fprintf(&b, "## %s\n\n![%s](%s.svg)\n\n", name, name, name) + } + return b.String() +} diff --git a/internal/generator/railroad_generator_test.go b/internal/generator/railroad_generator_test.go new file mode 100644 index 00000000..ab82fd5e --- /dev/null +++ b/internal/generator/railroad_generator_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package generator + +import ( + "encoding/xml" + "strings" + "testing" + + "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/test" +) + +const railroadGeneratorFixtureGrammar = `grammar Test + +interface Foo { + Name string + Items []Item +} + +interface Item { + Name string +} + +entry Foo returns Foo: + "begin" Name=ID Items+=Item* + +Item returns Item: + Name=ID + +composite Combo: "prefix" Item; + +token ID: /[a-zA-Z_][a-zA-Z0-9_]*/ +` + +func TestGenerateRailroadDiagrams(t *testing.T) { + sc := grammar.CreateServices() + f := test.New(t, sc) + doc := f.Parse(railroadGeneratorFixtureGrammar) + doc.AssertNoParseErrors() + + g, ok := doc.Root().(grammar.Grammar) + if !ok { + t.Fatalf("document root = %T, want grammar.Grammar", doc.Root()) + } + + diagrams := GenerateRailroadDiagrams(g) + + for _, name := range []string{"Foo", "Item", "Combo"} { + svg, ok := diagrams[name] + if !ok { + t.Errorf("no diagram generated for rule %q (got: %v)", name, diagramNames(diagrams)) + continue + } + var root struct{ XMLName xml.Name } + if err := xml.Unmarshal([]byte(svg), &root); err != nil { + t.Errorf("diagram for %q is not well-formed XML: %v\n%s", name, err, svg) + } + } + + if _, ok := diagrams["ID"]; ok { + t.Error("a token rule should not have a diagram generated for it") + } +} + +func TestGenerateRailroadIndexMarkdown(t *testing.T) { + md := GenerateRailroadIndexMarkdown("mylang", []string{"Foo", "Item"}) + for _, want := range []string{"# Railroad diagrams for mylang", "## Foo", "![Foo](Foo.svg)", "## Item", "![Item](Item.svg)"} { + if !strings.Contains(md, want) { + t.Errorf("index markdown missing %q, got:\n%s", want, md) + } + } +} + +func diagramNames(diagrams map[string]string) []string { + names := make([]string, 0, len(diagrams)) + for name := range diagrams { + names = append(names, name) + } + return names +} diff --git a/internal/grammar/server/main.go b/internal/grammar/server/main.go index 9acb9c5f..7f685048 100644 --- a/internal/grammar/server/main.go +++ b/internal/grammar/server/main.go @@ -9,6 +9,7 @@ import ( "log" "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/internal/grammarhover" "typefox.dev/fastbelt/server" "typefox.dev/fastbelt/util/service" ) @@ -18,6 +19,7 @@ func CreateServices() *service.Container { sc := service.NewContainer() grammar.SetupServices(sc) grammar.SetupGeneratedServerServices(sc) + service.Put(sc, grammarhover.NewHoverProvider(sc)) server.SetupDefaultServices(sc) server.SetupStdioServices(sc) sc.Seal() diff --git a/internal/grammarhover/provider.go b/internal/grammarhover/provider.go new file mode 100644 index 00000000..e10b630b --- /dev/null +++ b/internal/grammarhover/provider.go @@ -0,0 +1,93 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +// Package grammarhover provides a server.HoverProvider for fastbelt's own +// self-hosted grammar language (internal/grammar): the same +// documentation-comment content as server.DefaultHoverProvider, plus - for +// grammar rule nodes - a railroad syntax diagram of the rule's concrete +// syntax. +// +// It exists as its own package - rather than living in package server - for +// two reasons. First, railroad diagrams only ever make sense for a +// language whose AST has a "grammar rule" concept, i.e. only +// internal/grammar; no other language server built with fastbelt has any +// use for it, so it doesn't belong in the generic framework package. +// Second, it couldn't live in server even if that were desirable: +// internal/railroad already imports internal/grammar (to map grammar AST +// nodes to diagram nodes), and internal/grammar's own test files import +// package server (to exercise the real language server) - so server +// importing internal/grammar or internal/railroad, directly or transitively, +// would close an import cycle for those tests. This package is the one +// place allowed to depend on all three, since it's imported only by the +// grammar language server's entrypoint (internal/grammar/server) and by +// this package's own tests. +package grammarhover + +import ( + "context" + "encoding/base64" + "fmt" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/internal/railroad" + "typefox.dev/fastbelt/server" + "typefox.dev/fastbelt/util/service" + "typefox.dev/lsp" +) + +// HoverProvider is the grammar language's implementation of +// [server.HoverProvider]. +type HoverProvider struct { + sc *service.Container +} + +func NewHoverProvider(sc *service.Container) server.HoverProvider { + return &HoverProvider{sc: sc} +} + +func (p *HoverProvider) HandleHoverRequest(ctx context.Context, params *lsp.HoverParams) (*lsp.Hover, error) { + target, sourceRange, ok := server.ResolveHoverTarget(ctx, p.sc, params) + if !ok { + return nil, nil + } + + content := "" + if docProvider, err := service.Get[server.DocumentationProvider](p.sc); err == nil { + content = docProvider.Documentation(target) + } + if diagram := diagramMarkdown(ctx, target); diagram != "" { + if content != "" { + content += "\n\n---\n\n" + } + content += diagram + } + if content == "" { + return nil, nil + } + + return &lsp.Hover{ + Contents: lsp.MarkupContent{ + Kind: lsp.Markdown, + Value: content, + }, + Range: sourceRange, + }, nil +} + +// diagramMarkdown returns a Markdown image embedding a base64-encoded SVG +// railroad diagram for node, or "" if node isn't a grammar rule with a +// meaningful concrete-syntax diagram (e.g. a terminal/token rule). +func diagramMarkdown(ctx context.Context, node core.AstNode) string { + rule, ok := node.(grammar.AbstractRule) + if !ok { + return "" + } + diagram, ok := railroad.BuildRuleDiagram(ctx, rule) + if !ok { + return "" + } + encoded := base64.StdEncoding.EncodeToString([]byte(diagram.SVG())) + return fmt.Sprintf("![Railroad diagram](data:image/svg+xml;base64,%s)", encoded) +} diff --git a/internal/grammarhover/provider_test.go b/internal/grammarhover/provider_test.go new file mode 100644 index 00000000..429e0876 --- /dev/null +++ b/internal/grammarhover/provider_test.go @@ -0,0 +1,171 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package grammarhover + +import ( + "encoding/base64" + "encoding/xml" + "strings" + "testing" + + "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/server" + "typefox.dev/fastbelt/test" + "typefox.dev/fastbelt/util/service" + "typefox.dev/lsp" +) + +const dataImagePrefix = "data:image/svg+xml;base64," + +func newHoverFixture(t *testing.T) (*service.Container, *test.Fixture) { + t.Helper() + sc := service.NewContainer() + grammar.SetupServices(sc) + service.Put(sc, NewHoverProvider(sc)) + server.SetupDefaultServices(sc) + sc.Seal() + return sc, test.New(t, sc) +} + +func hoverAt(t *testing.T, sc *service.Container, doc *test.Doc, label string) *lsp.Hover { + t.Helper() + rng, err := doc.MarkerRange(label) + if err != nil { + t.Fatalf("marker %q: %v", label, err) + } + hoverProvider := service.MustGet[server.HoverProvider](sc) + result, err := hoverProvider.HandleHoverRequest(doc.Ctx(), &lsp.HoverParams{ + TextDocumentPositionParams: lsp.TextDocumentPositionParams{ + TextDocument: lsp.TextDocumentIdentifier{URI: doc.Document.URI.DocumentURI()}, + Position: doc.Document.TextDoc.PositionAt(int(rng.Start)), + }, + }) + if err != nil { + t.Fatalf("hover request at %q failed: %v", label, err) + } + return result +} + +func extractSVG(t *testing.T, markdown string) string { + t.Helper() + idx := strings.Index(markdown, dataImagePrefix) + if idx < 0 { + t.Fatalf("hover content does not contain %q: %s", dataImagePrefix, markdown) + } + encoded := markdown[idx+len(dataImagePrefix):] + if end := strings.IndexByte(encoded, ')'); end >= 0 { + encoded = encoded[:end] + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + t.Fatalf("base64 decode failed: %v", err) + } + var root struct{ XMLName xml.Name } + if err := xml.Unmarshal(decoded, &root); err != nil { + t.Fatalf("embedded SVG is not well-formed XML: %v\n%s", err, decoded) + } + if root.XMLName.Local != "svg" { + t.Fatalf("embedded root element = %q, want svg", root.XMLName.Local) + } + return string(decoded) +} + +const railroadHoverFixtureGrammar = `grammar Test + +interface Foo { + Name string + Items []Item +} + +interface Item { + Name string +} + +entry Foo returns Foo: + "begin" Name=<|idUsage:ID|> Items+=<|itemUsage:Item|>* + +<|itemDecl:Item|> returns Item: + Name=ID + +token ID: /[a-zA-Z_][a-zA-Z0-9_]*/ +` + +func TestRailroadHover_RuleDeclarationEmbedsSVG(t *testing.T) { + sc, f := newHoverFixture(t) + doc := f.Parse(railroadHoverFixtureGrammar) + doc.AssertNoParseErrors() + + hover := hoverAt(t, sc, doc, "itemDecl") + if hover == nil { + t.Fatal("expected a hover result at the rule declaration, got nil") + } + extractSVG(t, hover.Contents.Value) +} + +func TestRailroadHover_UsageResolvesToSameContentAsDeclaration(t *testing.T) { + sc, f := newHoverFixture(t) + doc := f.Parse(railroadHoverFixtureGrammar) + doc.AssertNoParseErrors() + + decl := hoverAt(t, sc, doc, "itemDecl") + usage := hoverAt(t, sc, doc, "itemUsage") + if decl == nil || usage == nil { + t.Fatalf("expected hover results at both positions, got decl=%v usage=%v", decl, usage) + } + if decl.Contents.Value != usage.Contents.Value { + t.Errorf("hover content differs between declaration and usage:\ndecl: %s\nusage: %s", decl.Contents.Value, usage.Contents.Value) + } +} + +func TestRailroadHover_TokenRuleHasNoDiagram(t *testing.T) { + sc, f := newHoverFixture(t) + doc := f.Parse(railroadHoverFixtureGrammar) + doc.AssertNoParseErrors() + + hover := hoverAt(t, sc, doc, "idUsage") + if hover != nil && strings.Contains(hover.Contents.Value, "data:image") { + t.Errorf("hovering a token rule usage should not embed a diagram, got: %s", hover.Contents.Value) + } +} + +const railroadHoverInfixFixtureGrammar = `grammar Test2 + +interface Expr {} + +interface BinaryExpression extends Expr { + Left Expr + Operator string + Right Expr +} + +entry Expr: + PrimaryExpression + +infix <|infixDecl:BinaryExpression|> on PrimaryExpression: + "+" | "-" + > MulOp + +token group MulOp { + "*" + "/" +} + +PrimaryExpression returns Expr: + Value=NUMBER + +token NUMBER: /[0-9]+/ +` + +func TestRailroadHover_InfixRuleEmbedsSVG(t *testing.T) { + sc, f := newHoverFixture(t) + doc := f.Parse(railroadHoverInfixFixtureGrammar) + doc.AssertNoParseErrors() + + hover := hoverAt(t, sc, doc, "infixDecl") + if hover == nil { + t.Fatal("expected a hover result at the infix rule declaration, got nil") + } + extractSVG(t, hover.Contents.Value) +} diff --git a/internal/railroad/choice.go b/internal/railroad/choice.go new file mode 100644 index 00000000..bc4e50cb --- /dev/null +++ b/internal/railroad/choice.go @@ -0,0 +1,120 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package railroad + +import "math" + +// Choice stacks its items vertically, with the item at index Normal drawn +// on the main centerline and the rest branching above (indices < Normal) or +// below (indices > Normal) via quarter-circle arcs. Optional and ZeroOrMore +// are built on top of this. +type Choice struct { + Normal int + Items []Node +} + +func (c *Choice) Width() float64 { + w := 0.0 + for _, it := range c.Items { + w = math.Max(w, it.Width()) + } + return w + arcRadius*4 +} + +func (c *Choice) Up() float64 { + up := 0.0 + for i, it := range c.Items { + switch { + case i < c.Normal: + up += math.Max(arcRadius, it.Up()+it.Down()+verticalSeparation) + case i == c.Normal: + up += math.Max(arcRadius, it.Up()) + } + } + return up +} + +func (c *Choice) Down() float64 { + down := 0.0 + for i, it := range c.Items { + switch { + case i == c.Normal: + down += math.Max(arcRadius, it.Down()) + case i > c.Normal: + down += math.Max(arcRadius, verticalSeparation+it.Up()+it.Down()) + } + } + return down +} + +func (c *Choice) needsSpace() bool { return false } + +func (c *Choice) render(x, y, width float64) *svgElement { + g := el("g") + if len(c.Items) == 0 { + // Defensive: a malformed/partial document could in principle + // produce an Alternatives node with no alternatives. Render as a + // straight passthrough rather than indexing an empty slice. + gapL, gapR := determineGaps(width, c.Width()) + g.add(newPath(x, y).h(gapL).element()) + g.add(newPath(x+gapL+c.Width(), y).h(gapR).element()) + return g + } + + gapL, gapR := determineGaps(width, c.Width()) + g.add(newPath(x, y).h(gapL).element()) + g.add(newPath(x+gapL+c.Width(), y).h(gapR).element()) + x += gapL + + last := len(c.Items) - 1 + innerWidth := c.Width() - arcRadius*4 + + // Items above the main centerline, curving up and back down. + distanceFromY := 0.0 + for i := c.Normal - 1; i >= 0; i-- { + item := c.Items[i] + if i == c.Normal-1 { + distanceFromY = math.Max(arcRadius*2, c.Items[i+1].Up()+verticalSeparation+item.Down()) + } + g.add(newPath(x, y).arc("se").up(distanceFromY - arcRadius*2).arc("wn").element()) + g.add(item.render(x+arcRadius*2, y-distanceFromY, innerWidth)) + g.add(newPath(x+arcRadius*2+innerWidth, y-distanceFromY).arc("ne").down(distanceFromY - arcRadius*2).arc("ws").element()) + prevDown := 0.0 + if i != 0 { + prevDown = c.Items[i-1].Down() + } + distanceFromY += math.Max(arcRadius, item.Up()+verticalSeparation+prevDown) + } + + // The straight-line path through the normal item. + g.add(newPath(x, y).right(arcRadius * 2).element()) + g.add(c.Items[c.Normal].render(x+arcRadius*2, y, innerWidth)) + g.add(newPath(x+arcRadius*2+innerWidth, y).right(arcRadius * 2).element()) + + // Items below the main centerline, curving down and back up. + distanceFromY = 0.0 + for i := c.Normal + 1; i <= last; i++ { + item := c.Items[i] + if i == c.Normal+1 { + distanceFromY = math.Max(arcRadius*2, c.Items[i-1].Down()+verticalSeparation+item.Up()) + } + g.add(newPath(x, y).arc("ne").down(distanceFromY - arcRadius*2).arc("ws").element()) + g.add(item.render(x+arcRadius*2, y+distanceFromY, innerWidth)) + g.add(newPath(x+arcRadius*2+innerWidth, y+distanceFromY).arc("se").up(distanceFromY - arcRadius*2).arc("wn").element()) + nextUp := 0.0 + if i != last { + nextUp = c.Items[i+1].Up() + } + distanceFromY += math.Max(arcRadius, item.Down()+verticalSeparation+nextUp) + } + + return g +} + +// Optional renders item as the through-path with a Skip branch curving +// above it, matching railroad-diagrams@1.0.0's default Optional(item). +func Optional(item Node) Node { + return &Choice{Normal: 1, Items: []Node{Skip{}, item}} +} diff --git a/internal/railroad/diagram.go b/internal/railroad/diagram.go new file mode 100644 index 00000000..3712c644 --- /dev/null +++ b/internal/railroad/diagram.go @@ -0,0 +1,68 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package railroad + +import "math" + +const diagramPadding = 20.0 + +// Diagram is a complete railroad diagram for a single grammar rule: a Start +// marker, the rule's body, and an End marker, laid out left to right. +type Diagram struct { + Item Node +} + +func (d *Diagram) items() []Node { + return []Node{Start{}, d.Item, End{}} +} + +// SVG renders the diagram to a standalone SVG document: no XML declaration +// (unnecessary for an inline/embedded image), a light