General purpose web IDE for MLscript

.github/workflows/nix.yml

@@ -10,6 +10,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Install Nix
         uses: DeterminateSystems/nix-installer-action@main
       - uses: rrbutani/use-nix-shell-action@v1
@@ -30,3 +32,37 @@ jobs:
       - name: Fail if tests failed
         if: steps.run_test.outcome == 'failure'
         run: exit 1
+      - name: Check Web IDE deploy changes
+        if: github.event_name == 'push' && github.ref == 'refs/heads/hkmc2'
+        id: web_ide_changes
+        run: |
+          before="${{ github.event.before }}"
+          after="${{ github.sha }}"
+          if [ "$before" = "0000000000000000000000000000000000000000" ]; then
+            changed=true
+          elif git diff --quiet "$before" "$after" -- wrangler.toml hkmc2/shared/src/test/mlscript-packages/web-ide/; then
+            changed=false
+          else
+            diff_status=$?
+            if [ "$diff_status" -eq 1 ]; then
+              changed=true
+            else
+              exit "$diff_status"
+            fi
+          fi
+          echo "changed=${changed}" >> "$GITHUB_OUTPUT"
+      - name: Build Web IDE deploy assets
+        if: github.event_name == 'push' && github.ref == 'refs/heads/hkmc2' && steps.web_ide_changes.outputs.changed == 'true'
+        run: |
+          sbt -J-Xmx4096M -J-Xss8M hkmc2JS/fullOptJS
+          rm -rf hkmc2/shared/src/test/mlscript-packages/web-ide/build
+          mkdir -p hkmc2/shared/src/test/mlscript-packages/web-ide/build
+          cp -R hkmc2/js/target/scala-*/hkmc2-opt/. hkmc2/shared/src/test/mlscript-packages/web-ide/build/
+      - name: Deploy Web IDE to Cloudflare Workers
+        if: github.event_name == 'push' && github.ref == 'refs/heads/hkmc2' && steps.web_ide_changes.outputs.changed == 'true'
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          wranglerVersion: "4.29.0"
+          command: deploy --config wrangler.toml

.github/workflows/web-ide-preview.yml

@@ -0,0 +1,79 @@
+name: Web IDE Preview
+
+on:
+  push:
+    branches-ignore:
+      - hkmc2
+    paths:
+      - .github/workflows/web-ide-preview.yml
+      - wrangler.toml
+      - hkmc2/shared/src/test/mlscript-packages/web-ide/**
+
+jobs:
+  preview:
+    runs-on: ubuntu-latest
+    env:
+      CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+      CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@main
+      - uses: rrbutani/use-nix-shell-action@v1
+        with:
+          devShell: .#default
+      - name: Install npm dependencies (TypeScript, Binaryen, etc.)
+        run: npm ci
+      - name: Run Web IDE package test
+        run: |
+          sbt -J-Xmx4096M -J-Xss8M hkmc2JVM/test "hkmc2PackagesTest/testOnly hkmc2.PackageTestRunner -- -z web-ide"
+      - name: Check no changes
+        run: |
+          git update-index -q --refresh
+          git diff-files -p --exit-code
+      - name: Build Web IDE deploy assets
+        run: |
+          sbt -J-Xmx4096M -J-Xss8M hkmc2JS/fullOptJS
+          rm -rf hkmc2/shared/src/test/mlscript-packages/web-ide/build
+          mkdir -p hkmc2/shared/src/test/mlscript-packages/web-ide/build
+          cp -R hkmc2/js/target/scala-*/hkmc2-opt/. hkmc2/shared/src/test/mlscript-packages/web-ide/build/
+      - name: Compute preview alias
+        id: preview_alias
+        run: |
+          node <<'NODE'
+          const fs = require("fs");
+          const workerName = "mlscript-web-ide";
+          const maxAliasLength = 63 - workerName.length - 1;
+          const branchName = process.env.GITHUB_REF_NAME || "preview";
+          let alias = branchName
+            .toLowerCase()
+            .replace(/[^a-z0-9-]+/g, "-")
+            .replace(/-+/g, "-")
+            .replace(/^-+|-+$/g, "");
+          if (!alias) alias = "preview";
+          if (!/^[a-z]/.test(alias)) alias = `branch-${alias}`;
+          if (alias.length > maxAliasLength) {
+            alias = alias.slice(0, maxAliasLength).replace(/-+$/g, "");
+          }
+          fs.appendFileSync(process.env.GITHUB_OUTPUT, `alias=${alias}\n`);
+          console.log(`Preview alias: ${alias}`);
+          NODE
+      - name: Enable Web IDE preview URLs
+        if: env.CLOUDFLARE_API_TOKEN != '' && env.CLOUDFLARE_ACCOUNT_ID != ''
+        run: |
+          curl --fail -X POST \
+            -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
+            -H "Content-Type: application/json" \
+            --data '{"enabled":false,"previews_enabled":true}' \
+            "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/workers/scripts/mlscript-web-ide/subdomain"
+      - name: Upload Web IDE preview to Cloudflare Workers
+        if: env.CLOUDFLARE_API_TOKEN != '' && env.CLOUDFLARE_ACCOUNT_ID != ''
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          wranglerVersion: "4.29.0"
+          command: versions upload --preview-alias ${{ steps.preview_alias.outputs.alias }} --config wrangler.toml
+      - name: Skip Web IDE preview upload
+        if: env.CLOUDFLARE_API_TOKEN == '' || env.CLOUDFLARE_ACCOUNT_ID == ''
+        run: echo "Cloudflare preview secrets are not configured; skipping preview upload."

README.md

@@ -172,6 +172,8 @@ and then use one of the following commands.
 - `hkmc2DiffTests/test` for running only the main diff-tests, in `hkmc2/shared/src/test/mlscript`.
 - `hkmc2MainTests/test` for running the above two.
 - `hkmc2AppsTests/test` for running the applications compile and diff tests.
+- `hkmc2PackagesTest/test` for compiling the packages in `hkmc2/shared/src/test/mlscript-packages`.
+  Each folder represents a package, which can be imported directly using its name.
 - `hkmc2NofibTests/test` for running the nofib compile and diff tests.
 - `hkmc2WasmTests/test` for running the wasm compile and diff tests.
 - `hkmc2LlirTests/test` for running the llir diff tests.
@@ -224,4 +226,3 @@ Links to the individual paper artifact repositories are provided at the correspo
 
 
 
-

build.sbt

@@ -64,6 +64,48 @@ lazy val hkmc2 = crossProject(JSPlatform, JVMPlatform).in(file("hkmc2"))
       _.withModuleKind(ModuleKind.ESModule)
         .withOutputPatterns(OutputPatterns.fromJSFile("MLscript.mjs"))
     },
+    Compile / sourceGenerators += Def.task {
+      val rootDir = (ThisBuild / baseDirectory).value
+      val stdDir = rootDir / "hkmc2" / "shared" / "src" / "test" / "mlscript-compile"
+      val declsDir = rootDir / "hkmc2" / "shared" / "src" / "test" / "mlscript" / "decls"
+      val out = (Compile / sourceManaged).value / "hkmc2" / "WebIDEStd.scala"
+      val preludeFile = declsDir / "Prelude.mls"
+      val stdFiles = ((stdDir * "*.mls") +++ (stdDir * "*.mjs")).get
+        .filterNot(_.getName == "Prelude.mls")
+        .sortBy(_.getName)
+
+      def scalaString(value: String): String =
+        "\"" + value.flatMap {
+          case '\\' => "\\\\"
+          case '"' => "\\\""
+          case '\n' => "\\n"
+          case '\r' => "\\r"
+          case '\t' => "\\t"
+          case c if c.isControl => f"\\u${c.toInt}%04x"
+          case c => c.toString
+        } + "\""
+
+      val entries = stdFiles.map { file =>
+        s"""js.Array(${scalaString("/std/" + file.getName)}, ${scalaString(IO.read(file))})"""
+      }
+      val source =
+        s"""|package hkmc2
+            |
+            |import scala.scalajs.js
+            |import scala.scalajs.js.annotation.JSExportTopLevel
+            |
+            |object WebIDEStd:
+            |  @JSExportTopLevel("std")
+            |  val std: js.Dynamic = js.Dynamic.literal(
+            |    prelude = ${scalaString(IO.read(preludeFile))},
+            |    files = js.Array(
+            |      ${entries.mkString(",\n      ")}
+            |    )
+            |  )
+            |""".stripMargin
+      IO.write(out, source)
+      Seq(out)
+    }.taskValue,
     libraryDependencies += "org.scala-js" %%% "scalajs-dom" % "2.2.0",
   )
   .dependsOn(core)
@@ -116,6 +158,20 @@ lazy val hkmc2AppsTests = hkmc2TestSubproject("hkmc2AppsTests", Some("AppsCompil
 lazy val hkmc2WasmTests = hkmc2TestSubproject("hkmc2WasmTests", Some("WasmCompileTestRunner"), "WasmDiffTestRunner")
 lazy val hkmc2LlirTests = hkmc2TestSubproject("hkmc2LlirTests", None, "LlirDiffTestRunner")
 
+lazy val hkmc2PackagesTest = project.in(file("hkmc2PackagesTest"))
+  .dependsOn(hkmc2JVM % "compile->compile;test->test")
+  .settings(
+    scalaVersion := scala3Version,
+
+    libraryDependencies += "org.scalactic" %%% "scalactic" % scalaTestVersion,
+    libraryDependencies += "org.scalatest" %%% "scalatest" % scalaTestVersion % "test",
+    libraryDependencies += "com.lihaoyi" %% "ujson" % "4.4.3",
+
+    Test / test := (Test / testOnly).toTask(" hkmc2.PackageTestRunner").value,
+
+    Test/run/fork := true, // so that CTRL+C actually terminates the watcher
+  )
+
 lazy val hkmc2MainTests = project.in(file("hkmc2MainTests"))
   .settings(
     Test / test := (
@@ -130,19 +186,21 @@ lazy val hkmc2MostTests = project.in(file("hkmc2MostTests"))
       (hkmc2DiffTests / Test / test)
         .dependsOn(hkmc2NofibTests / Test / test)
         .dependsOn(hkmc2AppsTests / Test / test)
+        .dependsOn(hkmc2PackagesTest / Test / test)
         .dependsOn(hkmc2WasmTests / Test / test)
         .dependsOn(hkmc2JVM / Test / test)
     ).value
   )
 
 lazy val hkmc2AllTests = project.in(file("hkmc2AllTests"))
   .settings(
-    Test / test := (
+    Test / test := Def.sequential(
+      hkmc2JVM / Test / test, // prepares compile-test `.mjs` outputs used by JS tests
       (hkmc2DiffTests / Test / test)
         .dependsOn(hkmc2NofibTests / Test / test)
         .dependsOn(hkmc2AppsTests / Test / test)
+        .dependsOn(hkmc2PackagesTest / Test / test)
         .dependsOn(hkmc2WasmTests / Test / test)
-        .dependsOn(hkmc2JVM / Test / test)
         .dependsOn(hkmc2JS / Test / test)
         .dependsOn(hkmc2Benchmarks / Test / compile)
     ).value