diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2720da1d7..c5a074c37 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -266,3 +266,95 @@ jobs: # The tag the publish job verified against protected main, not the raw input. RELEASE_TAG: ${{ needs.publish.outputs.tag }} run: gh release upload "${RELEASE_TAG}" "${RUNNER_TEMP}"/installers/* --repo "${GITHUB_REPOSITORY}" --clobber + + distribution: + # The release distribution (S3): gentle-pi at the verified tag and the exact + # Pi, with the pnpm-lock.yaml our pinned pnpm resolves for them. It needs the + # npm publication, since gentle-pi@ must resolve from the registry. + needs: publish + if: github.repository == 'Gentleman-Programming/gentle-shell' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + ref: ${{ github.sha }} + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Build the distribution assets + env: + # The tag the publish job verified against protected main, not the raw input. + RELEASE_TAG: ${{ needs.publish.outputs.tag }} + run: node scripts/build-distribution.mjs build --shell "${RELEASE_TAG#v}" --out "${RUNNER_TEMP}/distribution" + + - name: Keep the assets for verification + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: gentle-shell-distribution + path: ${{ runner.temp }}/distribution/ + if-no-files-found: error + retention-days: 7 + + distribution-verify: + # The exact assets install with --frozen-lockfile on every system, with our + # pinned Node and pnpm (and Go on Windows), and the launcher runs them. + needs: distribution + if: github.repository == 'Gentleman-Programming/gentle-shell' + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + ref: ${{ github.sha }} + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Fetch the distribution assets + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: gentle-shell-distribution + path: ${{ runner.temp }}/distribution + + - name: Install them frozen and run the launcher + shell: bash + run: node scripts/build-distribution.mjs verify --assets "${RUNNER_TEMP}/distribution" + + distribution-assets: + # Attached only after every verify lane passed, with stable names. + needs: [publish, distribution-verify] + if: github.repository == 'Gentleman-Programming/gentle-shell' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Fetch the verified distribution assets + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: gentle-shell-distribution + path: ${{ runner.temp }}/distribution + + - name: Attach the distribution assets to the release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.publish.outputs.tag }} + run: gh release upload "${RELEASE_TAG}" "${RUNNER_TEMP}/distribution/gentle-shell-distribution.json" "${RUNNER_TEMP}/distribution/gentle-shell-distribution-lock.yaml" --repo "${GITHUB_REPOSITORY}" --clobber diff --git a/docs/bundled-install.md b/docs/bundled-install.md index b01f5c0dc..62518dfbb 100644 --- a/docs/bundled-install.md +++ b/docs/bundled-install.md @@ -63,7 +63,66 @@ These settings mean: `installVersion` then runs `node pnpm.mjs install --frozen-lockfile` in the version folder. -After an install that skips build scripts, pnpm 11 rewrites `pnpm-workspace.yaml` in place. It adds each skipped package under `allowBuilds` with the placeholder value `set this to true or false`. The release lockfile (T1b) must therefore be resolved from the unmodified `distributionFiles()` output. It must never come from a `pnpm-workspace.yaml` read back from a folder pnpm already installed into. +After an install that skips build scripts, pnpm 11 rewrites `pnpm-workspace.yaml` in place. It adds each skipped package under `allowBuilds` with the placeholder value `set this to true or false`. The release lockfile is therefore resolved from the unmodified `distributionFiles()` output. It never comes from a `pnpm-workspace.yaml` read back from a folder pnpm already installed into. + +## Release distribution assets + +Each release publishes two assets with stable names. `scripts/build-distribution.mjs` builds and verifies them. + +| Asset | Contents | +|---|---| +| `gentle-shell-distribution.json` | `schema` (1), `shell` (gentle-pi), `pi`, `id` (`-`), `generatedWith` (our Node and pnpm pins), `files` (the exact `package.json` and `pnpm-workspace.yaml` that `installVersion` writes) and `lockfile` (`name` and `sha256`). | +| `gentle-shell-distribution-lock.yaml` | The `pnpm-lock.yaml` for those files. | + +- **Exact versions.** `shell` is the release version. `pi` is `PI_INSTALL_VERSION` from `scripts/installer-preflight.mjs`, the one Pi pin. A range, a tag such as `latest`, a `v` prefix or build metadata is refused before pnpm runs. +- **Resolution.** `buildDistribution` writes `distributionFiles()` into a fresh folder under the prefix's `tmp/` and runs our pinned pnpm with `install --lockfile-only` in `pnpmEnvironment(layout)`, the environment `installVersion` uses. Only `pnpm-lock.yaml` is read back. +- **Just published.** The registry may not list `gentle-pi@` for a few minutes after `npm publish`. When pnpm reports `ERR_PNPM_NO_MATCHING_VERSION` for exactly that gentle-pi (pnpm prints it on stdout), the resolution is retried in a fresh folder: 10 attempts, 30 seconds apart, about five minutes in all. It then fails with `gentle-pi@ is not on the registry after 10 attempts`. Any other failure, including a missing Pi, stops at once. +- **Deterministic.** `distributionAsset(manifest, lockfile)` is plain data in a fixed order, so the same inputs give the same bytes. The lockfile itself records the registry at resolution time, which is why it is built once per release and then only installed. +- **Reading.** `readDistribution(json, lockfile)` returns `{ manifest, lockfile }` only when the versions are exact, the lockfile matches the recorded `sha256`, and `files` equals what this module's `installVersion` writes. `generatedWith` is recorded and not compared. + +### One lockfile for every system + +pnpm resolves every optional dependency into the lockfile, with its `os`, `cpu` and `libc` fields, whatever system resolves it. Only the install skips the packages for other systems. So no `supportedArchitectures` setting is needed, and none is set. + +Observed with pnpm 11.1.1 for gentle-pi 4.0.0 and Pi 1.0.0: + +- A lockfile resolved on macOS arm64 lists every `@yuuang/ffi-rs-*`, `@ff-labs/fff-bin-*` and `@esbuild/*` variant for Linux (gnu and musl), Windows and macOS. +- A `--frozen-lockfile` install of that lockfile on macOS with `supportedArchitectures` forced to Linux and Windows installed the Linux and Windows variants. The lockfile was left unchanged. +- The regular frozen install on macOS installed only the darwin-arm64 variants. + +### Minimum release age + +pnpm 11.1.1 defaults `minimum-release-age` to 1440 minutes (one day), in non-strict mode. The default is kept on purpose, and no value is set: + +- **Ranged dependencies.** For a ranged transitive dependency, pnpm picks the highest matching version that is at least one day old. A version published minutes before a release does not enter the lockfile. This is a supply-chain guard. +- **Exact pins.** gentle-pi and Pi are exact. In non-strict mode, when no matching version is old enough, pnpm falls back to the matching version anyway (`pickRespectingMinReleaseAge`). So a gentle-pi published minutes ago still resolves. +- **Never set it explicitly.** pnpm 11.1.1 turns `minimumReleaseAgeStrict` on whenever `minimumReleaseAge` is set explicitly and the strict flag is not. A strict check would refuse the gentle-pi the release just published. + +Observed with pnpm 11.1.1: + +- `pnpm_config_minimum_release_age=5256000` alone failed. +- The same value with `pnpm_config_minimum_release_age_strict=false` resolved gentle-pi 4.0.0 exactly. +- The default resolved it too. + +`pnpmEnvironment` drops every inherited `pnpm_*` variable, so a host setting cannot change this. + +### Release workflow + +`.github/workflows/publish.yml` runs three jobs after `publish`, because `gentle-pi@` must resolve from npm first: + +1. **`distribution`** checks out the verified release commit. It runs `build-distribution.mjs build --shell --out ` with our pinned Node and pnpm in a temporary prefix, and keeps both files as a workflow artifact. +2. **`distribution-verify`** runs on `ubuntu-latest`, `macos-latest` and `windows-latest`. It runs `build-distribution.mjs verify --assets `, which: + - downloads our pinned Node and pnpm into a temporary prefix (with Go on Windows); + - installs the exact pair with `installVersion` (`--frozen-lockfile`); + - switches `current` and writes the launcher; + - requires `gentle-shell --version` to report exactly `gentle-shell ` and `pi `. + + On Windows a fresh temporary folder stands in for the private-folder claim, which has its own native tests. The launcher runs in `pnpmEnvironment`, the environment our pnpm ran in, without any `PI_*` or `GENTLE_*` variable. So no host `NODE_OPTIONS`, `NODE_PATH`, npm or pnpm setting, or Pi override reaches it. +3. **`distribution-assets`** runs only after every verify lane passed. It attaches both files to the release with `gh release upload --clobber`. + +Both `build` and `verify` create their temporary prefix (`gsd-*` in the system temporary folder) only after their arguments are accepted. They always remove it, whether the command succeeded or failed. + +The release gets the same bytes that the three systems installed. ## npm is not needed diff --git a/odd/tasks/bundled-gentle-shell.md b/odd/tasks/bundled-gentle-shell.md index da173900b..b2564dc1e 100644 --- a/odd/tasks/bundled-gentle-shell.md +++ b/odd/tasks/bundled-gentle-shell.md @@ -15,10 +15,11 @@ Branch: feat/bundled-gentle-shell (design approved; one work-unit commit and PR - S10. Removal: the installer's user-tool detection and reuse logic (user Node/npm/pnpm/Go probes, shim parsers, PATHEXT, Corepack/mise, persistence variants, globalBin pnpm rules, private PNPM_HOME decision, reused-folder notice, npm prefix configuration) is deleted once the bundled path ships; the download/verification of the pinned Node, pnpm and Go and the Windows private-folder claim are reused. - S11. Windows Go: today the Windows Gentle AI binary is built from source in the postinstall (no signed Windows archive). Publishing a signed Windows Gentle AI binary in gentle-ai releases would remove Go from the Windows release path (cross-repo follow-up). +- S12. User decision (verbatim): "Dale vamos con tu recomendación" — the parent's recommendation: our isolated pnpm reads only the network, registry and authentication keys of the user's ~/.npmrc (registry and scoped registries, per-host auth tokens and credentials, proxy, https-proxy, noproxy, ca/cafile/strict-ssl), copied into the prefix's npmrc auth file at install time; every other key is ignored. The frozen lockfile still pins every version and integrity hash, so these keys cannot change what is installed. Implemented with T1c. ## Tasks - T1a | S2,S3,S6,S7 | delegated writer (context backstop) | committed 55efdf527; verifier corrections (B1-B3, A1, A2, A4, A5, A9) implemented, awaiting parent review, CI gate update (31 to 35 native tests) and commit | Shared bundled-install module (used by the web installer and `gentle-shell upgrade`): claim the prefix (POSIX private dir; Windows the bootstrap's private-folder claim), install pinned Node and pnpm (and Go only when asked) into runtime/ reusing the verified downloads, pnpm store/cache/state/config inside the prefix, install a version into versions/-/ from a distribution manifest + frozen lockfile, atomic `current` switch keeping 2 versions, the one PATH entry and the bin/gentle-shell shim. Unit tests with injected download/run adapters; no release wiring yet. -- T1b | S3 | pending | Release CI: build and publish per release the distribution manifest (gentle-pi + exact Pi) and its pnpm-lock.yaml as release assets, verified by a frozen install in CI on the three OSes. -- T1c | S1,S2,S3 | pending | Web installer uses the bundled-install module for new installations (release channel first). +- T1b | S3 | delegated writer; committed b9b181a6e on feat/bundled-release-lockfile; diag runs 38092721527, 38092811888 and 38092917737: ubuntu and macOS pass, Windows blocked by the pinned Go download defect fixed in #2064 (rerun after main is merged); verifier advisories A1-A5 corrected, awaiting parent review and commit | Release CI: build and publish per release the distribution manifest (gentle-pi + exact Pi) and its pnpm-lock.yaml as release assets, verified by a frozen install in CI on the three OSes. +- T1c | S1,S2,S3,S12 | pending | Web installer uses the bundled-install module for new installations (release channel first). - T2 | S8,S5 | pending | Launcher in bundled mode: exact Pi from `current`, no fallbacks; `pi update`/`gentle-shell update` redirect; `gentle-shell upgrade` = new version + verify + switch + keep 2. - T3 | S4 | pending | Migration from global pnpm/npm installs with one confirmation. - T4 | S9 | pending | Isolation leaks (pi-pretty fff path, gentle-ai state, Engram npm). @@ -34,3 +35,7 @@ Branch: feat/bundled-gentle-shell (design approved; one work-unit commit and PR - L5. User (2026-10-10, verbatim): "Si" — approves the design, including the parent's three proposals: the ~/.gentle-shell layout (Windows: the bootstrap's private folder), keeping 2 versions for rollback, and no fallback to another Pi in the bundled launcher (fail with a clear message). T1 split into T1a (module), T1b (release CI lockfile), T1c (web installer wiring). - L6. T1a (writer): scripts/bundled-install.mjs + tests/bundled-install.test.ts (19 tests, test-first) + docs/bundled-install.md; installer-windows.mjs exports ensureWindowsPrivateFolder (the PNPM_HOME claim with folder/marker as data; ensureWindowsPnpmHome unchanged). Evidence from pnpm 11.1.1 sources and a real run: pnpm reads `pnpm_config_*` env vars, its config folder only from XDG_CONFIG_HOME (or the OS default), and ~/.npmrc unless `pnpm_config_npmrc_auth_file` is set; `--allow-build` exists only for `add`, so `install` takes `allowBuilds` from pnpm-workspace.yaml, which also stops the upward workspace search; pnpm 11 fails ignored builds by default (strictDepBuilds) and Pi's tree has build scripts, so the distribution sets `strictDepBuilds: false`; lifecycle scripts use the `node` on PATH. npm is not needed (pnpm runs on node; node-gyp is bundled). Real check: pinned Node 24.21.0 (darwin tar.gz, win zip) and pnpm 11.1.1 verified and run; lockfile-only + frozen install of gentle-pi 4.0.0 + Pi 1.1.0 ran the postinstall, published by rename, and the launcher printed both versions with only ~/.gentle-shell written under HOME. Decisions for review: Go lives at runtime/go//go (acquireGo layout) instead of runtime/go-/; Windows fallback folder is %USERPROFILE%\.gentle-shell-bundle (the agent home's ~/.gentle-shell is not empty, so it cannot be claimed); launchers are per platform (sh on POSIX, .cmd on Windows); the HKCU registry adapter has no default yet (T1c). Follow-ups: T1b must resolve the lockfile from distributionFiles(); verify-package-files should list scripts/bundled-install.mjs. - L7. T1a corrections (writer), after the independent verifier's blockers. B3: installVersion installs in place in versions/ (claimed by mkdir, marker `installing ` replaced by `` with one rename only after verification; a dead-process leftover of the same id is removed, a live one stops the install, anything else is never replaced), because pnpm 11.1.1 falls back to absolute-path junctions on Windows (symlink-dir resolveSrcOnWinJunction) that a rename would leave dangling; runtime/ keeps rename publication since its extracted trees hold no links. B1: a profile is edited only when it is a regular, writable file in real folders under home (or absent and creatable there); a symlink, read-only file, non-file or linked folder yields `kind: manual` with the exact line, and a line already present through a link yields `none`; appends use O_NOFOLLOW and a newline separator, and applyPathEntry returns a receipt so removePathEntry restores the file byte for byte. B2: macOS bash uses the first existing of ~/.bash_profile, ~/.bash_login, ~/.profile (bash(1) INVOCATION) and creates ~/.bash_profile only when none exists; a created file is deleted when empty after removal; checked with a real /bin/bash -l. A1: the pnpm tarball's executable bits are kept (owner only). A2: `manage-package-manager-versions` does not exist in pnpm 11.1.1; observed `pnpm --version` printing 9.0.0 under a package.json with packageManager pnpm@9.0.0 because pm-on-fail defaults to download, so the env sets pnpm_config_pm_on_fail=ignore (and runtime_on_fail=ignore). A4: documented that pnpm rewrites pnpm-workspace.yaml with `set this to true or false` placeholders for skipped builds, so T1b resolves the lockfile from distributionFiles() only. A5: pruneVersions deletes nothing unless current names an installed version. A9: installVersion on Windows refuses to run without the pinned Go under runtime/go. Launchers print a clear message and exit 1 without an active version; the .cmd uses DisableDelayedExpansion. Four native Windows tests added (real pinned Node and pnpm downloaded once and verified, local fixture packages via an injected `distribution` adapter); the native total becomes 35, so .github/workflows/ci.yml and the structural test in tests/installer-posix-bootstrap.test.ts must move from 31 to 35. +- L8. T1b (writer, S3): scripts/build-distribution.mjs `build --shell [--pi ] --out ` resolves the lockfile with our pinned pnpm (`install --lockfile-only` in pnpmEnvironment, from the unmodified distributionFiles(), only pnpm-lock.yaml read back) and writes `gentle-shell-distribution.json` (schema 1, shell, pi, id, generatedWith node/pnpm, files, lockfile name + sha256; built by distributionAsset in bundled-install.mjs, read by readDistribution, which T1c uses) and `gentle-shell-distribution-lock.yaml`; Pi comes from PI_INSTALL_VERSION, the existing single pin; non-exact versions are refused before pnpm runs. `verify --assets ` installs the pair in a temporary prefix (Go on Windows), activates it and requires `gentle-shell --version` to print exactly both versions. publish.yml: publish → distribution (ubuntu) → distribution-verify (ubuntu, macOS, Windows, fail-fast false) → distribution-assets (`gh release upload --clobber`, needs publish and every verify lane); existing jobs unchanged; upload/download-artifact pinned by SHA. Evidence with the real pnpm 11.1.1 (gentle-pi 4.0.0 + Pi 1.0.0, isolated HOME and TMPDIR): pnpm records every platform's optional packages (ffi-rs, fff-bin, esbuild; gnu and musl) whatever system resolves, so no supportedArchitectures is set; a macOS frozen install forced to linux/win32 installed those variants with the lockfile unchanged; the real verify installed 185 packages (darwin-arm64 variants only), ran gentle-ai's postinstall (v4.0.0) and the launcher printed `gentle-shell 4.0.0` / `pi 1.0.0`; pnpm rewrote the installed pnpm-workspace.yaml with placeholders (A4 confirmed) while the lockfile stayed byte-identical. Follow-ups: the release skill checks both assets; the Windows lane (Go source build of Gentle AI, path lengths) is unproven until the first release run. +- L9. T1b corrections (writer), after the independent verifier. Its only blocker, the Windows lane failing with "Download truncated" on the pinned Go, is the installer-downloads.mjs defect fixed separately in #2064 (dl.google.com gzipped the archive; fetch now asks for the identity encoding). The diag runs 38092721527, 38092811888 and 38092917737 passed on ubuntu and macOS. Advisories: A2, the CLI creates its `gsd-*` prefix only after its arguments are accepted and always removes it (finally, with retries for Windows), on success and failure, for build and verify; `main` is exported with injectable platform, env, temporary root and adapters. A3, every temporary folder the build-distribution tests create is removed by `t.after` (tests/bundled-install.test.ts still leaves about 30 `bundled-install-*` folders per run through its `home()` helper, outside this task's edit surfaces). A4, the launcher check runs in pnpmEnvironment (no host NODE_OPTIONS, NODE_PATH, npm_*/pnpm_*) minus PI_*/GENTLE_*. A1, pnpm 11.1.1 defaults minimum-release-age to 1440 minutes, non-strict; it is kept and documented, not set: ranged transitive dependencies resolve to versions at least one day old (a supply-chain guard), exact pins fall back to a younger matching version (pickRespectingMinReleaseAge), and an explicit minimumReleaseAge turns minimumReleaseAgeStrict on (observed: an explicit age failed, the same age with strict false resolved gentle-pi 4.0.0). `build` retries only `ERR_PNPM_NO_MATCHING_VERSION` for this exact gentle-pi (pnpm prints it on stdout), 10 attempts 30 seconds apart, then fails with `gentle-pi@ is not on the registry after 10 attempts`; a missing Pi or another version fails at once. Real run with an isolated HOME and TMPDIR: build and verify of 4.0.0 + 1.0.0 passed, left the temporary folder empty (also after a failing verify), ignored a host npm_config_registry, and reproduced the same lockfile sha256 as L8 (fc9afd56...). +- L10. User (2026-10-11, verbatim): "Dale vamos con tu recomendación" — read only the network/registry/auth keys of the user's ~/.npmrc (S12). +- L11. 3-OS diag of the release distribution (runs 38093755486, 38094010400) after #2064 (identity download): ubuntu-latest and macos-latest install frozen and the launcher prints `gentle-shell 4.0.0 / pi 1.0.0`; windows-latest acquires the pinned Go, installs all 184 packages, then the published gentle-pi 4.0.0 postinstall fails building Gentle AI from source — the deep-package-path Go build fixed in #2035, which is on main but not yet in a published gentle-pi. The next release ships the fix; a failing lane never uploads assets. installVersion errors now carry pnpm's stdout too (pnpm prints install and postinstall failures there). diff --git a/scripts/build-distribution.mjs b/scripts/build-distribution.mjs new file mode 100644 index 000000000..f02136619 --- /dev/null +++ b/scripts/build-distribution.mjs @@ -0,0 +1,157 @@ +#!/usr/bin/env node +// Release distribution assets (S3, T1b): gentle-pi at the release version and +// the exact Pi (PI_INSTALL_VERSION), with the pnpm-lock.yaml our pinned pnpm +// resolves for them, published as two release assets that installVersion +// installs with --frozen-lockfile. pnpm records every platform's optional +// packages (ffi-rs, fff-bin, esbuild prebuilds) in the lockfile whatever system +// resolves it, so one lockfile serves Linux, macOS and Windows. +// +// node scripts/build-distribution.mjs build --shell [--pi ] --out +// node scripts/build-distribution.mjs verify --assets +// +// Both commands work in a fresh temporary prefix with our own pinned Node and +// pnpm (and Go on Windows); the user's tools and home are never used. +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; +import { DISTRIBUTION_ASSETS, activateVersion, claimPrefix, distributionAsset, distributionFiles, ensureLauncher, ensureRuntime, installVersion, + pnpmEnvironment, readDistribution, runCommand } from "./bundled-install.mjs"; +import { PI_INSTALL_VERSION } from "./installer-preflight.mjs"; + +const RESOLVE_TIMEOUT = 10 * 60 * 1000; +// Right after `npm publish` the registry may not list gentle-pi@ yet: +// 10 attempts 30 seconds apart wait about five minutes for it. +const RESOLVE_ATTEMPTS = 10; +const RESOLVE_DELAY = 30 * 1000; +const output = (result) => `${result.stdout ?? ""}${result.stderr ?? ""}`.slice(-4000); + +/** Resolves the lockfile for gentle-pi `shell` and Pi `pi` with our pnpm + * (`install --lockfile-only`) in a fresh folder under the prefix, from the + * unmodified distributionFiles() output and the same environment installVersion + * uses. Only "no matching version" for this gentle-pi is retried (pnpm prints it + * on stdout); any other failure stops at once. Returns { manifest, assets } + * keyed by the release asset names. + */ +export async function buildDistribution({ layout, shell, pi = PI_INSTALL_VERSION, env = {}, adapters = {} }) { + const manifest = { shell, pi }; + const files = distributionFiles(manifest); + const sleep = adapters.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + // `shell` is an exact SemVer here, so only its dots need escaping. + const unpublished = new RegExp(`ERR_PNPM_NO_(?:MATURE_)?MATCHING_VERSION\\] No matching version found for gentle-pi@${shell.replaceAll(".", "\\.")}(?![\\w.+-])`); + for (let attempt = 1; ; attempt += 1) { + const folder = mkdtempSync(join(layout.tmp, "lockfile-")); + try { + for (const [name, text] of Object.entries(files)) writeFileSync(join(folder, name), text, { flag: "wx", mode: 0o600 }); + const result = await (adapters.run ?? runCommand)(layout.node, [layout.pnpm, "install", "--lockfile-only"], + { cwd: folder, env: pnpmEnvironment(layout, { env }), timeout: RESOLVE_TIMEOUT }); + if (result.status === 0) { + // Only the lockfile is read back: pnpm may rewrite pnpm-workspace.yaml with placeholders. + const lockfile = readFileSync(join(folder, "pnpm-lock.yaml"), "utf8"); + return { manifest, assets: { [DISTRIBUTION_ASSETS.distribution]: distributionAsset(manifest, lockfile), [DISTRIBUTION_ASSETS.lockfile]: lockfile } }; + } + if (!unpublished.test(output(result))) throw Object.assign(new Error("pnpm lockfile resolution failed"), { output: output(result) }); + if (attempt === RESOLVE_ATTEMPTS) { + throw Object.assign(new Error(`gentle-pi@${shell} is not on the registry after ${RESOLVE_ATTEMPTS} attempts; check its npm publication`), { output: output(result) }); + } + } finally { + rmSync(folder, { recursive: true, force: true }); + } + await sleep(RESOLVE_DELAY); + } +} + +/** Runs the launcher with `--version`; the .cmd one through cmd.exe. */ +function launch(launcher, { env, platform }) { + const options = { env, encoding: "utf8", timeout: 120000, windowsHide: true }; + if (platform !== "win32") return spawnSync(launcher, ["--version"], options); + const cmd = join(env.SystemRoot ?? env.SYSTEMROOT ?? "C:\\Windows", "System32", "cmd.exe"); + return spawnSync(cmd, ["/d", "/s", "/c", `""${launcher}" --version"`], { ...options, windowsVerbatimArguments: true }); +} + +/** Proves a release asset pair: our runtimes (Go too on Windows, as + * installVersion requires), a frozen install of the pair, `current` switched to + * it, the launcher written, and `gentle-shell --version` reporting exactly the + * pair's gentle-pi and Pi. Returns { id, version }. + */ +export async function verifyDistribution({ layout, distribution, lockfile, platform, arch, env = {}, adapters = {} }) { + const { manifest } = readDistribution(distribution, lockfile); + const runtime = await ensureRuntime({ layout, platform, arch, go: platform === "win32", env, adapters }); + const { id } = await installVersion({ layout, manifest, lockfile, platform, env, go: runtime.go, adapters }); + activateVersion(layout, id); + ensureLauncher(layout); + // The environment our pnpm ran in (no NODE_OPTIONS, NODE_PATH or npm/pnpm + // settings from the host), without any Pi or Gentle override that could point + // the launcher at another Pi or agent home. + const launchEnv = Object.fromEntries(Object.entries(pnpmEnvironment(layout, { platform, env })).filter(([key]) => !/^(?:gentle_|pi_)/i.test(key))); + const result = await (adapters.launch ?? launch)(layout.launcher, { env: launchEnv, platform }); + const version = String(result.stdout ?? "").trim(); + const [shellLine, piLine] = version.split(/\r?\n/); + if (result.status !== 0 || shellLine !== `gentle-shell ${manifest.shell}` || piLine !== `pi ${manifest.pi}`) { + throw Object.assign(new Error(`The launcher reported ${JSON.stringify(version)}, not gentle-shell ${manifest.shell} with pi ${manifest.pi}`), + { output: output(result) }); + } + return { id, version }; +} + +/** A fresh prefix in a new temporary folder. On Windows the private-folder claim + * (owner, ACL, marker) has its own native tests; here the new folder stands in + * for it, as in those tests. + */ +function temporaryPrefix(root, platform, base) { + if (platform !== "win32") { + const home = join(root, "home"); + mkdirSync(home, { mode: 0o700 }); + const env = { ...base, HOME: home }; + return { env, layout: claimPrefix({ platform, env, home }) }; + } + const [local, roaming, profile] = ["local", "roaming", "profile"].map((name) => join(root, name)); + for (const folder of [local, roaming, profile]) mkdirSync(folder); + const kept = Object.entries(base).filter(([key]) => !/^(?:localappdata|appdata|userprofile|home)$/i.test(key)); + const env = { ...Object.fromEntries(kept), LOCALAPPDATA: local, APPDATA: roaming, USERPROFILE: profile, HOME: profile }; + const claim = (folder) => { mkdirSync(folder); return "claimed"; }; + return { env, layout: claimPrefix({ platform, env, home: profile, adapters: { storage: () => {}, claim } }) }; +} + +/** The CLI. Its temporary prefix (`gsd-*`) is always removed, whether the + * command succeeded or failed; invalid input is refused before it exists. + */ +export async function main(argv, { platform = process.platform, arch = process.arch, env = process.env, temporaryRoot = tmpdir(), adapters = {} } = {}) { + const { positionals: [command], values } = parseArgs({ args: argv, allowPositionals: true, + options: { shell: { type: "string" }, pi: { type: "string" }, out: { type: "string" }, assets: { type: "string" } } }); + const build = command === "build" && values.shell && values.out; + if (!build && !(command === "verify" && values.assets)) { + throw new Error("Usage: build-distribution.mjs build --shell [--pi ] --out | verify --assets "); + } + // A non-exact version is refused before anything is downloaded. + if (build) distributionFiles({ shell: values.shell, pi: values.pi ?? PI_INSTALL_VERSION }); + const root = realpathSync(mkdtempSync(join(temporaryRoot, "gsd-"))); + try { + const prefix = temporaryPrefix(root, platform, env); + if (build) { + await ensureRuntime({ layout: prefix.layout, platform, arch, env: prefix.env, adapters }); + const { manifest, assets } = await buildDistribution({ layout: prefix.layout, shell: values.shell, pi: values.pi, env: prefix.env, adapters }); + mkdirSync(values.out, { recursive: true }); + for (const [name, text] of Object.entries(assets)) writeFileSync(join(values.out, name), text, { flag: "wx" }); + console.log(`Built ${Object.keys(assets).join(" and ")} for gentle-pi ${manifest.shell} with pi ${manifest.pi} in ${values.out}`); + } else { + const read = (name) => readFileSync(join(values.assets, name), "utf8"); + const result = await verifyDistribution({ layout: prefix.layout, distribution: read(DISTRIBUTION_ASSETS.distribution), lockfile: read(DISTRIBUTION_ASSETS.lockfile), + platform, arch, env: prefix.env, adapters }); + console.log(`Verified ${result.id}:\n${result.version}`); + } + } finally { + // Windows may hold a file a moment after its process exits. + rmSync(root, { recursive: true, force: true, maxRetries: 5 }); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) { + main(process.argv.slice(2)).catch((error) => { + const detail = error.output ?? error.stderr; + console.error(detail ? `${error.message}\n${detail}` : error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/bundled-install.mjs b/scripts/bundled-install.mjs index da8441b1f..189a3a65f 100644 --- a/scripts/bundled-install.mjs +++ b/scripts/bundled-install.mjs @@ -4,7 +4,7 @@ // entry. The user's node, npm, pnpm and go are never run, read or changed. // Shared by the web installer and `gentle-shell upgrade`; every effect outside // plain path arithmetic goes through an injectable adapter for tests. -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { accessSync, chmodSync, closeSync, constants, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, readlinkSync, realpathSync, renameSync, rmSync, symlinkSync, unlinkSync, writeFileSync, writeSync } from "node:fs"; import { posix, win32 } from "node:path"; @@ -229,7 +229,7 @@ function zipMember(bytes, wanted) { } /** Runs a command without a shell; resolves { status, stdout, stderr }. */ -function runCommand(command, args, { cwd, env, timeout = 15000 }) { +export function runCommand(command, args, { cwd, env, timeout = 15000 }) { return new Promise((resolve) => { const child = spawn(command, args, { cwd, env, shell: false, windowsHide: true, stdio: ["ignore", "pipe", "pipe"] }); const output = { stdout: "", stderr: "" }; @@ -357,6 +357,45 @@ export function distributionFiles(manifest) { const project = { name: "gentle-shell-distribution", version: shell, private: true, dependencies: { "gentle-pi": shell, "@earendil-works/pi-coding-agent": pi } }; return { "package.json": `${JSON.stringify(project, null, "\t")}\n`, "pnpm-workspace.yaml": "allowBuilds:\n gentle-pi: true\nstrictDepBuilds: false\n" }; } + +/** The two release assets (T1b), under stable names. */ +export const DISTRIBUTION_ASSETS = Object.freeze({ distribution: "gentle-shell-distribution.json", lockfile: "gentle-shell-distribution-lock.yaml" }); +const DISTRIBUTION_SCHEMA = 1; +const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex"); +/** The distribution asset text: the exact versions, the pinned Node and pnpm + * the lockfile was resolved with, the files installVersion writes next to it + * and the lockfile's sha256. Plain data in a fixed order, so the same inputs + * always give the same bytes. + */ +export function distributionAsset(manifest, lockfile) { + const { shell, pi, id } = versionsOf(manifest); + if (typeof lockfile !== "string" || lockfile.length === 0) throw new Error("Release lockfile missing"); + const asset = { schema: DISTRIBUTION_SCHEMA, shell, pi, id, generatedWith: { node: pins.node, pnpm: pins.pnpm }, files: distributionFiles({ shell, pi }), + lockfile: { name: DISTRIBUTION_ASSETS.lockfile, sha256: sha256(lockfile) } }; + return `${JSON.stringify(asset, null, "\t")}\n`; +} +/** Reads a downloaded asset pair: returns { manifest, lockfile } only for exact + * versions, a lockfile whose sha256 the asset records, and workspace files equal + * to the ones this module's installVersion writes (a frozen install with other + * settings was never verified). generatedWith is recorded, not compared. + */ +export function readDistribution(text, lockfile) { + let asset; + try { asset = JSON.parse(text); } + catch { throw new Error("Distribution asset rejected"); } + if (asset?.schema !== DISTRIBUTION_SCHEMA) throw new Error("Distribution asset schema rejected"); + const manifest = { shell: asset.shell, pi: asset.pi }; + const { id } = versionsOf(manifest); + const files = distributionFiles(manifest); + const recorded = asset.files ?? {}; + if (asset.id !== id || Object.keys(recorded).length !== Object.keys(files).length || Object.entries(files).some(([name, content]) => recorded[name] !== content)) { + throw new Error("Distribution asset workspace files rejected"); + } + if (typeof lockfile !== "string" || asset.lockfile?.name !== DISTRIBUTION_ASSETS.lockfile || asset.lockfile?.sha256 !== sha256(lockfile)) { + throw new Error("Distribution lockfile sha256 mismatch"); + } + return { manifest, lockfile }; +} function versionDirectory(layout, id) { if (typeof id !== "string" || !VERSION_ID.test(id) || id.includes("..")) throw new Error("Unsafe version id"); return pathFor(process.platform).join(layout.versions, id); @@ -426,7 +465,11 @@ export async function installVersion({ layout, manifest, lockfile, platform = la for (const [name, text] of Object.entries({ ...files, "pnpm-lock.yaml": lockfile })) writeFileSync(path.join(destination, name), text, { flag: "wx", mode: 0o600 }); const result = await (adapters.run ?? runCommand)(layout.node, [layout.pnpm, "install", "--frozen-lockfile"], { cwd: destination, env: pnpmEnvironment(layout, { platform, env, go }), timeout: INSTALL_TIMEOUT }); - if (result.status !== 0) throw Object.assign(new Error("pnpm install failed"), { stderr: String(result.stderr ?? "").slice(-4000) }); + // pnpm prints install and postinstall failures on stdout: keep both tails. + if (result.status !== 0) { + throw Object.assign(new Error(`pnpm install failed (exit ${result.status})`), + { stderr: [String(result.stdout ?? "").slice(-6000), String(result.stderr ?? "").slice(-4000)].filter(Boolean).join("\n") }); + } const shellDirectory = installedPackage(destination, "gentle-pi", shell, "gentle-pi"); installedPackage(destination, "@earendil-works/pi-coding-agent", pi, "Pi"); if (!regular(path.join(shellDirectory, "bin", "gentle-shell.mjs"))) throw new Error("Installed gentle-pi has no launcher entry"); diff --git a/tests/build-distribution.test.ts b/tests/build-distribution.test.ts new file mode 100644 index 000000000..2da9699ae --- /dev/null +++ b/tests/build-distribution.test.ts @@ -0,0 +1,269 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test, { type TestContext } from "node:test"; +import { fileURLToPath } from "node:url"; +import { DISTRIBUTION_ASSETS, claimPrefix, distributionAsset, distributionFiles, installVersion, pnpmEnvironment, readDistribution } from "../scripts/bundled-install.mjs"; +import { buildDistribution, main, verifyDistribution } from "../scripts/build-distribution.mjs"; +import { artifactFor } from "../scripts/installer-downloads.mjs"; +import { PI_INSTALL_VERSION } from "../scripts/installer-preflight.mjs"; + +const ROOT = dirname(dirname(fileURLToPath(import.meta.url))); +// Real directories, modes and uids need a POSIX host. +const posixHost = process.platform === "win32" ? "POSIX filesystem fixtures need a POSIX host" : false; +const posixTest = (name: string, fn: (t: TestContext) => void | Promise) => test(name, { skip: posixHost }, fn); +const manifest = { shell: "4.1.0", pi: "1.0.2" }; +const LOCKFILE = "lockfileVersion: '9.0'\n\nimporters:\n\n .: {}\n"; +const sha256 = (text: string) => createHash("sha256").update(text, "utf8").digest("hex"); + +/** A new folder in the system tmpdir, removed when the test ends. */ +function temporary(t: TestContext) { + const root = mkdtempSync(join(realpathSync(tmpdir()), "build-distribution-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + return root; +} +function claimed(t: TestContext) { + const home = join(temporary(t), "home"); + mkdirSync(home, { mode: 0o700 }); + return claimPrefix({ platform: "darwin", env: { HOME: home }, home }); +} +type Run = { command: string; args: string[]; cwd: string; env: Record; files: Record }; +/** Our pnpm resolving the lockfile: it records what it saw, writes the lockfile, + * and, like pnpm 11 after skipped builds, may rewrite pnpm-workspace.yaml. */ +function fakeResolver({ lockfile = LOCKFILE, rewrite = false, status = 0, missing = 0 } = {}) { + const runs: Run[] = []; + const run = (command: string, args: string[], { cwd, env }: { cwd: string; env: Record }) => { + runs.push({ command, args, cwd, env, files: Object.fromEntries(readdirSync(cwd).map((name) => [name, readFileSync(join(cwd, name), "utf8")])) }); + // As pnpm 11.1.1 prints it, on stdout, before the registry lists a just-published version. + const name = JSON.parse(readFileSync(join(cwd, "package.json"), "utf8")).version; + if (runs.length <= missing) return { status: 1, stdout: `[ERR_PNPM_NO_MATCHING_VERSION] No matching version found for gentle-pi@${name} while fetching it from https://registry.npmjs.org/\n`, stderr: "" }; + if (status !== 0) return { status, stdout: "", stderr: "ERR_PNPM_FETCH_404" }; + writeFileSync(join(cwd, "pnpm-lock.yaml"), lockfile); + if (rewrite) writeFileSync(join(cwd, "pnpm-workspace.yaml"), "allowBuilds:\n gentle-pi: true\n koffi: set this to true or false\nstrictDepBuilds: false\n"); + return { status: 0, stdout: "", stderr: "" }; + }; + return { runs, run }; +} + +test("the distribution asset records the exact versions, the pins, the workspace files installVersion writes and the lockfile sha256", () => { + const text = distributionAsset(manifest, LOCKFILE); + assert.deepEqual(DISTRIBUTION_ASSETS, { distribution: "gentle-shell-distribution.json", lockfile: "gentle-shell-distribution-lock.yaml" }); + assert.deepEqual(JSON.parse(text), { + schema: 1, shell: "4.1.0", pi: "1.0.2", id: "4.1.0-1.0.2", + generatedWith: { node: artifactFor("node", "linux", "x64").version, pnpm: artifactFor("pnpm").version }, + files: distributionFiles(manifest), + lockfile: { name: "gentle-shell-distribution-lock.yaml", sha256: sha256(LOCKFILE) }, + }); + assert.equal(text.endsWith("}\n"), true); + assert.equal(distributionAsset({ ...manifest }, `${LOCKFILE}`), text, "the same inputs give the same bytes"); + assert.notEqual(distributionAsset(manifest, `${LOCKFILE}\n`), text); + for (const bad of [{ shell: "^4.1.0", pi: "1.0.2" }, { shell: "4.1", pi: "1.0.2" }, { shell: "4.1.0", pi: "latest" }, { shell: "4.1.0", pi: "v1.0.2" }, { shell: "4.1.0", pi: "1.x" }]) { + assert.throws(() => distributionAsset(bad, LOCKFILE), /version rejected/, JSON.stringify(bad)); + } + assert.throws(() => distributionAsset(manifest, ""), /lockfile/i); +}); + +test("readDistribution accepts only an unchanged pair whose workspace files are the ones installVersion writes", () => { + const text = distributionAsset(manifest, LOCKFILE); + assert.deepEqual(readDistribution(text, LOCKFILE), { manifest, lockfile: LOCKFILE }); + assert.throws(() => readDistribution(text, `${LOCKFILE}# changed\n`), /lockfile sha256/); + const asset = JSON.parse(text); + const tampered = (change: (value: typeof asset) => void) => { const copy = structuredClone(asset); change(copy); return `${JSON.stringify(copy, null, "\t")}\n`; }; + assert.throws(() => readDistribution(tampered((value) => { value.files["pnpm-workspace.yaml"] = "allowBuilds:\n koffi: true\n"; }), LOCKFILE), /workspace files/); + assert.throws(() => readDistribution(tampered((value) => { value.schema = 2; }), LOCKFILE), /schema/); + assert.throws(() => readDistribution(tampered((value) => { value.pi = "^1.0.2"; }), LOCKFILE), /version rejected/); + assert.throws(() => readDistribution("not json", LOCKFILE), /Distribution asset rejected/); + // A release built with other pinned runtimes still installs with ours: only recorded, not compared. + assert.deepEqual(readDistribution(tampered((value) => { value.generatedWith.pnpm = "11.2.0"; }), LOCKFILE).manifest, manifest); +}); + +posixTest("buildDistribution resolves the lockfile with our pnpm from the unmodified distribution files and returns both assets", async (t) => { + const layout = claimed(t); + const resolver = fakeResolver({ rewrite: true }); + const env = { PATH: "/usr/bin", npm_config_registry: "http://user.example" }; + const result = await buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", env, adapters: { run: resolver.run } }); + assert.equal(resolver.runs.length, 1); + const [run] = resolver.runs; + assert.equal(run.command, layout.node); + assert.deepEqual(run.args, [layout.pnpm, "install", "--lockfile-only"]); + assert.deepEqual(run.files, distributionFiles(manifest), "pnpm resolves exactly the files installVersion writes, and no lockfile"); + assert.deepEqual(run.env, pnpmEnvironment(layout, { env })); + assert.equal(dirname(run.cwd), layout.tmp); + assert.equal(existsSync(run.cwd), false, "the resolution folder is removed"); + assert.deepEqual(result.manifest, manifest); + assert.deepEqual(Object.keys(result.assets).sort(), [DISTRIBUTION_ASSETS.distribution, DISTRIBUTION_ASSETS.lockfile].sort()); + assert.equal(result.assets[DISTRIBUTION_ASSETS.lockfile], LOCKFILE); + // pnpm's rewritten workspace file (placeholders for skipped builds) never reaches the asset. + assert.equal(result.assets[DISTRIBUTION_ASSETS.distribution], distributionAsset(manifest, LOCKFILE)); + assert.equal(JSON.parse(result.assets[DISTRIBUTION_ASSETS.distribution]).lockfile.sha256, sha256(LOCKFILE)); + const again = await buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", env, adapters: { run: fakeResolver({ rewrite: true }).run } }); + assert.deepEqual(again.assets, result.assets, "the same lockfile gives byte-identical assets"); +}); + +posixTest("buildDistribution pins Pi to PI_INSTALL_VERSION and refuses a non-exact version or a failed resolution before publishing anything", async (t) => { + const layout = claimed(t); + const pinned = await buildDistribution({ layout, shell: "4.1.0", adapters: { run: fakeResolver().run } }); + assert.deepEqual(pinned.manifest, { shell: "4.1.0", pi: PI_INSTALL_VERSION }); + assert.equal(JSON.parse(pinned.assets[DISTRIBUTION_ASSETS.distribution]).pi, PI_INSTALL_VERSION); + for (const [shell, pi] of [["^4.1.0", "1.0.2"], ["4.1.0", "~1.0.2"], ["latest", "1.0.2"], ["4.1.0", "1.0.2 || 1.0.3"], ["4.1.0+build", "1.0.2"]]) { + const resolver = fakeResolver(); + await assert.rejects(buildDistribution({ layout, shell, pi, adapters: { run: resolver.run } }), /version rejected/, `${shell} ${pi}`); + assert.equal(resolver.runs.length, 0, "pnpm never runs for a non-exact version"); + } + const failed = fakeResolver({ status: 1 }); + await assert.rejects(buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", adapters: { run: failed.run, sleep: async () => {} } }), + (error: Error & { output?: string }) => /lockfile resolution failed/.test(error.message) && error.output === "ERR_PNPM_FETCH_404"); + assert.equal(failed.runs.length, 1, "only a gentle-pi missing from the registry is retried"); + await assert.rejects(buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", adapters: { run: fakeResolver({ lockfile: "" }).run } }), /lockfile/i); + assert.deepEqual(readdirSync(layout.tmp), [], "no resolution folder is left behind"); +}); + +posixTest("the assets install with installVersion: it writes the recorded workspace files and the lockfile byte for byte", async (t) => { + const layout = claimed(t); + const { assets } = await buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", adapters: { run: fakeResolver().run } }); + const { manifest: read, lockfile } = readDistribution(assets[DISTRIBUTION_ASSETS.distribution], assets[DISTRIBUTION_ASSETS.lockfile]); + let written: Record = {}; + const run = (_command: string, args: string[], { cwd }: { cwd: string }) => { + assert.deepEqual(args.slice(1), ["install", "--frozen-lockfile"]); + written = Object.fromEntries(["package.json", "pnpm-workspace.yaml", "pnpm-lock.yaml"].map((name) => [name, readFileSync(join(cwd, name), "utf8")])); + const shell = join(cwd, "node_modules", "gentle-pi"); const pi = join(cwd, "node_modules", "@earendil-works", "pi-coding-agent"); + mkdirSync(join(shell, "bin"), { recursive: true }); mkdirSync(pi, { recursive: true }); + writeFileSync(join(shell, "package.json"), JSON.stringify({ name: "gentle-pi", version: read.shell })); + writeFileSync(join(shell, "bin", "gentle-shell.mjs"), ""); + writeFileSync(join(pi, "package.json"), JSON.stringify({ name: "@earendil-works/pi-coding-agent", version: read.pi })); + return { status: 0, stdout: "", stderr: "" }; + }; + await installVersion({ layout, manifest: read, lockfile, platform: "darwin", env: {}, adapters: { run } }); + const recorded = JSON.parse(assets[DISTRIBUTION_ASSETS.distribution]).files; + assert.deepEqual(written, { ...recorded, "pnpm-lock.yaml": assets[DISTRIBUTION_ASSETS.lockfile] }); +}); + +posixTest("verifyDistribution installs the pair with our runtimes, activates it and requires the launcher to report both exact versions", async (t) => { + const layout = claimed(t); + // Runtimes already published under their pinned URLs are reused, so nothing is downloaded. + for (const [name, directory, entry] of [["node", layout.nodeDir, layout.node], ["pnpm", layout.pnpmDir, layout.pnpm]]) { + mkdirSync(dirname(entry), { recursive: true }); + writeFileSync(entry, ""); + writeFileSync(join(directory, ".gentle-shell-runtime"), `${artifactFor(name, "darwin", "arm64").url}\n`); + } + const distribution = distributionAsset(manifest, LOCKFILE); + const installs: string[][] = []; + const run = (_command: string, args: string[], { cwd }: { cwd: string }) => { + installs.push(args.slice(1)); + const shell = join(cwd, "node_modules", "gentle-pi"); const pi = join(cwd, "node_modules", "@earendil-works", "pi-coding-agent"); + mkdirSync(join(shell, "bin"), { recursive: true }); mkdirSync(pi, { recursive: true }); + writeFileSync(join(shell, "package.json"), JSON.stringify({ name: "gentle-pi", version: manifest.shell })); + writeFileSync(join(shell, "bin", "gentle-shell.mjs"), ""); + writeFileSync(join(pi, "package.json"), JSON.stringify({ name: "@earendil-works/pi-coding-agent", version: manifest.pi })); + return { status: 0, stdout: "", stderr: "" }; + }; + const launches: string[] = []; + const launch = (stdout: string) => (launcher: string) => { launches.push(launcher); return { status: 0, stdout, stderr: "" }; }; + const options = { layout, distribution, lockfile: LOCKFILE, platform: "darwin", arch: "arm64", env: {} }; + const result = await verifyDistribution({ ...options, adapters: { run, launch: launch("gentle-shell 4.1.0\npi 1.0.2\nhome isolated /h\n") } }); + assert.deepEqual(installs, [["install", "--frozen-lockfile"]]); + assert.deepEqual(launches, [layout.launcher]); + assert.deepEqual(result, { id: "4.1.0-1.0.2", version: "gentle-shell 4.1.0\npi 1.0.2\nhome isolated /h" }); + assert.equal(readFileSync(layout.launcher, "utf8").includes("gentle-shell bundled launcher"), true); + // Another Pi (a fallback) or another gentle-shell fails the verification. + await assert.rejects(verifyDistribution({ ...options, adapters: { run, launch: launch("gentle-shell 4.1.0\npi 1.1.0\n") } }), /launcher reported/); + await assert.rejects(verifyDistribution({ ...options, adapters: { run, launch: () => ({ status: 1, stdout: "", stderr: "boom" }) } }), /launcher reported/); + await assert.rejects(verifyDistribution({ ...options, lockfile: `${LOCKFILE}x`, adapters: { run, launch: launch("") } }), /lockfile sha256/); + // The launcher runs in the same filtered environment as our pnpm, without Pi or Gentle overrides. + const host = { HOME: "/h", PATH: "/usr/bin", NODE_OPTIONS: "--require /x.js", NODE_PATH: "/x", npm_config_prefix: "/x", PNPM_HOME: "/x", PI_CODING_AGENT_DIR: "/x", GENTLE_SHELL_PI: "/x" }; + let seen: Record = {}; + await verifyDistribution({ ...options, env: host, adapters: { run, launch: (_launcher: string, { env }: { env: Record }) => { + seen = env; return { status: 0, stdout: "gentle-shell 4.1.0\npi 1.0.2\n", stderr: "" }; + } } }); + const { PI_CODING_AGENT_DIR: _pi, GENTLE_SHELL_PI: _shell, ...kept } = host; + assert.deepEqual(seen, pnpmEnvironment(layout, { platform: "darwin", env: kept })); + for (const key of ["NODE_OPTIONS", "NODE_PATH", "npm_config_prefix", "PI_CODING_AGENT_DIR", "GENTLE_SHELL_PI"]) assert.equal(key in seen, false, key); + assert.equal(seen.PATH.split(":")[0], dirname(layout.node), "our Node first on PATH"); +}); + +posixTest("buildDistribution retries while the just-published gentle-pi is not on the registry, then fails clearly", async (t) => { + const layout = claimed(t); + const waits: number[] = []; + const sleep = async (ms: number) => { waits.push(ms); }; + const late = fakeResolver({ missing: 2 }); + const result = await buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", adapters: { run: late.run, sleep } }); + assert.equal(result.assets[DISTRIBUTION_ASSETS.lockfile], LOCKFILE); + assert.equal(late.runs.length, 3); + assert.deepEqual(waits, [30000, 30000]); + assert.equal(new Set(late.runs.map((run) => run.cwd)).size, 3, "each attempt resolves in a fresh folder"); + for (const run of late.runs) assert.deepEqual(run.files, distributionFiles(manifest)); + waits.length = 0; + const never = fakeResolver({ missing: 99 }); + await assert.rejects(buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", adapters: { run: never.run, sleep } }), + (error: Error & { output?: string }) => /gentle-pi@4\.1\.0 is not on the registry after 10 attempts/.test(error.message) && /ERR_PNPM_NO_MATCHING_VERSION/.test(error.output ?? "")); + assert.equal(never.runs.length, 10); + assert.equal(waits.reduce((sum, ms) => sum + ms, 0), 270000, "about five minutes in all"); + // A missing Pi, or another gentle-pi version, is a real failure: never retried. + for (const missing of ["@earendil-works/pi-coding-agent@1.0.2", "gentle-pi@4.1.0-beta.1", "gentle-pi@4.1.01"]) { + let runs = 0; + const run = () => { runs += 1; return { status: 1, stdout: `[ERR_PNPM_NO_MATCHING_VERSION] No matching version found for ${missing} while fetching it\n`, stderr: "" }; }; + await assert.rejects(buildDistribution({ layout, shell: "4.1.0", pi: "1.0.2", adapters: { run, sleep } }), /lockfile resolution failed/, missing); + assert.equal(runs, 1, missing); + } + assert.deepEqual(readdirSync(layout.tmp), [], "no resolution folder is left behind"); +}); + +posixTest("the CLI removes the temporary prefix it created when build or verify fails, and refuses bad input before creating one", async (t) => { + const root = temporary(t); + const options = { temporaryRoot: root, env: { PATH: "/usr/bin" }, adapters: { download: async () => Buffer.from("not the pinned archive") } }; + await assert.rejects(main(["verify", "--assets", join(root, "missing")], options), /ENOENT/); + assert.deepEqual(readdirSync(root), [], "verify with unreadable assets"); + const assets = join(root, "assets"); mkdirSync(assets); + writeFileSync(join(assets, DISTRIBUTION_ASSETS.distribution), distributionAsset(manifest, LOCKFILE)); + writeFileSync(join(assets, DISTRIBUTION_ASSETS.lockfile), LOCKFILE); + await assert.rejects(main(["verify", "--assets", assets], options), /Download integrity mismatch/); + assert.deepEqual(readdirSync(root), ["assets"], "verify whose runtime download fails"); + await assert.rejects(main(["build", "--shell", "4.1.0", "--out", join(root, "out")], options), /Download integrity mismatch/); + assert.deepEqual(readdirSync(root), ["assets"], "build whose runtime download fails"); + await assert.rejects(main(["build", "--shell", "^4.1.0", "--out", join(root, "out")], options), /version rejected/); + await assert.rejects(main(["publish"], options), /Usage/); + assert.deepEqual(readdirSync(root), ["assets"]); +}); + +// publish.yml: the distribution assets reach the release only after the frozen +// install passed on all three systems, and the existing jobs keep their shape. +function jobs(workflow: string) { + const body = workflow.slice(workflow.indexOf("\njobs:\n") + "\njobs:\n".length); + return new Map([...body.matchAll(/^ {2}([A-Za-z0-9_-]+):\n([\s\S]*?)(?=^ {2}[A-Za-z0-9_-]+:\n|(?![\s\S]))/gm)].map((match) => [match[1], match[2]])); +} +test("publish.yml builds the distribution after npm publication, verifies it on three systems and only then uploads it", () => { + const workflow = readFileSync(join(ROOT, ".github", "workflows", "publish.yml"), "utf8"); + const all = jobs(workflow); + assert.deepEqual([...all.keys()], ["publish", "installers", "distribution", "distribution-verify", "distribution-assets"]); + for (const name of ["publish", "installers"]) assert.doesNotMatch(all.get(name)!, /distribution/, `${name} is unchanged`); + assert.match(workflow, /^permissions:\n {2}contents: read\n/m, "the workflow default stays read-only"); + for (const use of workflow.matchAll(/uses: (\S+)/g)) assert.match(use[1], /^[\w-]+\/[\w-]+@[0-9a-f]{40}$/, `${use[1]} is pinned by commit SHA`); + + const build = all.get("distribution")!; + assert.match(build, /^ {4}needs: publish$/m, "gentle-pi@ must be on npm before it is resolved"); + assert.match(build, /^ {4}permissions:\n {6}contents: read\n(?! {6})/m); + assert.match(build, /ref: \$\{\{ github\.sha \}\}/, "built on the verified release commit"); + assert.match(build, /RELEASE_TAG: \$\{\{ needs\.publish\.outputs\.tag \}\}/); + assert.match(build, /node scripts\/build-distribution\.mjs build --shell "\$\{RELEASE_TAG#v\}" --out "\$\{RUNNER_TEMP\}\/distribution"/); + assert.match(build, /uses: actions\/upload-artifact@[0-9a-f]{40} # v[\d.]+\n {8}with:\n {10}name: gentle-shell-distribution\n/); + + const verify = all.get("distribution-verify")!; + assert.match(verify, /^ {4}needs: distribution$/m); + assert.match(verify, /^ {4}permissions:\n {6}contents: read\n(?! {6})/m); + assert.match(verify, /matrix:\n {8}os: \[ubuntu-latest, macos-latest, windows-latest\]\n/); + assert.match(verify, /fail-fast: false/); + assert.match(verify, /runs-on: \$\{\{ matrix\.os \}\}/); + assert.match(verify, /ref: \$\{\{ github\.sha \}\}/); + assert.match(verify, /uses: actions\/download-artifact@[0-9a-f]{40} # v[\d.]+\n {8}with:\n {10}name: gentle-shell-distribution\n/); + assert.match(verify, /node scripts\/build-distribution\.mjs verify --assets "\$\{RUNNER_TEMP\}\/distribution"/); + assert.doesNotMatch(verify, /gh release|contents: write/); + + const upload = all.get("distribution-assets")!; + assert.match(upload, /^ {4}needs: \[publish, distribution-verify\]$/m, "uploaded only after every verify lane passed"); + assert.match(upload, /^ {4}permissions:\n {6}contents: write\n(?! {6})/m); + assert.match(upload, /RELEASE_TAG: \$\{\{ needs\.publish\.outputs\.tag \}\}/); + assert.match(upload, /gh release upload "\$\{RELEASE_TAG\}" "\$\{RUNNER_TEMP\}\/distribution\/gentle-shell-distribution\.json" "\$\{RUNNER_TEMP\}\/distribution\/gentle-shell-distribution-lock\.yaml" --repo "\$\{GITHUB_REPOSITORY\}" --clobber/); + for (const [name, job] of all) if (name !== "distribution-assets") assert.doesNotMatch(job, /gentle-shell-distribution\.json/, `${name} never uploads the assets`); +}); diff --git a/tests/bundled-install.test.ts b/tests/bundled-install.test.ts index 0fd042515..66211b838 100644 --- a/tests/bundled-install.test.ts +++ b/tests/bundled-install.test.ts @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, readlinkSync, realpathSync, renameSync, symlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, rmSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, readlinkSync, realpathSync, renameSync, symlinkSync, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join, posix } from "node:path"; -import test, { type TestContext } from "node:test"; +import test, { after, type TestContext } from "node:test"; import { crc32, gzipSync } from "node:zlib"; import { activateVersion, activeVersion, applyPathEntry, claimPrefix, distributionFiles, ensureLauncher, ensureRuntime, installVersion, nodeExecutable, pathEntryPlan, pnpmEnvironment, prefixLayout, pruneVersions, removePathEntry } from "../scripts/bundled-install.mjs"; @@ -15,8 +15,17 @@ const posixTest = (name: string, fn: (t: TestContext) => void | Promise) = const NODE = "24.21.0"; const PNPM = "11.1.1"; +// Every temporary home this file creates is removed when the file's tests end. +const roots: string[] = []; +after(() => { + for (const root of roots) { + try { chmodSync(root, 0o700); } catch {} + rmSync(root, { recursive: true, force: true }); + } +}); function home() { const root = mkdtempSync(join(realpathSync(tmpdir()), "bundled-install-")); + roots.push(root); const dir = join(root, "home ü"); mkdirSync(dir, { mode: 0o700 }); return dir; @@ -371,6 +380,11 @@ posixTest("installVersion rejects a wrong installed version, a foreign folder an assert.deepEqual(readdirSync(layout.versions), []); await assert.rejects(installVersion({ layout, manifest, lockfile: "x", platform: "darwin", env: {}, adapters: { run: () => ({ status: 1, stdout: "", stderr: "ERR_PNPM" }) } }), /pnpm install failed/); assert.deepEqual(readdirSync(layout.versions), []); + // pnpm prints install and postinstall failures on stdout: the error keeps both streams. + await assert.rejects(installVersion({ layout, manifest, lockfile: "x", platform: "darwin", env: {}, + adapters: { run: () => ({ status: 1, stdout: "postinstall: Gentle AI build failed\n[ELIFECYCLE] exit 1", stderr: "" }) } }), + (error: Error & { stderr?: string }) => /pnpm install failed \(exit 1\)/.test(error.message) && /Gentle AI build failed/.test(String(error.stderr))); + assert.deepEqual(readdirSync(layout.versions), []); mkdirSync(join(layout.versions, "4.1.0-1.0.2")); writeFileSync(join(layout.versions, "4.1.0-1.0.2", "user.txt"), "keep"); await assert.rejects(installVersion({ layout, manifest, lockfile: "x", platform: "darwin", env: {}, adapters: { run: fakePnpm().run } }), /Conflicting version/);