From ae774d4f63b2056e0ee8ac978e6d0886a2370e38 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Sun, 12 Jul 2026 17:50:33 -0700 Subject: [PATCH 1/4] ci(deps): add candidate compatibility workflow Signed-off-by: Ho Lim --- .../workflows/candidate-compatibility.yaml | 343 +++++++ ci/source-shape-test-budget.json | 5 + test/candidate-compat.test.ts | 411 ++++++++ tools/candidate-compat.mts | 919 ++++++++++++++++++ 4 files changed, 1678 insertions(+) create mode 100644 .github/workflows/candidate-compatibility.yaml create mode 100644 test/candidate-compat.test.ts create mode 100644 tools/candidate-compat.mts diff --git a/.github/workflows/candidate-compatibility.yaml b/.github/workflows/candidate-compatibility.yaml new file mode 100644 index 0000000000..8a1f2d9b29 --- /dev/null +++ b/.github/workflows/candidate-compatibility.yaml @@ -0,0 +1,343 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Dependencies / Candidate Compatibility +run-name: "Candidate ${{ inputs.component }} ${{ inputs.candidate }} at ${{ inputs.nemoclaw_ref }}" + +on: + workflow_dispatch: + inputs: + nemoclaw_ref: + description: Exact NemoClaw branch, tag, or commit to resolve once. + required: true + type: string + component: + description: Official dependency candidate to exercise. + required: true + type: choice + options: + - openshell + - openclaw + - hermes + - dcode + candidate: + description: Exact official version, or vX.Y.Z release tag for Hermes. + required: true + type: string + +permissions: + contents: read + +concurrency: + group: candidate-compatibility-${{ inputs.nemoclaw_ref }}-${{ inputs.component }}-${{ inputs.candidate }} + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + nemoclaw_sha: ${{ steps.identity.outputs.nemoclaw_sha }} + resolution_id: ${{ steps.identity.outputs.resolution_id }} + steps: + - name: Check out trusted compatibility controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.sha }} + path: controller + fetch-depth: 1 + persist-credentials: false + + - name: Resolve and check out requested NemoClaw ref + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ inputs.nemoclaw_ref }} + path: candidate-source + fetch-depth: 1 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - id: identity + name: Resolve official candidate provenance + env: + CANDIDATE: ${{ inputs.candidate }} + COMPONENT: ${{ inputs.component }} + GITHUB_TOKEN: ${{ github.token }} + WORKFLOW_REF: ${{ github.ref }} + shell: bash + run: | + set -euo pipefail + [[ "$WORKFLOW_REF" == refs/heads/main ]] || { echo "::error::candidate compatibility must be dispatched from main"; exit 1; } + nemoclaw_sha="$(git -C candidate-source rev-parse --verify HEAD^{commit})" + [[ "$nemoclaw_sha" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::NemoClaw ref did not resolve to a full commit SHA"; exit 1; } + [[ -z "$(git -C candidate-source status --short --untracked-files=no)" ]] || { echo "::error::checkout is not clean before candidate resolution"; exit 1; } + node --experimental-strip-types controller/tools/candidate-compat.mts resolve \ + --nemoclaw-sha "$nemoclaw_sha" \ + --component "$COMPONENT" \ + --candidate "$CANDIDATE" \ + --output candidate-receipt.json + resolution_id="$(node -e 'const r=require("./candidate-receipt.json"); process.stdout.write(r.resolutionId)')" + [[ "$resolution_id" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::resolver emitted an invalid identity"; exit 1; } + printf 'nemoclaw_sha=%s\nresolution_id=%s\n' "$nemoclaw_sha" "$resolution_id" >> "$GITHUB_OUTPUT" + + - id: plan + name: Plan deterministic and live compatibility lanes + env: + COMPONENT: ${{ inputs.component }} + shell: bash + run: | + set -euo pipefail + node --experimental-strip-types controller/tools/candidate-compat.mts plan \ + --component "$COMPONENT" \ + --e2e-workflow candidate-source/.github/workflows/e2e.yaml \ + --e2e-registry candidate-source/test/e2e/registry/definitions/baseline.ts \ + --output candidate-plan.json + matrix="$(node -e 'const p=require("./candidate-plan.json"); process.stdout.write(JSON.stringify({lane:p.deterministic.filter(x=>x.status==="selected").map(x=>x.id)}))')" + printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT" + + - name: Upload immutable resolution and plan + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: candidate-resolution-${{ steps.identity.outputs.resolution_id }} + path: | + candidate-receipt.json + candidate-plan.json + if-no-files-found: error + retention-days: 30 + + deterministic: + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }} + steps: + - name: Check out trusted compatibility controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.sha }} + path: controller + fetch-depth: 1 + persist-credentials: false + + - name: Check out resolved NemoClaw commit + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ needs.resolve.outputs.nemoclaw_sha }} + path: candidate-source + fetch-depth: 1 + persist-credentials: false + + - name: Verify checkout identity + env: + EXPECTED_SHA: ${{ needs.resolve.outputs.nemoclaw_sha }} + shell: bash + run: | + set -euo pipefail + [[ "$(git -C candidate-source rev-parse --verify HEAD)" == "$EXPECTED_SHA" ]] || { echo "::error::lane checkout differs from resolved NemoClaw SHA"; exit 1; } + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + cache: npm + cache-dependency-path: candidate-source/package-lock.json + + - name: Install repository dependencies + working-directory: candidate-source + run: npm ci --ignore-scripts + + - name: Download immutable resolution and plan + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: candidate-resolution-${{ needs.resolve.outputs.resolution_id }} + path: candidate-input + + - id: candidate + name: Materialize and verify candidate runtime + continue-on-error: true + env: + LANE: ${{ matrix.lane }} + shell: bash + run: | + set -euo pipefail + node --experimental-strip-types controller/tools/candidate-compat.mts materialize \ + --receipt candidate-input/candidate-receipt.json \ + --resolution-id "${{ needs.resolve.outputs.resolution_id }}" \ + --directory "${RUNNER_TEMP}/candidate-runtime" \ + --output "candidate-observed-${LANE}.json" \ + --github-env "$GITHUB_ENV" 2>&1 | tee "candidate-materialize-${LANE}.log" + + - id: lane + name: Run ${{ matrix.lane }} lane + if: ${{ steps.candidate.outcome == 'success' }} + continue-on-error: true + env: + LANE: ${{ matrix.lane }} + shell: bash + working-directory: candidate-source + run: | + set -euo pipefail + exec > >(tee "candidate-lane-${LANE}.log") 2>&1 + case "$LANE" in + source-unit) + npx vitest run --project cli + ;; + integration) + npm run clean:cli + npm run build:cli + npx vitest run --project integration + ;; + installer) + npx vitest run --project installer-integration + ;; + package-contract) + npm ci --prefix nemoclaw --ignore-scripts + npm run clean:cli + npm --prefix nemoclaw run clean + npm run build:cli + npm --prefix nemoclaw run build + npx vitest run --project package-contract + ;; + plugin) + npm ci --prefix nemoclaw --ignore-scripts + npx vitest run --project plugin + ;; + e2e-support) + npx vitest run --project e2e-support + ;; + *) + echo "::error::untrusted lane id: $LANE" + exit 1 + ;; + esac + + - name: Record lane result + if: ${{ always() }} + env: + CANDIDATE_OUTCOME: ${{ steps.candidate.outcome }} + LANE: ${{ matrix.lane }} + LANE_OUTCOME: ${{ steps.lane.outcome }} + RESOLUTION_ID: ${{ needs.resolve.outputs.resolution_id }} + shell: bash + run: | + set -euo pipefail + conclusion=failure + if [[ "$CANDIDATE_OUTCOME" == success && "$LANE_OUTCOME" == success ]]; then + conclusion=success + fi + mkdir -p candidate-results + CONCLUSION="$conclusion" node -e ' + const fs = require("node:fs"); + const observedPath = `candidate-observed-${process.env.LANE}.json`; + const observed = fs.existsSync(observedPath) ? JSON.parse(fs.readFileSync(observedPath, "utf8")) : {}; + const result = { + conclusion: process.env.CONCLUSION, + lane: process.env.LANE, + observedOutput: observed.observedOutput, + observedVersion: observed.observedVersion, + resolutionId: process.env.RESOLUTION_ID, + }; + fs.writeFileSync(`candidate-results/${process.env.LANE}.json`, JSON.stringify(result) + "\n", {mode: 0o600}); + ' + + - name: Upload lane evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: candidate-result-${{ needs.resolve.outputs.resolution_id }}-${{ matrix.lane }} + path: | + candidate-results/${{ matrix.lane }}.json + candidate-observed-${{ matrix.lane }}.json + candidate-materialize-${{ matrix.lane }}.log + candidate-source/candidate-lane-${{ matrix.lane }}.log + if-no-files-found: error + retention-days: 30 + + - name: Enforce lane result + if: ${{ always() && (steps.candidate.outcome != 'success' || steps.lane.outcome != 'success') }} + run: exit 1 + + evidence: + if: ${{ always() && needs.resolve.result == 'success' }} + needs: + - resolve + - deterministic + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out trusted compatibility controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.sha }} + path: controller + fetch-depth: 1 + persist-credentials: false + + - name: Download immutable resolution and plan + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: candidate-resolution-${{ needs.resolve.outputs.resolution_id }} + path: candidate-input + + - name: Download deterministic lane evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: candidate-result-${{ needs.resolve.outputs.resolution_id }}-* + path: candidate-results + merge-multiple: true + + - name: Finalize auditable evidence + env: + RUN_ATTEMPT: ${{ github.run_attempt }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + node --experimental-strip-types controller/tools/candidate-compat.mts finalize \ + --receipt candidate-input/candidate-receipt.json \ + --plan candidate-input/candidate-plan.json \ + --results candidate-results \ + --run-id "$RUN_ID" \ + --attempt "$RUN_ATTEMPT" \ + --output candidate-compatibility-evidence.json + node <<'NODE' >> "$GITHUB_STEP_SUMMARY" + const evidence = require("./candidate-compatibility-evidence.json"); + console.log("## Candidate compatibility evidence\n"); + console.log(`- NemoClaw SHA: \`${evidence.receipt.nemoclawSha}\``); + console.log(`- Candidate: \`${evidence.receipt.component} ${evidence.receipt.requestedCandidate}\``); + console.log(`- Resolution: \`${evidence.receipt.resolutionId}\``); + console.log(`- Overall deterministic result: **${evidence.overall}**\n`); + console.log("| Lane | Selection | Result / reason |"); + console.log("| --- | --- | --- |"); + const results = new Map(evidence.results.map((result) => [result.lane, result.conclusion])); + for (const lane of evidence.plan.deterministic) { + console.log(`| \`${lane.id}\` | ${lane.status} | ${results.get(lane.id) ?? lane.reason} |`); + } + for (const lane of evidence.plan.live) { + console.log(`| \`e2e:${lane.id}\` | ${lane.status} | ${lane.reason} |`); + } + NODE + + - name: Upload compatibility evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: candidate-compatibility-${{ needs.resolve.outputs.resolution_id }}-run-${{ github.run_id }}-${{ github.run_attempt }} + path: | + candidate-compatibility-evidence.json + candidate-input/candidate-receipt.json + candidate-input/candidate-plan.json + candidate-results/ + if-no-files-found: error + retention-days: 30 + + - name: Enforce aggregate result + env: + DETERMINISTIC_RESULT: ${{ needs.deterministic.result }} + run: test "$DETERMINISTIC_RESULT" = success diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 3f10d934c8..5f435c7d63 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -61,6 +61,11 @@ "test": "verifies the pinned Brev CLI digest before extracting it", "category": "security" }, + { + "file": "test/candidate-compat.test.ts", + "test": "keeps the manual controller read-only and separate from candidate source (#6691)", + "category": "security" + }, { "file": "test/cloudflared-update-check-workflow.test.ts", "test": "keeps automatic and on-demand update checks reachable and credential-free", diff --git a/test/candidate-compat.test.ts b/test/candidate-compat.test.ts new file mode 100644 index 0000000000..bf887f4eba --- /dev/null +++ b/test/candidate-compat.test.ts @@ -0,0 +1,411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { parse as parseYaml } from "yaml"; + +import { + buildCandidatePlan, + downloadCandidateArtifact, + finalizeEvidence, + parseCandidateReceipt, + resolveCandidate, + verifyDigest, + verifyObservedVersion, +} from "../tools/candidate-compat.mts"; + +const SHA = "a".repeat(40); +const E2E_SOURCES = [".github/workflows/e2e.yaml", "test/e2e/registry/definitions/baseline.ts"] + .map((path) => readFileSync(resolve(path), "utf8")) + .join("\n"); + +function response(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + status, + }); +} + +describe("candidate compatibility contract", () => { + // source-shape-contract: security -- This pins the trusted controller and read-only workflow boundary around candidate code execution. + it("keeps the manual controller read-only and separate from candidate source (#6691)", () => { + const source = readFileSync(resolve(".github/workflows/candidate-compatibility.yaml"), "utf8"); + const workflow = parseYaml(source) as { + jobs: Record; + on: { workflow_dispatch: { inputs: Record } }; + permissions: Record; + }; + expect(Object.keys(workflow.on.workflow_dispatch.inputs).sort()).toEqual([ + "candidate", + "component", + "nemoclaw_ref", + ]); + expect(workflow.permissions).toEqual({ contents: "read" }); + expect(Object.keys(workflow.jobs).sort()).toEqual(["deterministic", "evidence", "resolve"]); + expect(source).toContain("candidate compatibility must be dispatched from main"); + expect(source).toContain("path: controller"); + expect(source).toContain("path: candidate-source"); + expect(source).toContain("git -C candidate-source rev-parse --verify HEAD^{commit}"); + const toolSource = readFileSync(resolve("tools/candidate-compat.mts"), "utf8"); + expect(toolSource).toContain("NEMOCLAW_CANDIDATE_RECEIPT"); + expect(source).toContain('--resolution-id "${{ needs.resolve.outputs.resolution_id }}"'); + expect(source).not.toMatch(/\b(?:git push|gh pr|npm publish|docker push)\b/u); + }); + + it("rejects ambiguous and unsafe candidate input before metadata access (#6691)", async () => { + const fetcher = async () => { + throw new Error("metadata must not be fetched"); + }; + await expect( + resolveCandidate({ candidate: "latest", component: "openclaw", fetcher, nemoclawSha: SHA }), + ).rejects.toThrow("exact version"); + await expect( + resolveCandidate({ + candidate: "v1.2.3\nINJECTED=1", + component: "openshell", + fetcher, + nemoclawSha: SHA, + }), + ).rejects.toThrow("unsafe characters"); + await expect( + resolveCandidate({ + candidate: "1.2.3", + component: "unknown", + fetcher, + nemoclawSha: SHA, + }), + ).rejects.toThrow("component must be one of"); + }); + + it("binds OpenShell evidence to an official release asset digest (#6691)", async () => { + const fetcher = async () => + response({ + assets: [ + { + browser_download_url: + "https://github.com/NVIDIA/OpenShell/releases/download/v0.0.82/openshell-x86_64-unknown-linux-musl.tar.gz", + digest: `sha256:${"b".repeat(64)}`, + name: "openshell-x86_64-unknown-linux-musl.tar.gz", + }, + ], + draft: false, + id: 82, + tag_name: "v0.0.82", + }); + const first = await resolveCandidate({ + candidate: "v0.0.82", + component: "openshell", + fetcher, + nemoclawSha: SHA, + }); + const rerun = await resolveCandidate({ + candidate: "v0.0.82", + component: "openshell", + fetcher, + nemoclawSha: SHA, + }); + expect(first).toEqual(rerun); + expect(first).toMatchObject({ + nemoclawSha: SHA, + officialSource: "github:NVIDIA/OpenShell:release:82", + requestedCandidate: "v0.0.82", + resolvedCandidate: "0.0.82", + }); + expect(first.artifacts[0]?.digest).toBe("b".repeat(64)); + expect(first.resolutionId).toMatch(/^[a-f0-9]{64}$/u); + expect(parseCandidateReceipt(first, first.resolutionId)).toEqual(first); + expect(() => + parseCandidateReceipt( + { + ...first, + artifacts: [ + { + ...first.artifacts[0], + url: "https://example.com/openshell-x86_64-unknown-linux-musl.tar.gz", + }, + ], + }, + first.resolutionId, + ), + ).toThrow("approved official HTTPS host"); + expect(() => + parseCandidateReceipt({ ...first, resolutionId: "f".repeat(64) }, first.resolutionId), + ).toThrow("trusted resolution id"); + }); + + it("fails closed on missing upstream provenance and metadata errors (#6691)", async () => { + await expect( + resolveCandidate({ + candidate: "0.0.82", + component: "openshell", + fetcher: async () => response({ assets: [], draft: false, id: 82, tag_name: "v0.0.82" }), + nemoclawSha: SHA, + }), + ).rejects.toThrow("missing openshell-x86_64"); + await expect( + resolveCandidate({ + candidate: "2026.7.1", + component: "openclaw", + fetcher: async () => response({ message: "unavailable" }, 503), + nemoclawSha: SHA, + }), + ).rejects.toThrow("official metadata request failed (503)"); + }); + + it("rejects unofficial npm tarballs even with valid integrity (#6691)", async () => { + const integrity = createHash("sha512").update("archive").digest("base64"); + await expect( + resolveCandidate({ + candidate: "2026.7.1", + component: "openclaw", + fetcher: async () => + response({ + dist: { + integrity: `sha512-${integrity}`, + tarball: "https://example.com/openclaw-2026.7.1.tgz", + }, + version: "2026.7.1", + }), + nemoclawSha: SHA, + }), + ).rejects.toThrow("approved official HTTPS host"); + }); + + it("peels Hermes tags and resolves exact Hermes and DCode wheels (#6691)", async () => { + const integrity = createHash("sha512").update("hermes-npm").digest("base64"); + const commit = "c".repeat(40); + const responses = new Map([ + [ + "https://api.github.com/repos/NousResearch/hermes-agent/releases/tags/v2026.7.1", + response({ draft: false, id: 701, tag_name: "v2026.7.1" }), + ], + [ + "https://api.github.com/repos/NousResearch/hermes-agent/git/ref/tags/v2026.7.1", + response({ + object: { + sha: "d".repeat(40), + type: "tag", + url: "https://api.github.com/repos/NousResearch/hermes-agent/git/tags/annotated", + }, + }), + ], + [ + "https://api.github.com/repos/NousResearch/hermes-agent/git/tags/annotated", + response({ object: { sha: commit, type: "commit" } }), + ], + [ + `https://raw.githubusercontent.com/NousResearch/hermes-agent/${commit}/pyproject.toml`, + new Response('[project]\nversion = "0.18.0"\n'), + ], + [ + "https://registry.npmjs.org/hermes-agent/0.18.0", + response({ + dist: { + integrity: `sha512-${integrity}`, + tarball: "https://registry.npmjs.org/hermes-agent/-/hermes-agent-0.18.0.tgz", + }, + version: "0.18.0", + }), + ], + [ + "https://pypi.org/pypi/hermes-agent/0.18.0/json", + response({ + info: { name: "hermes-agent", version: "0.18.0" }, + urls: [ + { + digests: { sha256: "e".repeat(64) }, + filename: "hermes_agent-0.18.0-py3-none-any.whl", + packagetype: "bdist_wheel", + url: "https://files.pythonhosted.org/packages/hermes_agent-0.18.0.whl", + yanked: false, + }, + ], + }), + ], + [ + "https://pypi.org/pypi/deepagents-code/0.1.34/json", + response({ + info: { name: "deepagents-code", version: "0.1.34" }, + urls: [ + { + digests: { sha256: "f".repeat(64) }, + filename: "deepagents_code-0.1.34-py3-none-any.whl", + packagetype: "bdist_wheel", + url: "https://files.pythonhosted.org/packages/deepagents_code-0.1.34.whl", + yanked: false, + }, + ], + }), + ], + ]); + const fetcher = async (url: string) => + responses.get(url) ?? response({ message: "unexpected URL" }, 404); + const hermes = await resolveCandidate({ + candidate: "v2026.7.1", + component: "hermes", + fetcher, + nemoclawSha: SHA, + }); + expect(hermes).toMatchObject({ resolvedCandidate: "0.18.0", resolvedCommit: commit }); + expect(hermes.artifacts.map((item) => item.kind)).toEqual(["npm", "wheel"]); + const dcode = await resolveCandidate({ + candidate: "0.1.34", + component: "dcode", + fetcher, + nemoclawSha: SHA, + }); + expect(dcode).toMatchObject({ + officialSource: "pypi:deepagents-code==0.1.34", + resolvedCandidate: "0.1.34", + }); + }); + + it("detects digest and observed runtime version mismatches (#6691)", () => { + const bytes = new TextEncoder().encode("candidate"); + expect(() => + verifyDigest(bytes, { + digest: "0".repeat(64), + digestAlgorithm: "sha256", + kind: "archive", + name: "candidate.tgz", + url: "https://registry.npmjs.org/openclaw/-/candidate.tgz", + }), + ).toThrow("sha256 mismatch"); + expect(() => + verifyObservedVersion( + { + artifacts: [], + component: "openclaw", + nemoclawSha: SHA, + officialSource: "npm:openclaw@2026.7.1", + requestedCandidate: "2026.7.1", + resolutionId: "c".repeat(64), + resolvedCandidate: "2026.7.1", + schemaVersion: 1, + }, + "openclaw 2026.7.2", + ), + ).toThrow("does not match 2026.7.1"); + }); + + it("validates every redirect before writing a digest-bound artifact (#6691)", async () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-candidate-download-")); + const body = "candidate archive"; + const candidateArtifact = { + digest: createHash("sha256").update(body).digest("hex"), + digestAlgorithm: "sha256" as const, + kind: "npm" as const, + name: "openclaw-2026.7.1.tgz", + url: "https://registry.npmjs.org/openclaw/-/openclaw-2026.7.1.tgz", + }; + try { + const requests: string[] = []; + await expect( + downloadCandidateArtifact(candidateArtifact, directory, 0, async (url, init) => { + requests.push(url); + expect(init?.redirect).toBe("manual"); + return new Response(null, { + headers: { location: "http://127.0.0.1/internal" }, + status: 302, + }); + }), + ).rejects.toThrow("not approved"); + expect(requests).toEqual([candidateArtifact.url]); + + const target = await downloadCandidateArtifact( + candidateArtifact, + directory, + 0, + async () => new Response(body), + ); + expect(basename(target)).toBe("candidate-0.tgz"); + expect(readFileSync(target, "utf8")).toBe(body); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("records every deterministic lane and validates live selectors against E2E (#6691)", () => { + const plan = buildCandidatePlan("openshell", E2E_SOURCES); + expect(plan.deterministic).toHaveLength(6); + expect( + plan.deterministic.filter((lane) => lane.status === "selected").map((lane) => lane.id), + ).toEqual(["source-unit", "integration", "installer", "e2e-support"]); + expect(plan.deterministic.filter((lane) => lane.status === "skipped")).toEqual([ + expect.objectContaining({ id: "package-contract", reason: expect.any(String) }), + expect.objectContaining({ id: "plugin", reason: expect.any(String) }), + ]); + expect(plan.live).toEqual([ + expect.objectContaining({ id: "full-e2e", selector: "job", status: "skipped" }), + expect.objectContaining({ + id: "openshell-gateway-auth-contract", + selector: "job", + status: "skipped", + }), + ]); + expect(() => buildCandidatePlan("openshell", E2E_SOURCES.replace(" full-e2e:\n", ""))).toThrow( + "does not declare job full-e2e", + ); + expect(buildCandidatePlan("dcode", E2E_SOURCES).live).toEqual([ + expect.objectContaining({ + id: "ubuntu-repo-cloud-langchain-deepagents-code", + selector: "target", + status: "skipped", + }), + ]); + }); + + it("finalizes only complete results from the same candidate resolution (#6691)", () => { + const plan = buildCandidatePlan("hermes", E2E_SOURCES); + const receipt = { + artifacts: [], + component: "hermes" as const, + nemoclawSha: SHA, + officialSource: "github:NousResearch/hermes-agent:release:1", + requestedCandidate: "v2026.7.1", + resolutionId: "d".repeat(64), + resolvedCandidate: "0.18.0", + resolvedCommit: "e".repeat(40), + schemaVersion: 1 as const, + }; + const results = ["source-unit", "integration", "e2e-support"].map((lane) => ({ + conclusion: "success" as const, + lane: lane as "source-unit" | "integration" | "e2e-support", + observedOutput: "hermes 0.18.0", + observedVersion: "0.18.0", + resolutionId: receipt.resolutionId, + })); + const evidence = finalizeEvidence({ attempt: "2", plan, receipt, results, runId: "123" }); + expect(evidence).toMatchObject({ + execution: { attempt: "2", runId: "123" }, + overall: "success", + schemaVersion: 1, + }); + expect(() => + finalizeEvidence({ attempt: "2", plan, receipt, results: results.slice(1), runId: "123" }), + ).toThrow("account for every selected deterministic lane exactly once"); + expect(() => + finalizeEvidence({ + attempt: "2", + plan, + receipt, + results: [{ ...results[0]!, resolutionId: "f".repeat(64) }, ...results.slice(1)], + runId: "123", + }), + ).toThrow("used a different candidate resolution"); + expect(() => + finalizeEvidence({ + attempt: "2", + plan, + receipt, + results: [{ ...results[0]!, observedVersion: "0.17.0" }, ...results.slice(1)], + runId: "123", + }), + ).toThrow("has no matching observed candidate version"); + }); +}); diff --git a/tools/candidate-compat.mts b/tools/candidate-compat.mts new file mode 100644 index 0000000000..7398b1b332 --- /dev/null +++ b/tools/candidate-compat.mts @@ -0,0 +1,919 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import { basename, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const COMPONENTS = ["openshell", "openclaw", "hermes", "dcode"] as const; +export type CandidateComponent = (typeof COMPONENTS)[number]; + +export type Artifact = { + digest: string; + digestAlgorithm: "sha256" | "sha512"; + kind: "archive" | "npm" | "wheel"; + name: string; + url: string; +}; + +export type CandidateReceipt = { + artifacts: Artifact[]; + component: CandidateComponent; + nemoclawSha: string; + officialSource: string; + requestedCandidate: string; + resolutionId: string; + resolvedCandidate: string; + resolvedCommit?: string; + schemaVersion: 1; +}; + +export type CandidatePlan = { + component: CandidateComponent; + deterministic: Array<{ + id: DeterministicLane; + reason: string; + status: "selected" | "skipped"; + }>; + live: Array<{ + id: string; + reason: string; + selector: "job" | "target"; + status: "skipped"; + }>; + schemaVersion: 1; +}; + +type LaneResult = { + conclusion: "failure" | "success"; + lane: DeterministicLane; + observedOutput?: string; + observedVersion?: string; + resolutionId: string; +}; + +type FetchLike = (input: string, init?: RequestInit) => Promise; +type DeterministicLane = + | "source-unit" + | "integration" + | "installer" + | "package-contract" + | "plugin" + | "e2e-support"; + +const FULL_SHA = /^[a-f0-9]{40}$/; +const VERSION = /^[0-9]+(?:\.[0-9]+){2}(?:[-+][0-9A-Za-z.-]+)?$/; +const HERMES_TAG = /^v[0-9]+(?:\.[0-9]+){2}$/; +const DIGEST = /^[a-f0-9]+$/; +const OFFICIAL_HOSTS = new Set([ + "api.github.com", + "codeload.github.com", + "files.pythonhosted.org", + "github.com", + "registry.npmjs.org", +]); +const DOWNLOAD_HOSTS = new Set([...OFFICIAL_HOSTS, "release-assets.githubusercontent.com"]); +const MAX_DOWNLOAD_REDIRECTS = 5; +const ALL_LANES: DeterministicLane[] = [ + "source-unit", + "integration", + "installer", + "package-contract", + "plugin", + "e2e-support", +]; + +const SELECTED_LANES: Record> = { + openshell: new Set(["source-unit", "integration", "installer", "e2e-support"]), + openclaw: new Set(["source-unit", "integration", "package-contract", "plugin", "e2e-support"]), + hermes: new Set(["source-unit", "integration", "e2e-support"]), + dcode: new Set(["source-unit", "integration", "e2e-support"]), +}; + +const LIVE_SELECTORS: Record< + CandidateComponent, + ReadonlyArray<{ id: string; selector: "job" | "target" }> +> = { + openshell: [ + { id: "full-e2e", selector: "job" }, + { id: "openshell-gateway-auth-contract", selector: "job" }, + ], + openclaw: [ + { id: "full-e2e", selector: "job" }, + { id: "openclaw-skill-cli", selector: "job" }, + ], + hermes: [{ id: "hermes-e2e", selector: "job" }], + dcode: [{ id: "ubuntu-repo-cloud-langchain-deepagents-code", selector: "target" }], +}; + +function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .filter(([, item]) => item !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function assertComponent(value: string): asserts value is CandidateComponent { + if (!COMPONENTS.includes(value as CandidateComponent)) { + throw new Error(`component must be one of: ${COMPONENTS.join(", ")}`); + } +} + +function assertCandidate(component: CandidateComponent, candidate: string): void { + if (candidate.length > 128 || candidate.includes("/") || /[\x00-\x20\x7f]/u.test(candidate)) { + throw new Error("candidate contains unsafe characters"); + } + if (component === "hermes") { + if (!HERMES_TAG.test(candidate)) throw new Error("Hermes candidate must match vX.Y.Z"); + return; + } + const normalized = component === "openshell" ? candidate.replace(/^v/u, "") : candidate; + if (!VERSION.test(normalized)) { + throw new Error(`${component} candidate must be an exact version`); + } +} + +function assertOfficialUrl(raw: string, expectedPathPrefix: string): URL { + const url = new URL(raw); + if (url.protocol !== "https:" || !OFFICIAL_HOSTS.has(url.hostname)) { + throw new Error(`candidate artifact is not on an approved official HTTPS host: ${raw}`); + } + if (!url.pathname.startsWith(expectedPathPrefix)) { + throw new Error(`candidate artifact has unexpected official-source path: ${raw}`); + } + if (url.username || url.password || url.search || url.hash) { + throw new Error( + `candidate artifact URL must not contain credentials, query, or fragment: ${raw}`, + ); + } + return url; +} + +async function fetchJson(fetcher: FetchLike, url: string, token?: string): Promise { + const response = await fetcher(url, { + headers: { + Accept: "application/vnd.github+json, application/json", + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + redirect: "error", + }); + if (!response.ok) + throw new Error(`official metadata request failed (${response.status}): ${url}`); + return response.json(); +} + +function record(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as Record; +} + +function stringField(value: Record, key: string, label: string): string { + const field = value[key]; + if (typeof field !== "string" || field.length === 0) { + throw new Error(`${label}.${key} must be a non-empty string`); + } + return field; +} + +function artifact(input: Artifact, expectedPathPrefix: string): Artifact { + const url = assertOfficialUrl(input.url, expectedPathPrefix); + if (!DIGEST.test(input.digest)) throw new Error(`${input.name} has an invalid digest`); + const expectedLength = input.digestAlgorithm === "sha256" ? 64 : 128; + if (input.digest.length !== expectedLength) { + throw new Error(`${input.name} has an invalid ${input.digestAlgorithm} digest`); + } + return { ...input, url: url.href }; +} + +async function resolveOpenShell( + candidate: string, + fetcher: FetchLike, + token?: string, +): Promise> { + const version = candidate.replace(/^v/u, ""); + const tag = `v${version}`; + const metadata = record( + await fetchJson( + fetcher, + `https://api.github.com/repos/NVIDIA/OpenShell/releases/tags/${tag}`, + token, + ), + "OpenShell release", + ); + if (metadata.draft === true) throw new Error("OpenShell candidate release is a draft"); + if (stringField(metadata, "tag_name", "OpenShell release") !== tag) { + throw new Error("OpenShell release tag does not match the requested candidate"); + } + if (!Array.isArray(metadata.assets)) throw new Error("OpenShell release assets must be an array"); + const name = "openshell-x86_64-unknown-linux-musl.tar.gz"; + const assetMetadata = metadata.assets + .map((item) => record(item, "OpenShell asset")) + .find((item) => item.name === name); + if (!assetMetadata) throw new Error(`OpenShell release is missing ${name}`); + const digest = stringField(assetMetadata, "digest", "OpenShell asset"); + if (!digest.startsWith("sha256:")) + throw new Error("OpenShell asset is missing SHA-256 provenance"); + return { + artifacts: [ + artifact( + { + digest: digest.slice("sha256:".length), + digestAlgorithm: "sha256", + kind: "archive", + name, + url: stringField(assetMetadata, "browser_download_url", "OpenShell asset"), + }, + `/NVIDIA/OpenShell/releases/download/${tag}/`, + ), + ], + component: "openshell", + officialSource: `github:NVIDIA/OpenShell:release:${String(metadata.id)}`, + requestedCandidate: candidate, + resolvedCandidate: version, + }; +} + +function npmArtifact(metadata: Record, packageName: string): Artifact { + const dist = record(metadata.dist, `${packageName} dist`); + const integrity = stringField(dist, "integrity", `${packageName} dist`); + if (!integrity.startsWith("sha512-")) + throw new Error(`${packageName} requires sha512 npm integrity`); + let digest: string; + try { + digest = Buffer.from(integrity.slice("sha512-".length), "base64").toString("hex"); + } catch { + throw new Error(`${packageName} has invalid npm integrity`); + } + return artifact( + { + digest, + digestAlgorithm: "sha512", + kind: "npm", + name: `${packageName}-${stringField(metadata, "version", packageName)}.tgz`, + url: stringField(dist, "tarball", `${packageName} dist`), + }, + `/${packageName}/-/`, + ); +} + +async function resolveNpm( + component: "openclaw", + candidate: string, + fetcher: FetchLike, +): Promise> { + const packageName = component; + const metadata = record( + await fetchJson(fetcher, `https://registry.npmjs.org/${packageName}/${candidate}`), + `${packageName} metadata`, + ); + const resolvedCandidate = stringField(metadata, "version", `${packageName} metadata`); + if (resolvedCandidate !== candidate) + throw new Error(`${packageName} metadata resolved a different version`); + return { + artifacts: [npmArtifact(metadata, packageName)], + component, + officialSource: `npm:${packageName}@${resolvedCandidate}`, + requestedCandidate: candidate, + resolvedCandidate, + }; +} + +async function peelGitTag(fetcher: FetchLike, candidate: string, token?: string): Promise { + let object = record( + record( + await fetchJson( + fetcher, + `https://api.github.com/repos/NousResearch/hermes-agent/git/ref/tags/${candidate}`, + token, + ), + "Hermes tag ref", + ).object, + "Hermes tag object", + ); + for (let depth = 0; depth < 2 && object.type === "tag"; depth += 1) { + const tag = record( + await fetchJson(fetcher, stringField(object, "url", "Hermes tag object"), token), + "Hermes annotated tag", + ); + object = record(tag.object, "Hermes annotated tag object"); + } + if (object.type !== "commit") throw new Error("Hermes tag does not resolve to a commit"); + const commit = stringField(object, "sha", "Hermes tag object"); + if (!FULL_SHA.test(commit)) throw new Error("Hermes tag resolved an invalid commit SHA"); + return commit; +} + +async function resolveHermes( + candidate: string, + fetcher: FetchLike, + token?: string, +): Promise> { + const release = record( + await fetchJson( + fetcher, + `https://api.github.com/repos/NousResearch/hermes-agent/releases/tags/${candidate}`, + token, + ), + "Hermes release", + ); + if (release.draft === true) throw new Error("Hermes candidate release is a draft"); + if (stringField(release, "tag_name", "Hermes release") !== candidate) { + throw new Error("Hermes release tag does not match the requested candidate"); + } + const commit = await peelGitTag(fetcher, candidate, token); + const pyprojectResponse = await fetcher( + `https://raw.githubusercontent.com/NousResearch/hermes-agent/${commit}/pyproject.toml`, + { redirect: "error" }, + ); + if (!pyprojectResponse.ok) + throw new Error("failed to read Hermes package version at resolved commit"); + const pyproject = await pyprojectResponse.text(); + const version = pyproject.match(/^version\s*=\s*"([^"]+)"\s*$/mu)?.[1]; + if (!version || !VERSION.test(version)) + throw new Error("Hermes pyproject has no exact package version"); + const npmMetadata = record( + await fetchJson(fetcher, `https://registry.npmjs.org/hermes-agent/${version}`), + "Hermes npm metadata", + ); + if (stringField(npmMetadata, "version", "Hermes npm metadata") !== version) { + throw new Error("Hermes npm package version does not match the resolved source"); + } + const wheel = await resolvePypiWheel("hermes-agent", version, fetcher); + return { + artifacts: [npmArtifact(npmMetadata, "hermes-agent"), wheel], + component: "hermes", + officialSource: `github:NousResearch/hermes-agent:release:${String(release.id)}`, + requestedCandidate: candidate, + resolvedCandidate: version, + resolvedCommit: commit, + }; +} + +async function resolvePypiWheel( + packageName: string, + version: string, + fetcher: FetchLike, +): Promise { + const metadata = record( + await fetchJson(fetcher, `https://pypi.org/pypi/${packageName}/${version}/json`), + `${packageName} PyPI metadata`, + ); + const info = record(metadata.info, `${packageName} PyPI info`); + if (stringField(info, "name", `${packageName} PyPI info`).toLowerCase() !== packageName) { + throw new Error(`PyPI returned a different package for ${packageName}`); + } + if (stringField(info, "version", `${packageName} PyPI info`) !== version) { + throw new Error(`PyPI resolved a different ${packageName} version`); + } + if (!Array.isArray(metadata.urls)) throw new Error(`${packageName} PyPI files must be an array`); + const wheel = metadata.urls + .map((item) => record(item, `${packageName} PyPI file`)) + .find((item) => item.packagetype === "bdist_wheel" && item.yanked !== true); + if (!wheel) throw new Error(`${packageName} release has no non-yanked wheel`); + const digests = record(wheel.digests, `${packageName} PyPI file digests`); + return artifact( + { + digest: stringField(digests, "sha256", `${packageName} PyPI file digests`), + digestAlgorithm: "sha256", + kind: "wheel", + name: stringField(wheel, "filename", `${packageName} PyPI file`), + url: stringField(wheel, "url", `${packageName} PyPI file`), + }, + "/packages/", + ); +} + +async function resolveDcode( + candidate: string, + fetcher: FetchLike, +): Promise> { + const wheel = await resolvePypiWheel("deepagents-code", candidate, fetcher); + return { + artifacts: [wheel], + component: "dcode", + officialSource: `pypi:deepagents-code==${candidate}`, + requestedCandidate: candidate, + resolvedCandidate: candidate, + }; +} + +export async function resolveCandidate(input: { + candidate: string; + component: string; + fetcher?: FetchLike; + githubToken?: string; + nemoclawSha: string; +}): Promise { + assertComponent(input.component); + if (!FULL_SHA.test(input.nemoclawSha)) throw new Error("NemoClaw ref must resolve to a full SHA"); + assertCandidate(input.component, input.candidate); + const fetcher = input.fetcher ?? fetch; + const resolved = + input.component === "openshell" + ? await resolveOpenShell(input.candidate, fetcher, input.githubToken) + : input.component === "openclaw" + ? await resolveNpm(input.component, input.candidate, fetcher) + : input.component === "hermes" + ? await resolveHermes(input.candidate, fetcher, input.githubToken) + : await resolveDcode(input.candidate, fetcher); + const receiptWithoutId = { + ...resolved, + nemoclawSha: input.nemoclawSha, + schemaVersion: 1 as const, + }; + return { + ...receiptWithoutId, + resolutionId: sha256(stableJson(receiptWithoutId)), + }; +} + +function parseArtifact( + value: unknown, + component: CandidateComponent, + resolvedCandidate: string, + index: number, +): Artifact { + const input = record(value, `candidate artifact ${index}`); + const digestAlgorithm = stringField(input, "digestAlgorithm", `candidate artifact ${index}`); + if (digestAlgorithm !== "sha256" && digestAlgorithm !== "sha512") { + throw new Error(`candidate artifact ${index} has an invalid digest algorithm`); + } + const kind = stringField(input, "kind", `candidate artifact ${index}`); + if (kind !== "archive" && kind !== "npm" && kind !== "wheel") { + throw new Error(`candidate artifact ${index} has an invalid kind`); + } + const name = stringField(input, "name", `candidate artifact ${index}`); + if (name !== basename(name) || name.length > 255) { + throw new Error(`candidate artifact ${index} has an unsafe name`); + } + + const expected = + component === "openshell" + ? { + kind: "archive", + name: "openshell-x86_64-unknown-linux-musl.tar.gz", + path: `/NVIDIA/OpenShell/releases/download/v${resolvedCandidate}/`, + } + : component === "openclaw" + ? { + kind: "npm", + name: `openclaw-${resolvedCandidate}.tgz`, + path: "/openclaw/-/", + } + : component === "hermes" && index === 0 + ? { + kind: "npm", + name: `hermes-agent-${resolvedCandidate}.tgz`, + path: "/hermes-agent/-/", + } + : { kind: "wheel", name: null, path: "/packages/" }; + if (kind !== expected.kind || (expected.name !== null && name !== expected.name)) { + throw new Error(`candidate artifact ${index} does not match the resolved component`); + } + if (kind === "wheel" && !/^[0-9A-Za-z_.+-]+\.whl$/u.test(name)) { + throw new Error(`candidate artifact ${index} has an invalid wheel name`); + } + return artifact( + { + digest: stringField(input, "digest", `candidate artifact ${index}`), + digestAlgorithm, + kind, + name, + url: stringField(input, "url", `candidate artifact ${index}`), + }, + expected.path, + ); +} + +/** Parse an uploaded resolver receipt and bind it to the trusted resolve-job output. */ +export function parseCandidateReceipt( + value: unknown, + expectedResolutionId: string, +): CandidateReceipt { + if (!/^[a-f0-9]{64}$/u.test(expectedResolutionId)) { + throw new Error("expected candidate resolution id is invalid"); + } + const input = record(value, "candidate receipt"); + if (input.schemaVersion !== 1) throw new Error("candidate receipt schema version must be 1"); + const component = stringField(input, "component", "candidate receipt"); + assertComponent(component); + const requestedCandidate = stringField(input, "requestedCandidate", "candidate receipt"); + assertCandidate(component, requestedCandidate); + const resolvedCandidate = stringField(input, "resolvedCandidate", "candidate receipt"); + if (!VERSION.test(resolvedCandidate)) { + throw new Error("candidate receipt has an invalid resolved version"); + } + if ( + (component === "openshell" && requestedCandidate.replace(/^v/u, "") !== resolvedCandidate) || + ((component === "openclaw" || component === "dcode") && + requestedCandidate !== resolvedCandidate) + ) { + throw new Error("candidate receipt resolved version does not match the request"); + } + const nemoclawSha = stringField(input, "nemoclawSha", "candidate receipt"); + if (!FULL_SHA.test(nemoclawSha)) throw new Error("candidate receipt has an invalid NemoClaw SHA"); + if (!Array.isArray(input.artifacts)) + throw new Error("candidate receipt artifacts must be an array"); + const expectedArtifactCount = component === "hermes" ? 2 : 1; + if (input.artifacts.length !== expectedArtifactCount) { + throw new Error(`candidate receipt for ${component} has an invalid artifact count`); + } + const artifacts = input.artifacts.map((item, index) => + parseArtifact(item, component, resolvedCandidate, index), + ); + const officialSource = stringField(input, "officialSource", "candidate receipt"); + const sourcePattern = + component === "openshell" + ? /^github:NVIDIA\/OpenShell:release:[1-9][0-9]*$/u + : component === "openclaw" + ? new RegExp( + `^npm:openclaw@${resolvedCandidate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, + "u", + ) + : component === "hermes" + ? /^github:NousResearch\/hermes-agent:release:[1-9][0-9]*$/u + : new RegExp( + `^pypi:deepagents-code==${resolvedCandidate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, + "u", + ); + if (!sourcePattern.test(officialSource)) { + throw new Error("candidate receipt has an invalid official source"); + } + const resolvedCommitValue = input.resolvedCommit; + const resolvedCommit = + resolvedCommitValue === undefined + ? undefined + : stringField(input, "resolvedCommit", "candidate receipt"); + if ( + (component === "hermes" && (!resolvedCommit || !FULL_SHA.test(resolvedCommit))) || + (component !== "hermes" && resolvedCommit !== undefined) + ) { + throw new Error("candidate receipt has an invalid resolved commit"); + } + const receiptWithoutId = { + artifacts, + component, + nemoclawSha, + officialSource, + requestedCandidate, + resolvedCandidate, + ...(resolvedCommit ? { resolvedCommit } : {}), + schemaVersion: 1 as const, + }; + const resolutionId = sha256(stableJson(receiptWithoutId)); + if ( + stringField(input, "resolutionId", "candidate receipt") !== resolutionId || + resolutionId !== expectedResolutionId + ) { + throw new Error("candidate receipt does not match the trusted resolution id"); + } + return { ...receiptWithoutId, resolutionId }; +} + +export function buildCandidatePlan( + component: CandidateComponent, + e2eSources: string, +): CandidatePlan { + const selected = SELECTED_LANES[component]; + const live = LIVE_SELECTORS[component].map((entry) => { + const marker = entry.selector === "job" ? ` ${entry.id}:\n` : `id: "${entry.id}"`; + if (!e2eSources.includes(marker)) { + throw new Error(`E2E source of truth does not declare ${entry.selector} ${entry.id}`); + } + return { + ...entry, + reason: "candidate injection is not yet wired into the credential-bearing E2E boundary", + status: "skipped" as const, + }; + }); + return { + component, + deterministic: ALL_LANES.map((id) => ({ + id, + reason: selected.has(id) + ? `the ${id} lane exercises ${component}-owned source or integration boundaries` + : `the ${id} lane does not exercise an ${component}-owned boundary`, + status: selected.has(id) ? "selected" : "skipped", + })), + live, + schemaVersion: 1, + }; +} + +export function verifyDigest(bytes: Uint8Array, artifactValue: Artifact): void { + const actual = createHash(artifactValue.digestAlgorithm).update(bytes).digest("hex"); + if (actual !== artifactValue.digest) { + throw new Error(`${artifactValue.name} ${artifactValue.digestAlgorithm} mismatch`); + } +} + +export function verifyObservedVersion(receipt: CandidateReceipt, output: string): string { + const escaped = receipt.resolvedCandidate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + if (!new RegExp(`(^|[^0-9A-Za-z.])v?${escaped}([^0-9A-Za-z.]|$)`, "u").test(output.trim())) { + throw new Error( + `observed ${receipt.component} version does not match ${receipt.resolvedCandidate}: ${output.trim()}`, + ); + } + return receipt.resolvedCandidate; +} + +export function finalizeEvidence(input: { + attempt: string; + plan: CandidatePlan; + receipt: CandidateReceipt; + results: LaneResult[]; + runId: string; +}): Record { + if (!/^[1-9][0-9]*$/u.test(input.runId) || !/^[1-9][0-9]*$/u.test(input.attempt)) { + throw new Error("run id and attempt must be positive integers"); + } + const selected = input.plan.deterministic + .filter((lane) => lane.status === "selected") + .map((lane) => lane.id) + .sort(); + const actual = input.results.map((result) => result.lane).sort(); + if (new Set(actual).size !== actual.length || stableJson(actual) !== stableJson(selected)) { + throw new Error( + "lane results do not account for every selected deterministic lane exactly once", + ); + } + for (const result of input.results) { + if (result.resolutionId !== input.receipt.resolutionId) { + throw new Error(`lane ${result.lane} used a different candidate resolution`); + } + if (result.conclusion !== "success" && result.conclusion !== "failure") { + throw new Error(`lane ${result.lane} has an invalid conclusion`); + } + if ( + result.conclusion === "success" && + (result.observedVersion !== input.receipt.resolvedCandidate || !result.observedOutput) + ) { + throw new Error(`lane ${result.lane} has no matching observed candidate version`); + } + } + return { + execution: { attempt: input.attempt, runId: input.runId }, + overall: input.results.every((result) => result.conclusion === "success") + ? "success" + : "failure", + plan: input.plan, + receipt: input.receipt, + results: [...input.results].sort((left, right) => left.lane.localeCompare(right.lane)), + schemaVersion: 1, + }; +} + +function run( + command: string, + args: string[], + options: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, +) { + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: "utf8", + env: options.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) { + const detail = result.error?.message ?? result.stderr?.trim() ?? result.stdout?.trim() ?? ""; + throw new Error(`${command} ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`); + } + return result.stdout.trim(); +} + +function assertDownloadUrl(raw: string): URL { + const url = new URL(raw); + if ( + url.protocol !== "https:" || + !DOWNLOAD_HOSTS.has(url.hostname) || + url.username || + url.password || + url.hash || + url.href.length > 4096 + ) { + throw new Error(`candidate download URL is not approved: ${raw}`); + } + return url; +} + +async function fetchDownload(fetcher: FetchLike, rawUrl: string): Promise { + let current = assertDownloadUrl(rawUrl); + for (let redirectCount = 0; redirectCount <= MAX_DOWNLOAD_REDIRECTS; redirectCount += 1) { + const response = await fetcher(current.href, { redirect: "manual" }); + if (![301, 302, 303, 307, 308].includes(response.status)) return response; + if (redirectCount === MAX_DOWNLOAD_REDIRECTS) { + throw new Error("candidate download exceeded the redirect limit"); + } + const location = response.headers.get("location"); + if (!location) throw new Error("candidate download redirect has no location"); + current = assertDownloadUrl(new URL(location, current).href); + } + throw new Error("candidate download redirect handling failed"); +} + +export async function downloadCandidateArtifact( + artifactValue: Artifact, + directory: string, + index: number, + fetcher: FetchLike = fetch, +): Promise { + const response = await fetchDownload(fetcher, artifactValue.url); + if (!response.ok) throw new Error(`candidate download failed (${response.status})`); + const bytes = new Uint8Array(await response.arrayBuffer()); + verifyDigest(bytes, artifactValue); + const extension = + artifactValue.kind === "archive" ? ".tar.gz" : artifactValue.kind === "npm" ? ".tgz" : ".whl"; + const target = join(directory, `candidate-${index}${extension}`); + await writeFile(target, bytes, { mode: 0o600 }); + return target; +} + +export async function materializeCandidate(receipt: CandidateReceipt, directory: string) { + await mkdir(directory, { recursive: true, mode: 0o700 }); + const artifactValue = receipt.artifacts[0]; + if (!artifactValue) throw new Error("candidate receipt has no artifact"); + const archives = await Promise.all( + receipt.artifacts.map((candidateArtifact, index) => + downloadCandidateArtifact(candidateArtifact, directory, index), + ), + ); + const archive = archives[0]!; + let binDirectory: string; + let observedOutput: string; + if (receipt.component === "openshell") { + const extractDirectory = join(directory, "openshell"); + await mkdir(extractDirectory, { mode: 0o700 }); + run("tar", ["-xzf", archive, "-C", extractDirectory]); + const binary = run("find", [ + extractDirectory, + "-type", + "f", + "-name", + "openshell", + "-print", + "-quit", + ]); + if (!binary) throw new Error("OpenShell archive has no openshell binary"); + await chmod(binary, 0o700); + binDirectory = resolve(binary, ".."); + observedOutput = run(binary, ["--version"]); + } else if (receipt.component === "dcode" || receipt.component === "hermes") { + const venv = join(directory, "venv"); + run("python3", ["-m", "venv", venv]); + const wheel = + receipt.component === "dcode" + ? archive + : archives[receipt.artifacts.findIndex((item) => item.kind === "wheel")]; + if (!wheel) throw new Error(`${receipt.component} receipt has no verified wheel`); + run(join(venv, "bin", "python"), ["-m", "pip", "install", "--no-deps", "--no-index", wheel]); + binDirectory = join(venv, "bin"); + const packageName = receipt.component === "dcode" ? "deepagents-code" : "hermes-agent"; + observedOutput = run(join(venv, "bin", "python"), [ + "-c", + `import importlib.metadata; print(importlib.metadata.version("${packageName}"))`, + ]); + } else { + const prefix = join(directory, "npm"); + run("npm", [ + "install", + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--prefix", + prefix, + archive, + ]); + binDirectory = join(prefix, "node_modules", ".bin"); + const executable = receipt.component === "openclaw" ? "openclaw" : "hermes"; + observedOutput = run(join(binDirectory, executable), ["--version"]); + } + const observedVersion = verifyObservedVersion(receipt, observedOutput); + return { + binDirectory, + component: receipt.component, + observedOutput, + observedVersion, + resolutionId: receipt.resolutionId, + schemaVersion: 1 as const, + }; +} + +function parseArgs(argv: string[]): Map { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith("--") || value === undefined) + throw new Error(`invalid argument: ${key ?? ""}`); + if (values.has(key)) throw new Error(`duplicate argument: ${key}`); + values.set(key, value); + } + return values; +} + +function required(args: Map, name: string): string { + const value = args.get(name); + if (!value) throw new Error(`${name} is required`); + return value; +} + +async function main(argv: string[]): Promise { + const [command, ...rest] = argv; + const args = parseArgs(rest); + if (command === "resolve") { + const receipt = await resolveCandidate({ + candidate: required(args, "--candidate"), + component: required(args, "--component"), + githubToken: process.env.GITHUB_TOKEN, + nemoclawSha: required(args, "--nemoclaw-sha"), + }); + await writeFile(required(args, "--output"), `${stableJson(receipt)}\n`, { mode: 0o600 }); + return; + } + if (command === "plan") { + const component = required(args, "--component"); + assertComponent(component); + const sources = await Promise.all([ + readFile(required(args, "--e2e-workflow"), "utf8"), + readFile(required(args, "--e2e-registry"), "utf8"), + ]); + const plan = buildCandidatePlan(component, sources.join("\n")); + await writeFile(required(args, "--output"), `${stableJson(plan)}\n`, { mode: 0o600 }); + return; + } + if (command === "materialize") { + const receipt = parseCandidateReceipt( + JSON.parse(await readFile(required(args, "--receipt"), "utf8")), + required(args, "--resolution-id"), + ); + const observed = await materializeCandidate(receipt, required(args, "--directory")); + await writeFile(required(args, "--output"), `${stableJson(observed)}\n`, { mode: 0o600 }); + const githubEnv = args.get("--github-env"); + if (githubEnv) { + await writeFile( + githubEnv, + [ + `PATH=${observed.binDirectory}:${process.env.PATH ?? ""}`, + `NEMOCLAW_CANDIDATE_COMPONENT=${receipt.component}`, + `NEMOCLAW_CANDIDATE_RECEIPT=${resolve(required(args, "--receipt"))}`, + `NEMOCLAW_CANDIDATE_RESOLUTION_ID=${receipt.resolutionId}`, + `NEMOCLAW_CANDIDATE_VERSION=${receipt.resolvedCandidate}`, + "", + ].join("\n"), + { flag: "a" }, + ); + } + return; + } + if (command === "finalize") { + const receipt = JSON.parse( + await readFile(required(args, "--receipt"), "utf8"), + ) as CandidateReceipt; + const plan = JSON.parse(await readFile(required(args, "--plan"), "utf8")) as CandidatePlan; + const resultDirectory = required(args, "--results"); + const resultFiles = (await readdir(resultDirectory, { recursive: true })).filter((path) => + path.endsWith(".json"), + ); + const results = await Promise.all( + plan.deterministic + .filter((lane) => lane.status === "selected") + .map(async (lane) => { + const matches = resultFiles.filter((path) => basename(path) === `${lane.id}.json`); + if (matches.length !== 1) { + throw new Error(`expected exactly one result file for lane ${lane.id}`); + } + return JSON.parse(await readFile(join(resultDirectory, matches[0]!), "utf8")); + }), + ); + const evidence = finalizeEvidence({ + attempt: required(args, "--attempt"), + plan, + receipt, + results, + runId: required(args, "--run-id"), + }); + await writeFile(required(args, "--output"), `${stableJson(evidence)}\n`, { mode: 0o600 }); + return; + } + throw new Error("command must be resolve, plan, materialize, or finalize"); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)).catch((error) => { + console.error(`candidate-compat: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + }); +} From 375e45941972cd1a7add2710bea8773d20f97a08 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Tue, 14 Jul 2026 09:44:20 -0700 Subject: [PATCH 2/4] fix(ci): bind OpenShell candidate runtime evidence Signed-off-by: Ho Lim --- .../workflows/candidate-compatibility.yaml | 208 +++-- ci/source-shape-test-budget.json | 2 +- test/candidate-compat.test.ts | 451 ++++------ ...ll-gateway-auth-source-contract-helpers.ts | 5 +- ...shell-gateway-auth-source-contract.test.ts | 2 +- tools/candidate-compat.mts | 850 +++++++++--------- 6 files changed, 791 insertions(+), 727 deletions(-) diff --git a/.github/workflows/candidate-compatibility.yaml b/.github/workflows/candidate-compatibility.yaml index 8a1f2d9b29..c857ccf81e 100644 --- a/.github/workflows/candidate-compatibility.yaml +++ b/.github/workflows/candidate-compatibility.yaml @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: Dependencies / Candidate Compatibility -run-name: "Candidate ${{ inputs.component }} ${{ inputs.candidate }} at ${{ inputs.nemoclaw_ref }}" +name: Dependencies / OpenShell Candidate Compatibility +run-name: "OpenShell candidate ${{ inputs.candidate }} at ${{ inputs.nemoclaw_ref }}" on: workflow_dispatch: @@ -17,11 +17,8 @@ on: type: choice options: - openshell - - openclaw - - hermes - - dcode candidate: - description: Exact official version, or vX.Y.Z release tag for Hermes. + description: Exact official OpenShell version or vX.Y.Z release tag. required: true type: string @@ -164,12 +161,13 @@ jobs: continue-on-error: true env: LANE: ${{ matrix.lane }} + RESOLUTION_ID: ${{ needs.resolve.outputs.resolution_id }} shell: bash run: | set -euo pipefail node --experimental-strip-types controller/tools/candidate-compat.mts materialize \ --receipt candidate-input/candidate-receipt.json \ - --resolution-id "${{ needs.resolve.outputs.resolution_id }}" \ + --resolution-id "$RESOLUTION_ID" \ --directory "${RUNNER_TEMP}/candidate-runtime" \ --output "candidate-observed-${LANE}.json" \ --github-env "$GITHUB_ENV" 2>&1 | tee "candidate-materialize-${LANE}.log" @@ -185,40 +183,13 @@ jobs: run: | set -euo pipefail exec > >(tee "candidate-lane-${LANE}.log") 2>&1 - case "$LANE" in - source-unit) - npx vitest run --project cli - ;; - integration) - npm run clean:cli - npm run build:cli - npx vitest run --project integration - ;; - installer) - npx vitest run --project installer-integration - ;; - package-contract) - npm ci --prefix nemoclaw --ignore-scripts - npm run clean:cli - npm --prefix nemoclaw run clean - npm run build:cli - npm --prefix nemoclaw run build - npx vitest run --project package-contract - ;; - plugin) - npm ci --prefix nemoclaw --ignore-scripts - npx vitest run --project plugin - ;; - e2e-support) - npx vitest run --project e2e-support - ;; - *) - echo "::error::untrusted lane id: $LANE" - exit 1 - ;; - esac - - - name: Record lane result + [[ "$LANE" == installer ]] || { echo "::error::untrusted lane id: $LANE"; exit 1; } + "$OPENSHELL_BIN" --version + "$OPENSHELL_GATEWAY_BIN" --version + "$NEMOCLAW_OPENSHELL_SANDBOX_BIN" --version + npx vitest run --project installer-integration + + - name: Record receipt-bound lane result if: ${{ always() }} env: CANDIDATE_OUTCOME: ${{ steps.candidate.outcome }} @@ -228,24 +199,24 @@ jobs: shell: bash run: | set -euo pipefail - conclusion=failure + mkdir -p candidate-results if [[ "$CANDIDATE_OUTCOME" == success && "$LANE_OUTCOME" == success ]]; then - conclusion=success + node --experimental-strip-types controller/tools/candidate-compat.mts verify-invocations \ + --receipt candidate-input/candidate-receipt.json \ + --resolution-id "$RESOLUTION_ID" \ + --log "$NEMOCLAW_CANDIDATE_INVOCATION_LOG" \ + --lane "$LANE" \ + --output "candidate-results/${LANE}.json" + else + LANE_NAME="$LANE" node -e ' + const fs = require("node:fs"); + fs.writeFileSync(`candidate-results/${process.env.LANE_NAME}.json`, JSON.stringify({ + conclusion: "failure", + lane: process.env.LANE_NAME, + resolutionId: process.env.RESOLUTION_ID, + }) + "\n", {mode: 0o600}); + ' fi - mkdir -p candidate-results - CONCLUSION="$conclusion" node -e ' - const fs = require("node:fs"); - const observedPath = `candidate-observed-${process.env.LANE}.json`; - const observed = fs.existsSync(observedPath) ? JSON.parse(fs.readFileSync(observedPath, "utf8")) : {}; - const result = { - conclusion: process.env.CONCLUSION, - lane: process.env.LANE, - observedOutput: observed.observedOutput, - observedVersion: observed.observedVersion, - resolutionId: process.env.RESOLUTION_ID, - }; - fs.writeFileSync(`candidate-results/${process.env.LANE}.json`, JSON.stringify(result) + "\n", {mode: 0o600}); - ' - name: Upload lane evidence if: ${{ always() }} @@ -264,11 +235,129 @@ jobs: if: ${{ always() && (steps.candidate.outcome != 'success' || steps.lane.outcome != 'success') }} run: exit 1 + live: + needs: resolve + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Check out trusted compatibility controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.sha }} + path: controller + fetch-depth: 1 + persist-credentials: false + + - name: Check out resolved NemoClaw commit + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ needs.resolve.outputs.nemoclaw_sha }} + path: candidate-source + fetch-depth: 1 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + cache: npm + cache-dependency-path: candidate-source/package-lock.json + + - name: Install repository dependencies + working-directory: candidate-source + run: npm ci --ignore-scripts + + - name: Download immutable resolution and plan + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: candidate-resolution-${{ needs.resolve.outputs.resolution_id }} + path: candidate-input + + - id: candidate + name: Materialize verified OpenShell runtime + continue-on-error: true + env: + RESOLUTION_ID: ${{ needs.resolve.outputs.resolution_id }} + run: | + set -euo pipefail + node --experimental-strip-types controller/tools/candidate-compat.mts materialize \ + --receipt candidate-input/candidate-receipt.json \ + --resolution-id "$RESOLUTION_ID" \ + --directory "${RUNNER_TEMP}/candidate-runtime" \ + --output candidate-live-observed.json \ + --github-env "$GITHUB_ENV" + + - id: live_test + name: Run OpenShell gateway auth contract against candidate + if: ${{ steps.candidate.outcome == 'success' }} + continue-on-error: true + working-directory: candidate-source + env: + DOCKER_GRPC_PROBE_IMAGE: node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d + E2E_ARTIFACT_DIR: ${{ github.workspace }}/candidate-source/e2e-artifacts/live/openshell-gateway-auth-contract + E2E_JOB: "1" + E2E_TARGET_ID: openshell-gateway-auth-contract + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_RUN_LIVE_E2E: "1" + run: | + set -euo pipefail + npm run build:cli + docker pull "$DOCKER_GRPC_PROBE_IMAGE" + "$OPENSHELL_GATEWAY_BIN" --version + npx vitest run --project e2e-live \ + test/e2e/live/openshell-gateway-auth-source-contract.test.ts \ + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts + + - name: Record receipt-bound live result + if: ${{ always() }} + env: + CANDIDATE_OUTCOME: ${{ steps.candidate.outcome }} + LANE_OUTCOME: ${{ steps.live_test.outcome }} + RESOLUTION_ID: ${{ needs.resolve.outputs.resolution_id }} + run: | + set -euo pipefail + mkdir -p candidate-results + lane=live:openshell-gateway-auth-contract + if [[ "$CANDIDATE_OUTCOME" == success && "$LANE_OUTCOME" == success ]]; then + node --experimental-strip-types controller/tools/candidate-compat.mts verify-invocations \ + --receipt candidate-input/candidate-receipt.json \ + --resolution-id "$RESOLUTION_ID" \ + --log "$NEMOCLAW_CANDIDATE_INVOCATION_LOG" \ + --lane "$lane" \ + --output candidate-results/live-openshell-gateway-auth-contract.json + else + LANE_NAME="$lane" node -e ' + const fs = require("node:fs"); + fs.writeFileSync("candidate-results/live-openshell-gateway-auth-contract.json", JSON.stringify({ + conclusion: "failure", + lane: process.env.LANE_NAME, + resolutionId: process.env.RESOLUTION_ID, + }) + "\n", {mode: 0o600}); + ' + fi + + - name: Upload live evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: candidate-result-${{ needs.resolve.outputs.resolution_id }}-live-openshell-gateway-auth-contract + path: | + candidate-results/live-openshell-gateway-auth-contract.json + candidate-live-observed.json + candidate-source/e2e-artifacts/live/openshell-gateway-auth-contract/ + if-no-files-found: error + retention-days: 30 + + - name: Enforce live result + if: ${{ always() && (steps.candidate.outcome != 'success' || steps.live_test.outcome != 'success') }} + run: exit 1 + evidence: if: ${{ always() && needs.resolve.result == 'success' }} needs: - resolve - deterministic + - live runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -321,7 +410,7 @@ jobs: console.log(`| \`${lane.id}\` | ${lane.status} | ${results.get(lane.id) ?? lane.reason} |`); } for (const lane of evidence.plan.live) { - console.log(`| \`e2e:${lane.id}\` | ${lane.status} | ${lane.reason} |`); + console.log(`| \`e2e:${lane.id}\` | ${lane.status} | ${results.get(`live:${lane.id}`) ?? lane.reason} |`); } NODE @@ -340,4 +429,5 @@ jobs: - name: Enforce aggregate result env: DETERMINISTIC_RESULT: ${{ needs.deterministic.result }} - run: test "$DETERMINISTIC_RESULT" = success + LIVE_RESULT: ${{ needs.live.result }} + run: test "$DETERMINISTIC_RESULT" = success && test "$LIVE_RESULT" = success diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 38c156e641..acff07a4bc 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -63,7 +63,7 @@ }, { "file": "test/candidate-compat.test.ts", - "test": "keeps the manual controller read-only and separate from candidate source (#6691)", + "test": "keeps the manual controller read-only and runs digest-bound deterministic and live lanes (#6691)", "category": "security" }, { diff --git a/test/candidate-compat.test.ts b/test/candidate-compat.test.ts index bf887f4eba..2e992a557f 100644 --- a/test/candidate-compat.test.ts +++ b/test/candidate-compat.test.ts @@ -2,9 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import { createHash } from "node:crypto"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; import { describe, expect, it } from "vitest"; import { parse as parseYaml } from "yaml"; @@ -13,16 +14,29 @@ import { buildCandidatePlan, downloadCandidateArtifact, finalizeEvidence, + materializeCandidate, parseCandidateReceipt, resolveCandidate, + verifyCandidateInvocations, verifyDigest, verifyObservedVersion, + type CandidateReceipt, } from "../tools/candidate-compat.mts"; const SHA = "a".repeat(40); -const E2E_SOURCES = [".github/workflows/e2e.yaml", "test/e2e/registry/definitions/baseline.ts"] - .map((path) => readFileSync(resolve(path), "utf8")) - .join("\n"); +const VERSION = "0.0.82"; +const E2E_SOURCES = readFileSync(resolve(".github/workflows/e2e.yaml"), "utf8"); +const ASSETS = [ + { name: "openshell-x86_64-unknown-linux-musl.tar.gz", role: "cli" as const }, + { + name: "openshell-gateway-x86_64-unknown-linux-gnu.tar.gz", + role: "gateway" as const, + }, + { + name: "openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz", + role: "sandbox" as const, + }, +]; function response(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { @@ -31,9 +45,43 @@ function response(body: unknown, status = 200): Response { }); } -describe("candidate compatibility contract", () => { - // source-shape-contract: security -- This pins the trusted controller and read-only workflow boundary around candidate code execution. - it("keeps the manual controller read-only and separate from candidate source (#6691)", () => { +function releaseMetadata() { + return { + assets: ASSETS.map(({ name }, index) => ({ + browser_download_url: `https://github.com/NVIDIA/OpenShell/releases/download/v${VERSION}/${name}`, + digest: `sha256:${String(index + 1).repeat(64)}`, + name, + })), + draft: false, + id: 82, + tag_name: `v${VERSION}`, + }; +} + +function receipt(overrides: Partial = {}): CandidateReceipt { + const base = { + artifacts: ASSETS.map(({ name, role }, index) => ({ + digest: String(index + 1).repeat(64), + digestAlgorithm: "sha256" as const, + kind: "archive" as const, + name, + role, + url: `https://github.com/NVIDIA/OpenShell/releases/download/v${VERSION}/${name}`, + })), + component: "openshell" as const, + nemoclawSha: SHA, + officialSource: "github:NVIDIA/OpenShell:release:82", + requestedCandidate: `v${VERSION}`, + resolutionId: "d".repeat(64), + resolvedCandidate: VERSION, + schemaVersion: 1 as const, + }; + return { ...base, ...overrides }; +} + +describe("OpenShell candidate compatibility contract", () => { + // source-shape-contract: security -- The controller is read-only and candidate code is isolated from provenance resolution. + it("keeps the manual controller read-only and runs digest-bound deterministic and live lanes (#6691)", () => { const source = readFileSync(resolve(".github/workflows/candidate-compatibility.yaml"), "utf8"); const workflow = parseYaml(source) as { jobs: Record; @@ -46,23 +94,32 @@ describe("candidate compatibility contract", () => { "nemoclaw_ref", ]); expect(workflow.permissions).toEqual({ contents: "read" }); - expect(Object.keys(workflow.jobs).sort()).toEqual(["deterministic", "evidence", "resolve"]); + expect(Object.keys(workflow.jobs).sort()).toEqual([ + "deterministic", + "evidence", + "live", + "resolve", + ]); expect(source).toContain("candidate compatibility must be dispatched from main"); expect(source).toContain("path: controller"); expect(source).toContain("path: candidate-source"); - expect(source).toContain("git -C candidate-source rev-parse --verify HEAD^{commit}"); - const toolSource = readFileSync(resolve("tools/candidate-compat.mts"), "utf8"); - expect(toolSource).toContain("NEMOCLAW_CANDIDATE_RECEIPT"); - expect(source).toContain('--resolution-id "${{ needs.resolve.outputs.resolution_id }}"'); + expect(source).toContain("RESOLUTION_ID: ${{ needs.resolve.outputs.resolution_id }}"); + expect(source).toContain("verify-invocations"); + expect(source).toContain("openshell-gateway-auth-source-contract.test.ts"); expect(source).not.toMatch(/\b(?:git push|gh pr|npm publish|docker push)\b/u); }); - it("rejects ambiguous and unsafe candidate input before metadata access (#6691)", async () => { + it("rejects unsupported, ambiguous, and unsafe candidate input before metadata access (#6691)", async () => { const fetcher = async () => { throw new Error("metadata must not be fetched"); }; await expect( - resolveCandidate({ candidate: "latest", component: "openclaw", fetcher, nemoclawSha: SHA }), + resolveCandidate({ + candidate: "latest", + component: "openshell", + fetcher, + nemoclawSha: SHA, + }), ).rejects.toThrow("exact version"); await expect( resolveCandidate({ @@ -75,235 +132,85 @@ describe("candidate compatibility contract", () => { await expect( resolveCandidate({ candidate: "1.2.3", - component: "unknown", + component: "hermes", fetcher, nemoclawSha: SHA, }), - ).rejects.toThrow("component must be one of"); + ).rejects.toThrow("component must be one of: openshell"); }); - it("binds OpenShell evidence to an official release asset digest (#6691)", async () => { - const fetcher = async () => - response({ - assets: [ - { - browser_download_url: - "https://github.com/NVIDIA/OpenShell/releases/download/v0.0.82/openshell-x86_64-unknown-linux-musl.tar.gz", - digest: `sha256:${"b".repeat(64)}`, - name: "openshell-x86_64-unknown-linux-musl.tar.gz", - }, - ], - draft: false, - id: 82, - tag_name: "v0.0.82", - }); + it("binds all required Linux runtime assets to official release digests (#6691)", async () => { + const fetcher = async () => response(releaseMetadata()); const first = await resolveCandidate({ - candidate: "v0.0.82", + candidate: `v${VERSION}`, component: "openshell", fetcher, nemoclawSha: SHA, }); const rerun = await resolveCandidate({ - candidate: "v0.0.82", + candidate: `v${VERSION}`, component: "openshell", fetcher, nemoclawSha: SHA, }); expect(first).toEqual(rerun); - expect(first).toMatchObject({ - nemoclawSha: SHA, - officialSource: "github:NVIDIA/OpenShell:release:82", - requestedCandidate: "v0.0.82", - resolvedCandidate: "0.0.82", - }); - expect(first.artifacts[0]?.digest).toBe("b".repeat(64)); + expect(first.artifacts.map(({ name, role }) => ({ name, role }))).toEqual(ASSETS); expect(first.resolutionId).toMatch(/^[a-f0-9]{64}$/u); expect(parseCandidateReceipt(first, first.resolutionId)).toEqual(first); expect(() => parseCandidateReceipt( { ...first, - artifacts: [ - { - ...first.artifacts[0], - url: "https://example.com/openshell-x86_64-unknown-linux-musl.tar.gz", - }, - ], + artifacts: first.artifacts.map((item, index) => + index === 1 ? { ...item, url: "https://127.0.0.1/internal" } : item, + ), }, first.resolutionId, ), ).toThrow("approved official HTTPS host"); - expect(() => - parseCandidateReceipt({ ...first, resolutionId: "f".repeat(64) }, first.resolutionId), - ).toThrow("trusted resolution id"); }); - it("fails closed on missing upstream provenance and metadata errors (#6691)", async () => { + it("fails closed when any required asset or SHA-256 provenance is absent (#6691)", async () => { + const missingGateway = releaseMetadata(); + missingGateway.assets = missingGateway.assets.filter(({ name }) => !name.includes("gateway")); await expect( resolveCandidate({ - candidate: "0.0.82", + candidate: VERSION, component: "openshell", - fetcher: async () => response({ assets: [], draft: false, id: 82, tag_name: "v0.0.82" }), + fetcher: async () => response(missingGateway), nemoclawSha: SHA, }), - ).rejects.toThrow("missing openshell-x86_64"); - await expect( - resolveCandidate({ - candidate: "2026.7.1", - component: "openclaw", - fetcher: async () => response({ message: "unavailable" }, 503), - nemoclawSha: SHA, - }), - ).rejects.toThrow("official metadata request failed (503)"); - }); - - it("rejects unofficial npm tarballs even with valid integrity (#6691)", async () => { - const integrity = createHash("sha512").update("archive").digest("base64"); + ).rejects.toThrow("missing openshell-gateway"); + const missingDigest = releaseMetadata(); + missingDigest.assets[2]!.digest = ""; await expect( resolveCandidate({ - candidate: "2026.7.1", - component: "openclaw", - fetcher: async () => - response({ - dist: { - integrity: `sha512-${integrity}`, - tarball: "https://example.com/openclaw-2026.7.1.tgz", - }, - version: "2026.7.1", - }), + candidate: VERSION, + component: "openshell", + fetcher: async () => response(missingDigest), nemoclawSha: SHA, }), - ).rejects.toThrow("approved official HTTPS host"); - }); - - it("peels Hermes tags and resolves exact Hermes and DCode wheels (#6691)", async () => { - const integrity = createHash("sha512").update("hermes-npm").digest("base64"); - const commit = "c".repeat(40); - const responses = new Map([ - [ - "https://api.github.com/repos/NousResearch/hermes-agent/releases/tags/v2026.7.1", - response({ draft: false, id: 701, tag_name: "v2026.7.1" }), - ], - [ - "https://api.github.com/repos/NousResearch/hermes-agent/git/ref/tags/v2026.7.1", - response({ - object: { - sha: "d".repeat(40), - type: "tag", - url: "https://api.github.com/repos/NousResearch/hermes-agent/git/tags/annotated", - }, - }), - ], - [ - "https://api.github.com/repos/NousResearch/hermes-agent/git/tags/annotated", - response({ object: { sha: commit, type: "commit" } }), - ], - [ - `https://raw.githubusercontent.com/NousResearch/hermes-agent/${commit}/pyproject.toml`, - new Response('[project]\nversion = "0.18.0"\n'), - ], - [ - "https://registry.npmjs.org/hermes-agent/0.18.0", - response({ - dist: { - integrity: `sha512-${integrity}`, - tarball: "https://registry.npmjs.org/hermes-agent/-/hermes-agent-0.18.0.tgz", - }, - version: "0.18.0", - }), - ], - [ - "https://pypi.org/pypi/hermes-agent/0.18.0/json", - response({ - info: { name: "hermes-agent", version: "0.18.0" }, - urls: [ - { - digests: { sha256: "e".repeat(64) }, - filename: "hermes_agent-0.18.0-py3-none-any.whl", - packagetype: "bdist_wheel", - url: "https://files.pythonhosted.org/packages/hermes_agent-0.18.0.whl", - yanked: false, - }, - ], - }), - ], - [ - "https://pypi.org/pypi/deepagents-code/0.1.34/json", - response({ - info: { name: "deepagents-code", version: "0.1.34" }, - urls: [ - { - digests: { sha256: "f".repeat(64) }, - filename: "deepagents_code-0.1.34-py3-none-any.whl", - packagetype: "bdist_wheel", - url: "https://files.pythonhosted.org/packages/deepagents_code-0.1.34.whl", - yanked: false, - }, - ], - }), - ], - ]); - const fetcher = async (url: string) => - responses.get(url) ?? response({ message: "unexpected URL" }, 404); - const hermes = await resolveCandidate({ - candidate: "v2026.7.1", - component: "hermes", - fetcher, - nemoclawSha: SHA, - }); - expect(hermes).toMatchObject({ resolvedCandidate: "0.18.0", resolvedCommit: commit }); - expect(hermes.artifacts.map((item) => item.kind)).toEqual(["npm", "wheel"]); - const dcode = await resolveCandidate({ - candidate: "0.1.34", - component: "dcode", - fetcher, - nemoclawSha: SHA, - }); - expect(dcode).toMatchObject({ - officialSource: "pypi:deepagents-code==0.1.34", - resolvedCandidate: "0.1.34", - }); + ).rejects.toThrow("non-empty string"); }); - it("detects digest and observed runtime version mismatches (#6691)", () => { - const bytes = new TextEncoder().encode("candidate"); - expect(() => - verifyDigest(bytes, { - digest: "0".repeat(64), - digestAlgorithm: "sha256", - kind: "archive", - name: "candidate.tgz", - url: "https://registry.npmjs.org/openclaw/-/candidate.tgz", - }), - ).toThrow("sha256 mismatch"); - expect(() => - verifyObservedVersion( - { - artifacts: [], - component: "openclaw", - nemoclawSha: SHA, - officialSource: "npm:openclaw@2026.7.1", - requestedCandidate: "2026.7.1", - resolutionId: "c".repeat(64), - resolvedCandidate: "2026.7.1", - schemaVersion: 1, - }, - "openclaw 2026.7.2", - ), - ).toThrow("does not match 2026.7.1"); - }); - - it("validates every redirect before writing a digest-bound artifact (#6691)", async () => { + it("validates the initial URL and every redirect before writing an artifact (#6691)", async () => { const directory = mkdtempSync(join(tmpdir(), "nemoclaw-candidate-download-")); const body = "candidate archive"; const candidateArtifact = { + ...receipt().artifacts[0]!, digest: createHash("sha256").update(body).digest("hex"), - digestAlgorithm: "sha256" as const, - kind: "npm" as const, - name: "openclaw-2026.7.1.tgz", - url: "https://registry.npmjs.org/openclaw/-/openclaw-2026.7.1.tgz", }; try { + await expect( + downloadCandidateArtifact( + { ...candidateArtifact, url: "http://127.0.0.1/internal" }, + directory, + 0, + async () => { + throw new Error("unapproved initial URL must not be fetched"); + }, + ), + ).rejects.toThrow("not approved"); const requests: string[] = []; await expect( downloadCandidateArtifact(candidateArtifact, directory, 0, async (url, init) => { @@ -316,96 +223,134 @@ describe("candidate compatibility contract", () => { }), ).rejects.toThrow("not approved"); expect(requests).toEqual([candidateArtifact.url]); - const target = await downloadCandidateArtifact( candidateArtifact, directory, 0, async () => new Response(body), ); - expect(basename(target)).toBe("candidate-0.tgz"); + expect(basename(target)).toBe("candidate-0.tar.gz"); expect(readFileSync(target, "utf8")).toBe(body); } finally { rmSync(directory, { force: true, recursive: true }); } }); - it("records every deterministic lane and validates live selectors against E2E (#6691)", () => { + it("materializes exact runtime binaries and records real wrapper invocations (#6691)", async () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-candidate-materialize-")); + const archives = new Map(); + try { + const assets = receipt().artifacts.map((artifact) => { + const binaryName = artifact.role === "cli" ? "openshell" : `openshell-${artifact.role}`; + const source = join(directory, binaryName); + writeFileSync(source, `#!/bin/sh\nprintf '%s\\n' '${binaryName} ${VERSION}'\n`); + chmodSync(source, 0o700); + const archive = join(directory, `${artifact.role}-exact.tar.gz`); + expect(spawnSync("tar", ["-czf", archive, "-C", directory, binaryName]).status).toBe(0); + const bytes = readFileSync(archive); + archives.set(artifact.url, bytes); + return { + browser_download_url: artifact.url, + digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + name: artifact.name, + }; + }); + const candidateReceipt = await resolveCandidate({ + candidate: `v${VERSION}`, + component: "openshell", + fetcher: async () => response({ assets, draft: false, id: 82, tag_name: `v${VERSION}` }), + nemoclawSha: SHA, + }); + const observed = await materializeCandidate( + candidateReceipt, + join(directory, "runtime"), + async (url) => new Response(archives.get(url)), + ); + const log = join(directory, "invocations.log"); + const env = { ...process.env, NEMOCLAW_CANDIDATE_INVOCATION_LOG: log }; + for (const binary of ["openshell", "openshell-gateway", "openshell-sandbox"]) { + expect(spawnSync(join(observed.binDirectory, binary), ["--version"], { env }).status).toBe( + 0, + ); + } + expect( + verifyCandidateInvocations({ + invocationLog: readFileSync(log, "utf8"), + lane: "installer", + receipt: candidateReceipt, + }), + ).toMatchObject({ conclusion: "success", observedVersion: VERSION }); + expect(() => + verifyCandidateInvocations({ + invocationLog: "gateway\t--version\n", + lane: "live:openshell-gateway-auth-contract", + receipt: candidateReceipt, + }), + ).toThrow("did not start the candidate runtime"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("selects only candidate-aware deterministic and live lanes (#6691)", () => { const plan = buildCandidatePlan("openshell", E2E_SOURCES); - expect(plan.deterministic).toHaveLength(6); expect( - plan.deterministic.filter((lane) => lane.status === "selected").map((lane) => lane.id), - ).toEqual(["source-unit", "integration", "installer", "e2e-support"]); - expect(plan.deterministic.filter((lane) => lane.status === "skipped")).toEqual([ - expect.objectContaining({ id: "package-contract", reason: expect.any(String) }), - expect.objectContaining({ id: "plugin", reason: expect.any(String) }), - ]); + plan.deterministic.filter(({ status }) => status === "selected").map(({ id }) => id), + ).toEqual(["installer"]); expect(plan.live).toEqual([ - expect.objectContaining({ id: "full-e2e", selector: "job", status: "skipped" }), expect.objectContaining({ id: "openshell-gateway-auth-contract", selector: "job", - status: "skipped", - }), - ]); - expect(() => buildCandidatePlan("openshell", E2E_SOURCES.replace(" full-e2e:\n", ""))).toThrow( - "does not declare job full-e2e", - ); - expect(buildCandidatePlan("dcode", E2E_SOURCES).live).toEqual([ - expect.objectContaining({ - id: "ubuntu-repo-cloud-langchain-deepagents-code", - selector: "target", - status: "skipped", + status: "selected", }), ]); }); - it("finalizes only complete results from the same candidate resolution (#6691)", () => { - const plan = buildCandidatePlan("hermes", E2E_SOURCES); - const receipt = { - artifacts: [], - component: "hermes" as const, - nemoclawSha: SHA, - officialSource: "github:NousResearch/hermes-agent:release:1", - requestedCandidate: "v2026.7.1", - resolutionId: "d".repeat(64), - resolvedCandidate: "0.18.0", - resolvedCommit: "e".repeat(40), - schemaVersion: 1 as const, - }; - const results = ["source-unit", "integration", "e2e-support"].map((lane) => ({ + it("finalizes only complete receipt-bound deterministic and live evidence (#6691)", () => { + const plan = buildCandidatePlan("openshell", E2E_SOURCES); + const candidateReceipt = receipt(); + const results = ["installer", "live:openshell-gateway-auth-contract"].map((lane) => ({ conclusion: "success" as const, - lane: lane as "source-unit" | "integration" | "e2e-support", - observedOutput: "hermes 0.18.0", - observedVersion: "0.18.0", - resolutionId: receipt.resolutionId, + lane, + observedOutput: `openshell ${VERSION}`, + observedVersion: VERSION, + resolutionId: candidateReceipt.resolutionId, })); - const evidence = finalizeEvidence({ attempt: "2", plan, receipt, results, runId: "123" }); - expect(evidence).toMatchObject({ - execution: { attempt: "2", runId: "123" }, - overall: "success", - schemaVersion: 1, - }); - expect(() => - finalizeEvidence({ attempt: "2", plan, receipt, results: results.slice(1), runId: "123" }), - ).toThrow("account for every selected deterministic lane exactly once"); + expect( + finalizeEvidence({ + attempt: "2", + plan, + receipt: candidateReceipt, + results, + runId: "123", + }), + ).toMatchObject({ overall: "success", schemaVersion: 1 }); expect(() => finalizeEvidence({ attempt: "2", plan, - receipt, - results: [{ ...results[0]!, resolutionId: "f".repeat(64) }, ...results.slice(1)], + receipt: candidateReceipt, + results: results.slice(1), runId: "123", }), - ).toThrow("used a different candidate resolution"); + ).toThrow("every selected compatibility lane"); expect(() => finalizeEvidence({ attempt: "2", plan, - receipt, - results: [{ ...results[0]!, observedVersion: "0.17.0" }, ...results.slice(1)], + receipt: candidateReceipt, + results: [{ ...results[0]!, resolutionId: "f".repeat(64) }, results[1]!], runId: "123", }), - ).toThrow("has no matching observed candidate version"); + ).toThrow("different candidate resolution"); + }); + + it("detects digest and runtime version mismatches (#6691)", () => { + expect(() => + verifyDigest(new TextEncoder().encode("candidate"), receipt().artifacts[0]!), + ).toThrow("sha256 mismatch"); + expect(() => verifyObservedVersion(receipt(), "openshell 0.0.81")).toThrow( + `does not match ${VERSION}`, + ); }); }); diff --git a/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts b/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts index f9cb0df8db..221c3500fb 100644 --- a/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts +++ b/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts @@ -147,12 +147,13 @@ export async function withOpenShellGatewayAuthArtifactSafety( function resolveGatewayBin(): string | null { for (const candidate of [ + process.env.OPENSHELL_GATEWAY_BIN, path.join(os.homedir(), ".local", "bin", "openshell-gateway"), "/opt/homebrew/bin/openshell-gateway", "/usr/local/bin/openshell-gateway", "/usr/bin/openshell-gateway", ]) { - if (fs.existsSync(candidate)) return candidate; + if (candidate && fs.existsSync(candidate)) return candidate; } const which = run("sh", ["-c", "command -v openshell-gateway"]); return which.status === 0 && which.stdout.trim() ? which.stdout.trim() : null; @@ -646,7 +647,7 @@ async function runOpenShellGatewayAuthSourceContractScenarioUnchecked( const version = run(gatewayBin, ["--version"]); expect(version.status, commandOutput(version)).toBe(0); - expect(commandOutput(version)).toContain("0.0.72"); + expect(commandOutput(version)).toContain(process.env.NEMOCLAW_CANDIDATE_VERSION ?? "0.0.72"); await requireDockerDaemon({ dockerBin, host, skip }); diff --git a/test/e2e/live/openshell-gateway-auth-source-contract.test.ts b/test/e2e/live/openshell-gateway-auth-source-contract.test.ts index 8e72703339..7e10be3d02 100644 --- a/test/e2e/live/openshell-gateway-auth-source-contract.test.ts +++ b/test/e2e/live/openshell-gateway-auth-source-contract.test.ts @@ -7,7 +7,7 @@ import { test } from "../fixtures/e2e-test.ts"; import { runOpenShellGatewayAuthSourceContractScenario } from "./openshell-gateway-auth-source-contract-helpers.ts"; const LIVE_TIMEOUT_MS = 8 * 60_000; -const OPENSHELL_GATEWAY_AUTH_CONTRACT_VERSION = "0.0.72"; +const OPENSHELL_GATEWAY_AUTH_CONTRACT_VERSION = process.env.NEMOCLAW_CANDIDATE_VERSION ?? "0.0.72"; test( `OpenShell ${OPENSHELL_GATEWAY_AUTH_CONTRACT_VERSION} Docker-driver gateway auth uses NemoClaw mTLS plus sandbox JWT`, diff --git a/tools/candidate-compat.mts b/tools/candidate-compat.mts index 7398b1b332..4dd86b8556 100644 --- a/tools/candidate-compat.mts +++ b/tools/candidate-compat.mts @@ -7,14 +7,15 @@ import { chmod, mkdir, readdir, readFile, writeFile } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -export const COMPONENTS = ["openshell", "openclaw", "hermes", "dcode"] as const; +export const COMPONENTS = ["openshell"] as const; export type CandidateComponent = (typeof COMPONENTS)[number]; export type Artifact = { digest: string; - digestAlgorithm: "sha256" | "sha512"; - kind: "archive" | "npm" | "wheel"; + digestAlgorithm: "sha256"; + kind: "archive"; name: string; + role: "cli" | "gateway" | "sandbox"; url: string; }; @@ -26,7 +27,6 @@ export type CandidateReceipt = { requestedCandidate: string; resolutionId: string; resolvedCandidate: string; - resolvedCommit?: string; schemaVersion: 1; }; @@ -41,14 +41,14 @@ export type CandidatePlan = { id: string; reason: string; selector: "job" | "target"; - status: "skipped"; + status: "selected" | "skipped"; }>; schemaVersion: 1; }; type LaneResult = { conclusion: "failure" | "success"; - lane: DeterministicLane; + lane: string; observedOutput?: string; observedVersion?: string; resolutionId: string; @@ -65,16 +65,16 @@ type DeterministicLane = const FULL_SHA = /^[a-f0-9]{40}$/; const VERSION = /^[0-9]+(?:\.[0-9]+){2}(?:[-+][0-9A-Za-z.-]+)?$/; -const HERMES_TAG = /^v[0-9]+(?:\.[0-9]+){2}$/; const DIGEST = /^[a-f0-9]+$/; const OFFICIAL_HOSTS = new Set([ "api.github.com", "codeload.github.com", - "files.pythonhosted.org", "github.com", - "registry.npmjs.org", ]); -const DOWNLOAD_HOSTS = new Set([...OFFICIAL_HOSTS, "release-assets.githubusercontent.com"]); +const DOWNLOAD_HOSTS = new Set([ + ...OFFICIAL_HOSTS, + "release-assets.githubusercontent.com", +]); const MAX_DOWNLOAD_REDIRECTS = 5; const ALL_LANES: DeterministicLane[] = [ "source-unit", @@ -85,27 +85,18 @@ const ALL_LANES: DeterministicLane[] = [ "e2e-support", ]; -const SELECTED_LANES: Record> = { - openshell: new Set(["source-unit", "integration", "installer", "e2e-support"]), - openclaw: new Set(["source-unit", "integration", "package-contract", "plugin", "e2e-support"]), - hermes: new Set(["source-unit", "integration", "e2e-support"]), - dcode: new Set(["source-unit", "integration", "e2e-support"]), +const SELECTED_LANES: Record< + CandidateComponent, + ReadonlySet +> = { + openshell: new Set(["installer"]), }; const LIVE_SELECTORS: Record< CandidateComponent, ReadonlyArray<{ id: string; selector: "job" | "target" }> > = { - openshell: [ - { id: "full-e2e", selector: "job" }, - { id: "openshell-gateway-auth-contract", selector: "job" }, - ], - openclaw: [ - { id: "full-e2e", selector: "job" }, - { id: "openclaw-skill-cli", selector: "job" }, - ], - hermes: [{ id: "hermes-e2e", selector: "job" }], - dcode: [{ id: "ubuntu-repo-cloud-langchain-deepagents-code", selector: "target" }], + openshell: [{ id: "openshell-gateway-auth-contract", selector: "job" }], }; function stableJson(value: unknown): string { @@ -113,7 +104,7 @@ function stableJson(value: unknown): string { if (value && typeof value === "object") { return `{${Object.entries(value as Record) .filter(([, item]) => item !== undefined) - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) .map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`) .join(",")}}`; } @@ -130,15 +121,18 @@ function assertComponent(value: string): asserts value is CandidateComponent { } } -function assertCandidate(component: CandidateComponent, candidate: string): void { - if (candidate.length > 128 || candidate.includes("/") || /[\x00-\x20\x7f]/u.test(candidate)) { +function assertCandidate( + component: CandidateComponent, + candidate: string, +): void { + if ( + candidate.length > 128 || + candidate.includes("/") || + /[\x00-\x20\x7f]/u.test(candidate) + ) { throw new Error("candidate contains unsafe characters"); } - if (component === "hermes") { - if (!HERMES_TAG.test(candidate)) throw new Error("Hermes candidate must match vX.Y.Z"); - return; - } - const normalized = component === "openshell" ? candidate.replace(/^v/u, "") : candidate; + const normalized = candidate.replace(/^v/u, ""); if (!VERSION.test(normalized)) { throw new Error(`${component} candidate must be an exact version`); } @@ -147,10 +141,14 @@ function assertCandidate(component: CandidateComponent, candidate: string): void function assertOfficialUrl(raw: string, expectedPathPrefix: string): URL { const url = new URL(raw); if (url.protocol !== "https:" || !OFFICIAL_HOSTS.has(url.hostname)) { - throw new Error(`candidate artifact is not on an approved official HTTPS host: ${raw}`); + throw new Error( + `candidate artifact is not on an approved official HTTPS host: ${raw}`, + ); } if (!url.pathname.startsWith(expectedPathPrefix)) { - throw new Error(`candidate artifact has unexpected official-source path: ${raw}`); + throw new Error( + `candidate artifact has unexpected official-source path: ${raw}`, + ); } if (url.username || url.password || url.search || url.hash) { throw new Error( @@ -160,7 +158,11 @@ function assertOfficialUrl(raw: string, expectedPathPrefix: string): URL { return url; } -async function fetchJson(fetcher: FetchLike, url: string, token?: string): Promise { +async function fetchJson( + fetcher: FetchLike, + url: string, + token?: string, +): Promise { const response = await fetcher(url, { headers: { Accept: "application/vnd.github+json, application/json", @@ -169,7 +171,9 @@ async function fetchJson(fetcher: FetchLike, url: string, token?: string): Promi redirect: "error", }); if (!response.ok) - throw new Error(`official metadata request failed (${response.status}): ${url}`); + throw new Error( + `official metadata request failed (${response.status}): ${url}`, + ); return response.json(); } @@ -180,7 +184,11 @@ function record(value: unknown, label: string): Record { return value as Record; } -function stringField(value: Record, key: string, label: string): string { +function stringField( + value: Record, + key: string, + label: string, +): string { const field = value[key]; if (typeof field !== "string" || field.length === 0) { throw new Error(`${label}.${key} must be a non-empty string`); @@ -190,10 +198,13 @@ function stringField(value: Record, key: string, label: string) function artifact(input: Artifact, expectedPathPrefix: string): Artifact { const url = assertOfficialUrl(input.url, expectedPathPrefix); - if (!DIGEST.test(input.digest)) throw new Error(`${input.name} has an invalid digest`); - const expectedLength = input.digestAlgorithm === "sha256" ? 64 : 128; + if (!DIGEST.test(input.digest)) + throw new Error(`${input.name} has an invalid digest`); + const expectedLength = 64; if (input.digest.length !== expectedLength) { - throw new Error(`${input.name} has an invalid ${input.digestAlgorithm} digest`); + throw new Error( + `${input.name} has an invalid ${input.digestAlgorithm} digest`, + ); } return { ...input, url: url.href }; } @@ -202,7 +213,9 @@ async function resolveOpenShell( candidate: string, fetcher: FetchLike, token?: string, -): Promise> { +): Promise< + Omit +> { const version = candidate.replace(/^v/u, ""); const tag = `v${version}`; const metadata = record( @@ -213,200 +226,65 @@ async function resolveOpenShell( ), "OpenShell release", ); - if (metadata.draft === true) throw new Error("OpenShell candidate release is a draft"); + if (metadata.draft === true) + throw new Error("OpenShell candidate release is a draft"); if (stringField(metadata, "tag_name", "OpenShell release") !== tag) { - throw new Error("OpenShell release tag does not match the requested candidate"); - } - if (!Array.isArray(metadata.assets)) throw new Error("OpenShell release assets must be an array"); - const name = "openshell-x86_64-unknown-linux-musl.tar.gz"; - const assetMetadata = metadata.assets - .map((item) => record(item, "OpenShell asset")) - .find((item) => item.name === name); - if (!assetMetadata) throw new Error(`OpenShell release is missing ${name}`); - const digest = stringField(assetMetadata, "digest", "OpenShell asset"); - if (!digest.startsWith("sha256:")) - throw new Error("OpenShell asset is missing SHA-256 provenance"); - return { - artifacts: [ - artifact( - { - digest: digest.slice("sha256:".length), - digestAlgorithm: "sha256", - kind: "archive", - name, - url: stringField(assetMetadata, "browser_download_url", "OpenShell asset"), - }, - `/NVIDIA/OpenShell/releases/download/${tag}/`, - ), - ], - component: "openshell", - officialSource: `github:NVIDIA/OpenShell:release:${String(metadata.id)}`, - requestedCandidate: candidate, - resolvedCandidate: version, - }; -} - -function npmArtifact(metadata: Record, packageName: string): Artifact { - const dist = record(metadata.dist, `${packageName} dist`); - const integrity = stringField(dist, "integrity", `${packageName} dist`); - if (!integrity.startsWith("sha512-")) - throw new Error(`${packageName} requires sha512 npm integrity`); - let digest: string; - try { - digest = Buffer.from(integrity.slice("sha512-".length), "base64").toString("hex"); - } catch { - throw new Error(`${packageName} has invalid npm integrity`); + throw new Error( + "OpenShell release tag does not match the requested candidate", + ); } - return artifact( + if (!Array.isArray(metadata.assets)) + throw new Error("OpenShell release assets must be an array"); + const requiredAssets = [ { - digest, - digestAlgorithm: "sha512", - kind: "npm", - name: `${packageName}-${stringField(metadata, "version", packageName)}.tgz`, - url: stringField(dist, "tarball", `${packageName} dist`), + name: "openshell-x86_64-unknown-linux-musl.tar.gz", + role: "cli" as const, }, - `/${packageName}/-/`, - ); -} - -async function resolveNpm( - component: "openclaw", - candidate: string, - fetcher: FetchLike, -): Promise> { - const packageName = component; - const metadata = record( - await fetchJson(fetcher, `https://registry.npmjs.org/${packageName}/${candidate}`), - `${packageName} metadata`, - ); - const resolvedCandidate = stringField(metadata, "version", `${packageName} metadata`); - if (resolvedCandidate !== candidate) - throw new Error(`${packageName} metadata resolved a different version`); - return { - artifacts: [npmArtifact(metadata, packageName)], - component, - officialSource: `npm:${packageName}@${resolvedCandidate}`, - requestedCandidate: candidate, - resolvedCandidate, - }; -} - -async function peelGitTag(fetcher: FetchLike, candidate: string, token?: string): Promise { - let object = record( - record( - await fetchJson( - fetcher, - `https://api.github.com/repos/NousResearch/hermes-agent/git/ref/tags/${candidate}`, - token, - ), - "Hermes tag ref", - ).object, - "Hermes tag object", - ); - for (let depth = 0; depth < 2 && object.type === "tag"; depth += 1) { - const tag = record( - await fetchJson(fetcher, stringField(object, "url", "Hermes tag object"), token), - "Hermes annotated tag", - ); - object = record(tag.object, "Hermes annotated tag object"); - } - if (object.type !== "commit") throw new Error("Hermes tag does not resolve to a commit"); - const commit = stringField(object, "sha", "Hermes tag object"); - if (!FULL_SHA.test(commit)) throw new Error("Hermes tag resolved an invalid commit SHA"); - return commit; -} - -async function resolveHermes( - candidate: string, - fetcher: FetchLike, - token?: string, -): Promise> { - const release = record( - await fetchJson( - fetcher, - `https://api.github.com/repos/NousResearch/hermes-agent/releases/tags/${candidate}`, - token, - ), - "Hermes release", - ); - if (release.draft === true) throw new Error("Hermes candidate release is a draft"); - if (stringField(release, "tag_name", "Hermes release") !== candidate) { - throw new Error("Hermes release tag does not match the requested candidate"); - } - const commit = await peelGitTag(fetcher, candidate, token); - const pyprojectResponse = await fetcher( - `https://raw.githubusercontent.com/NousResearch/hermes-agent/${commit}/pyproject.toml`, - { redirect: "error" }, - ); - if (!pyprojectResponse.ok) - throw new Error("failed to read Hermes package version at resolved commit"); - const pyproject = await pyprojectResponse.text(); - const version = pyproject.match(/^version\s*=\s*"([^"]+)"\s*$/mu)?.[1]; - if (!version || !VERSION.test(version)) - throw new Error("Hermes pyproject has no exact package version"); - const npmMetadata = record( - await fetchJson(fetcher, `https://registry.npmjs.org/hermes-agent/${version}`), - "Hermes npm metadata", - ); - if (stringField(npmMetadata, "version", "Hermes npm metadata") !== version) { - throw new Error("Hermes npm package version does not match the resolved source"); - } - const wheel = await resolvePypiWheel("hermes-agent", version, fetcher); - return { - artifacts: [npmArtifact(npmMetadata, "hermes-agent"), wheel], - component: "hermes", - officialSource: `github:NousResearch/hermes-agent:release:${String(release.id)}`, - requestedCandidate: candidate, - resolvedCandidate: version, - resolvedCommit: commit, - }; -} - -async function resolvePypiWheel( - packageName: string, - version: string, - fetcher: FetchLike, -): Promise { - const metadata = record( - await fetchJson(fetcher, `https://pypi.org/pypi/${packageName}/${version}/json`), - `${packageName} PyPI metadata`, - ); - const info = record(metadata.info, `${packageName} PyPI info`); - if (stringField(info, "name", `${packageName} PyPI info`).toLowerCase() !== packageName) { - throw new Error(`PyPI returned a different package for ${packageName}`); - } - if (stringField(info, "version", `${packageName} PyPI info`) !== version) { - throw new Error(`PyPI resolved a different ${packageName} version`); - } - if (!Array.isArray(metadata.urls)) throw new Error(`${packageName} PyPI files must be an array`); - const wheel = metadata.urls - .map((item) => record(item, `${packageName} PyPI file`)) - .find((item) => item.packagetype === "bdist_wheel" && item.yanked !== true); - if (!wheel) throw new Error(`${packageName} release has no non-yanked wheel`); - const digests = record(wheel.digests, `${packageName} PyPI file digests`); - return artifact( { - digest: stringField(digests, "sha256", `${packageName} PyPI file digests`), - digestAlgorithm: "sha256", - kind: "wheel", - name: stringField(wheel, "filename", `${packageName} PyPI file`), - url: stringField(wheel, "url", `${packageName} PyPI file`), + name: "openshell-gateway-x86_64-unknown-linux-gnu.tar.gz", + role: "gateway" as const, }, - "/packages/", + { + name: "openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz", + role: "sandbox" as const, + }, + ]; + const releaseAssets = metadata.assets.map((item) => + record(item, "OpenShell asset"), ); -} - -async function resolveDcode( - candidate: string, - fetcher: FetchLike, -): Promise> { - const wheel = await resolvePypiWheel("deepagents-code", candidate, fetcher); + const artifacts = requiredAssets.map(({ name, role }) => { + const assetMetadata = releaseAssets.find((item) => item.name === name); + if (!assetMetadata) throw new Error(`OpenShell release is missing ${name}`); + const digest = stringField( + assetMetadata, + "digest", + `OpenShell asset ${name}`, + ); + if (!digest.startsWith("sha256:")) { + throw new Error(`OpenShell asset ${name} is missing SHA-256 provenance`); + } + return artifact( + { + digest: digest.slice("sha256:".length), + digestAlgorithm: "sha256", + kind: "archive", + name, + role, + url: stringField( + assetMetadata, + "browser_download_url", + `OpenShell asset ${name}`, + ), + }, + `/NVIDIA/OpenShell/releases/download/${tag}/`, + ); + }); return { - artifacts: [wheel], - component: "dcode", - officialSource: `pypi:deepagents-code==${candidate}`, + artifacts, + component: "openshell", + officialSource: `github:NVIDIA/OpenShell:release:${String(metadata.id)}`, requestedCandidate: candidate, - resolvedCandidate: candidate, + resolvedCandidate: version, }; } @@ -418,17 +296,15 @@ export async function resolveCandidate(input: { nemoclawSha: string; }): Promise { assertComponent(input.component); - if (!FULL_SHA.test(input.nemoclawSha)) throw new Error("NemoClaw ref must resolve to a full SHA"); + if (!FULL_SHA.test(input.nemoclawSha)) + throw new Error("NemoClaw ref must resolve to a full SHA"); assertCandidate(input.component, input.candidate); const fetcher = input.fetcher ?? fetch; - const resolved = - input.component === "openshell" - ? await resolveOpenShell(input.candidate, fetcher, input.githubToken) - : input.component === "openclaw" - ? await resolveNpm(input.component, input.candidate, fetcher) - : input.component === "hermes" - ? await resolveHermes(input.candidate, fetcher, input.githubToken) - : await resolveDcode(input.candidate, fetcher); + const resolved = await resolveOpenShell( + input.candidate, + fetcher, + input.githubToken, + ); const receiptWithoutId = { ...resolved, nemoclawSha: input.nemoclawSha, @@ -442,17 +318,22 @@ export async function resolveCandidate(input: { function parseArtifact( value: unknown, - component: CandidateComponent, resolvedCandidate: string, index: number, ): Artifact { const input = record(value, `candidate artifact ${index}`); - const digestAlgorithm = stringField(input, "digestAlgorithm", `candidate artifact ${index}`); - if (digestAlgorithm !== "sha256" && digestAlgorithm !== "sha512") { - throw new Error(`candidate artifact ${index} has an invalid digest algorithm`); + const digestAlgorithm = stringField( + input, + "digestAlgorithm", + `candidate artifact ${index}`, + ); + if (digestAlgorithm !== "sha256") { + throw new Error( + `candidate artifact ${index} has an invalid digest algorithm`, + ); } const kind = stringField(input, "kind", `candidate artifact ${index}`); - if (kind !== "archive" && kind !== "npm" && kind !== "wheel") { + if (kind !== "archive") { throw new Error(`candidate artifact ${index} has an invalid kind`); } const name = stringField(input, "name", `candidate artifact ${index}`); @@ -460,31 +341,24 @@ function parseArtifact( throw new Error(`candidate artifact ${index} has an unsafe name`); } - const expected = - component === "openshell" - ? { - kind: "archive", - name: "openshell-x86_64-unknown-linux-musl.tar.gz", - path: `/NVIDIA/OpenShell/releases/download/v${resolvedCandidate}/`, - } - : component === "openclaw" - ? { - kind: "npm", - name: `openclaw-${resolvedCandidate}.tgz`, - path: "/openclaw/-/", - } - : component === "hermes" && index === 0 - ? { - kind: "npm", - name: `hermes-agent-${resolvedCandidate}.tgz`, - path: "/hermes-agent/-/", - } - : { kind: "wheel", name: null, path: "/packages/" }; - if (kind !== expected.kind || (expected.name !== null && name !== expected.name)) { - throw new Error(`candidate artifact ${index} does not match the resolved component`); - } - if (kind === "wheel" && !/^[0-9A-Za-z_.+-]+\.whl$/u.test(name)) { - throw new Error(`candidate artifact ${index} has an invalid wheel name`); + const expected = [ + { + name: "openshell-x86_64-unknown-linux-musl.tar.gz", + role: "cli" as const, + }, + { + name: "openshell-gateway-x86_64-unknown-linux-gnu.tar.gz", + role: "gateway" as const, + }, + { + name: "openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz", + role: "sandbox" as const, + }, + ][index]; + if (!expected || name !== expected.name || input.role !== expected.role) { + throw new Error( + `candidate artifact ${index} does not match the resolved component`, + ); } return artifact( { @@ -492,9 +366,10 @@ function parseArtifact( digestAlgorithm, kind, name, + role: expected.role, url: stringField(input, "url", `candidate artifact ${index}`), }, - expected.path, + `/NVIDIA/OpenShell/releases/download/v${resolvedCandidate}/`, ); } @@ -507,60 +382,53 @@ export function parseCandidateReceipt( throw new Error("expected candidate resolution id is invalid"); } const input = record(value, "candidate receipt"); - if (input.schemaVersion !== 1) throw new Error("candidate receipt schema version must be 1"); + if (input.schemaVersion !== 1) + throw new Error("candidate receipt schema version must be 1"); const component = stringField(input, "component", "candidate receipt"); assertComponent(component); - const requestedCandidate = stringField(input, "requestedCandidate", "candidate receipt"); + const requestedCandidate = stringField( + input, + "requestedCandidate", + "candidate receipt", + ); assertCandidate(component, requestedCandidate); - const resolvedCandidate = stringField(input, "resolvedCandidate", "candidate receipt"); + const resolvedCandidate = stringField( + input, + "resolvedCandidate", + "candidate receipt", + ); if (!VERSION.test(resolvedCandidate)) { throw new Error("candidate receipt has an invalid resolved version"); } - if ( - (component === "openshell" && requestedCandidate.replace(/^v/u, "") !== resolvedCandidate) || - ((component === "openclaw" || component === "dcode") && - requestedCandidate !== resolvedCandidate) - ) { - throw new Error("candidate receipt resolved version does not match the request"); + if (requestedCandidate.replace(/^v/u, "") !== resolvedCandidate) { + throw new Error( + "candidate receipt resolved version does not match the request", + ); } const nemoclawSha = stringField(input, "nemoclawSha", "candidate receipt"); - if (!FULL_SHA.test(nemoclawSha)) throw new Error("candidate receipt has an invalid NemoClaw SHA"); + if (!FULL_SHA.test(nemoclawSha)) + throw new Error("candidate receipt has an invalid NemoClaw SHA"); if (!Array.isArray(input.artifacts)) throw new Error("candidate receipt artifacts must be an array"); - const expectedArtifactCount = component === "hermes" ? 2 : 1; + const expectedArtifactCount = 3; if (input.artifacts.length !== expectedArtifactCount) { - throw new Error(`candidate receipt for ${component} has an invalid artifact count`); + throw new Error( + `candidate receipt for ${component} has an invalid artifact count`, + ); } const artifacts = input.artifacts.map((item, index) => - parseArtifact(item, component, resolvedCandidate, index), + parseArtifact(item, resolvedCandidate, index), + ); + const officialSource = stringField( + input, + "officialSource", + "candidate receipt", ); - const officialSource = stringField(input, "officialSource", "candidate receipt"); - const sourcePattern = - component === "openshell" - ? /^github:NVIDIA\/OpenShell:release:[1-9][0-9]*$/u - : component === "openclaw" - ? new RegExp( - `^npm:openclaw@${resolvedCandidate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, - "u", - ) - : component === "hermes" - ? /^github:NousResearch\/hermes-agent:release:[1-9][0-9]*$/u - : new RegExp( - `^pypi:deepagents-code==${resolvedCandidate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, - "u", - ); + const sourcePattern = /^github:NVIDIA\/OpenShell:release:[1-9][0-9]*$/u; if (!sourcePattern.test(officialSource)) { throw new Error("candidate receipt has an invalid official source"); } - const resolvedCommitValue = input.resolvedCommit; - const resolvedCommit = - resolvedCommitValue === undefined - ? undefined - : stringField(input, "resolvedCommit", "candidate receipt"); - if ( - (component === "hermes" && (!resolvedCommit || !FULL_SHA.test(resolvedCommit))) || - (component !== "hermes" && resolvedCommit !== undefined) - ) { + if (input.resolvedCommit !== undefined) { throw new Error("candidate receipt has an invalid resolved commit"); } const receiptWithoutId = { @@ -570,7 +438,6 @@ export function parseCandidateReceipt( officialSource, requestedCandidate, resolvedCandidate, - ...(resolvedCommit ? { resolvedCommit } : {}), schemaVersion: 1 as const, }; const resolutionId = sha256(stableJson(receiptWithoutId)); @@ -578,7 +445,9 @@ export function parseCandidateReceipt( stringField(input, "resolutionId", "candidate receipt") !== resolutionId || resolutionId !== expectedResolutionId ) { - throw new Error("candidate receipt does not match the trusted resolution id"); + throw new Error( + "candidate receipt does not match the trusted resolution id", + ); } return { ...receiptWithoutId, resolutionId }; } @@ -589,14 +458,18 @@ export function buildCandidatePlan( ): CandidatePlan { const selected = SELECTED_LANES[component]; const live = LIVE_SELECTORS[component].map((entry) => { - const marker = entry.selector === "job" ? ` ${entry.id}:\n` : `id: "${entry.id}"`; + const marker = + entry.selector === "job" ? ` ${entry.id}:\n` : `id: "${entry.id}"`; if (!e2eSources.includes(marker)) { - throw new Error(`E2E source of truth does not declare ${entry.selector} ${entry.id}`); + throw new Error( + `E2E source of truth does not declare ${entry.selector} ${entry.id}`, + ); } return { ...entry, - reason: "candidate injection is not yet wired into the credential-bearing E2E boundary", - status: "skipped" as const, + reason: + "the selected live job executes the digest-bound OpenShell gateway candidate", + status: "selected" as const, }; }); return { @@ -614,15 +487,29 @@ export function buildCandidatePlan( } export function verifyDigest(bytes: Uint8Array, artifactValue: Artifact): void { - const actual = createHash(artifactValue.digestAlgorithm).update(bytes).digest("hex"); + const actual = createHash(artifactValue.digestAlgorithm) + .update(bytes) + .digest("hex"); if (actual !== artifactValue.digest) { - throw new Error(`${artifactValue.name} ${artifactValue.digestAlgorithm} mismatch`); + throw new Error( + `${artifactValue.name} ${artifactValue.digestAlgorithm} mismatch`, + ); } } -export function verifyObservedVersion(receipt: CandidateReceipt, output: string): string { - const escaped = receipt.resolvedCandidate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - if (!new RegExp(`(^|[^0-9A-Za-z.])v?${escaped}([^0-9A-Za-z.]|$)`, "u").test(output.trim())) { +export function verifyObservedVersion( + receipt: CandidateReceipt, + output: string, +): string { + const escaped = receipt.resolvedCandidate.replace( + /[.*+?^${}()|[\]\\]/g, + "\\$&", + ); + if ( + !new RegExp(`(^|[^0-9A-Za-z.])v?${escaped}([^0-9A-Za-z.]|$)`, "u").test( + output.trim(), + ) + ) { throw new Error( `observed ${receipt.component} version does not match ${receipt.resolvedCandidate}: ${output.trim()}`, ); @@ -637,31 +524,46 @@ export function finalizeEvidence(input: { results: LaneResult[]; runId: string; }): Record { - if (!/^[1-9][0-9]*$/u.test(input.runId) || !/^[1-9][0-9]*$/u.test(input.attempt)) { + if ( + !/^[1-9][0-9]*$/u.test(input.runId) || + !/^[1-9][0-9]*$/u.test(input.attempt) + ) { throw new Error("run id and attempt must be positive integers"); } - const selected = input.plan.deterministic - .filter((lane) => lane.status === "selected") - .map((lane) => lane.id) - .sort(); + const selected = [ + ...input.plan.deterministic + .filter((lane) => lane.status === "selected") + .map((lane) => lane.id), + ...input.plan.live + .filter((lane) => lane.status === "selected") + .map((lane) => `live:${lane.id}`), + ].sort(); const actual = input.results.map((result) => result.lane).sort(); - if (new Set(actual).size !== actual.length || stableJson(actual) !== stableJson(selected)) { + if ( + new Set(actual).size !== actual.length || + stableJson(actual) !== stableJson(selected) + ) { throw new Error( - "lane results do not account for every selected deterministic lane exactly once", + "lane results do not account for every selected compatibility lane exactly once", ); } for (const result of input.results) { if (result.resolutionId !== input.receipt.resolutionId) { - throw new Error(`lane ${result.lane} used a different candidate resolution`); + throw new Error( + `lane ${result.lane} used a different candidate resolution`, + ); } if (result.conclusion !== "success" && result.conclusion !== "failure") { throw new Error(`lane ${result.lane} has an invalid conclusion`); } if ( result.conclusion === "success" && - (result.observedVersion !== input.receipt.resolvedCandidate || !result.observedOutput) + (result.observedVersion !== input.receipt.resolvedCandidate || + !result.observedOutput) ) { - throw new Error(`lane ${result.lane} has no matching observed candidate version`); + throw new Error( + `lane ${result.lane} has no matching observed candidate version`, + ); } } return { @@ -671,7 +573,9 @@ export function finalizeEvidence(input: { : "failure", plan: input.plan, receipt: input.receipt, - results: [...input.results].sort((left, right) => left.lane.localeCompare(right.lane)), + results: [...input.results].sort((left, right) => + left.lane < right.lane ? -1 : left.lane > right.lane ? 1 : 0, + ), schemaVersion: 1, }; } @@ -688,8 +592,14 @@ function run( stdio: ["ignore", "pipe", "pipe"], }); if (result.status !== 0) { - const detail = result.error?.message ?? result.stderr?.trim() ?? result.stdout?.trim() ?? ""; - throw new Error(`${command} ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`); + const detail = + result.error?.message ?? + result.stderr?.trim() ?? + result.stdout?.trim() ?? + ""; + throw new Error( + `${command} ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`, + ); } return result.stdout.trim(); } @@ -709,16 +619,24 @@ function assertDownloadUrl(raw: string): URL { return url; } -async function fetchDownload(fetcher: FetchLike, rawUrl: string): Promise { +async function fetchDownload( + fetcher: FetchLike, + rawUrl: string, +): Promise { let current = assertDownloadUrl(rawUrl); - for (let redirectCount = 0; redirectCount <= MAX_DOWNLOAD_REDIRECTS; redirectCount += 1) { + for ( + let redirectCount = 0; + redirectCount <= MAX_DOWNLOAD_REDIRECTS; + redirectCount += 1 + ) { const response = await fetcher(current.href, { redirect: "manual" }); if (![301, 302, 303, 307, 308].includes(response.status)) return response; if (redirectCount === MAX_DOWNLOAD_REDIRECTS) { throw new Error("candidate download exceeded the redirect limit"); } const location = response.headers.get("location"); - if (!location) throw new Error("candidate download redirect has no location"); + if (!location) + throw new Error("candidate download redirect has no location"); current = assertDownloadUrl(new URL(location, current).href); } throw new Error("candidate download redirect handling failed"); @@ -731,81 +649,154 @@ export async function downloadCandidateArtifact( fetcher: FetchLike = fetch, ): Promise { const response = await fetchDownload(fetcher, artifactValue.url); - if (!response.ok) throw new Error(`candidate download failed (${response.status})`); + if (!response.ok) + throw new Error(`candidate download failed (${response.status})`); const bytes = new Uint8Array(await response.arrayBuffer()); verifyDigest(bytes, artifactValue); - const extension = - artifactValue.kind === "archive" ? ".tar.gz" : artifactValue.kind === "npm" ? ".tgz" : ".whl"; - const target = join(directory, `candidate-${index}${extension}`); + const target = join(directory, `candidate-${index}.tar.gz`); await writeFile(target, bytes, { mode: 0o600 }); return target; } -export async function materializeCandidate(receipt: CandidateReceipt, directory: string) { +const ROLE_BINARY: Record = { + cli: "openshell", + gateway: "openshell-gateway", + sandbox: "openshell-sandbox", +}; + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function extractCandidateBinary( + archive: string, + artifactValue: Artifact, + runtimeDirectory: string, +): string { + const binaryName = ROLE_BINARY[artifactValue.role]; + const members = run("tar", ["-tzf", archive]); + if (members !== binaryName) { + throw new Error( + `${artifactValue.name} must contain exactly one ${binaryName} binary`, + ); + } + const listing = run("tar", ["-tvzf", archive]); + if ( + listing.includes("\n") || + !listing.startsWith("-") || + !listing.endsWith(` ${binaryName}`) + ) { + throw new Error( + `${artifactValue.name} must contain one regular ${binaryName} binary`, + ); + } + run("tar", ["-xzf", archive, "-C", runtimeDirectory, binaryName]); + return join(runtimeDirectory, binaryName); +} + +export function verifyCandidateInvocations(input: { + invocationLog: string; + lane: string; + receipt: CandidateReceipt; +}): LaneResult { + const entries = input.invocationLog + .split("\n") + .filter(Boolean) + .map((line) => { + const separator = line.indexOf("\t"); + if (separator <= 0) + throw new Error("candidate invocation log contains a malformed entry"); + return { + args: line.slice(separator + 1), + role: line.slice(0, separator), + }; + }); + const requiredRoles = + input.lane === "installer" ? ["cli", "gateway", "sandbox"] : ["gateway"]; + if ( + input.lane !== "installer" && + input.lane !== "live:openshell-gateway-auth-contract" + ) { + throw new Error(`unsupported candidate-aware lane: ${input.lane}`); + } + for (const role of requiredRoles) { + if ( + !entries.some( + (entry) => entry.role === role && entry.args === "--version", + ) + ) { + throw new Error(`lane ${input.lane} did not invoke ${role} --version`); + } + } + if ( + input.lane === "live:openshell-gateway-auth-contract" && + !entries.some( + (entry) => entry.role === "gateway" && entry.args !== "--version", + ) + ) { + throw new Error("live gateway lane did not start the candidate runtime"); + } + return { + conclusion: "success", + lane: input.lane, + observedOutput: entries + .map((entry) => `${entry.role} ${entry.args}`) + .join("\n"), + observedVersion: input.receipt.resolvedCandidate, + resolutionId: input.receipt.resolutionId, + }; +} + +export async function materializeCandidate( + receipt: CandidateReceipt, + directory: string, + fetcher: FetchLike = fetch, +) { await mkdir(directory, { recursive: true, mode: 0o700 }); - const artifactValue = receipt.artifacts[0]; - if (!artifactValue) throw new Error("candidate receipt has no artifact"); + const runtimeDirectory = join(directory, "runtime"); + const binDirectory = join(directory, "bin"); + await mkdir(runtimeDirectory, { mode: 0o700 }); + await mkdir(binDirectory, { mode: 0o700 }); const archives = await Promise.all( receipt.artifacts.map((candidateArtifact, index) => - downloadCandidateArtifact(candidateArtifact, directory, index), + downloadCandidateArtifact(candidateArtifact, directory, index, fetcher), ), ); - const archive = archives[0]!; - let binDirectory: string; - let observedOutput: string; - if (receipt.component === "openshell") { - const extractDirectory = join(directory, "openshell"); - await mkdir(extractDirectory, { mode: 0o700 }); - run("tar", ["-xzf", archive, "-C", extractDirectory]); - const binary = run("find", [ - extractDirectory, - "-type", - "f", - "-name", - "openshell", - "-print", - "-quit", - ]); - if (!binary) throw new Error("OpenShell archive has no openshell binary"); - await chmod(binary, 0o700); - binDirectory = resolve(binary, ".."); - observedOutput = run(binary, ["--version"]); - } else if (receipt.component === "dcode" || receipt.component === "hermes") { - const venv = join(directory, "venv"); - run("python3", ["-m", "venv", venv]); - const wheel = - receipt.component === "dcode" - ? archive - : archives[receipt.artifacts.findIndex((item) => item.kind === "wheel")]; - if (!wheel) throw new Error(`${receipt.component} receipt has no verified wheel`); - run(join(venv, "bin", "python"), ["-m", "pip", "install", "--no-deps", "--no-index", wheel]); - binDirectory = join(venv, "bin"); - const packageName = receipt.component === "dcode" ? "deepagents-code" : "hermes-agent"; - observedOutput = run(join(venv, "bin", "python"), [ - "-c", - `import importlib.metadata; print(importlib.metadata.version("${packageName}"))`, - ]); - } else { - const prefix = join(directory, "npm"); - run("npm", [ - "install", - "--ignore-scripts", - "--no-audit", - "--no-fund", - "--prefix", - prefix, + const observed: Record< + string, + { output: string; path: string; version: string } + > = {}; + for (const [index, artifactValue] of receipt.artifacts.entries()) { + const archive = archives[index]; + if (!archive) + throw new Error(`candidate artifact ${index} was not downloaded`); + const binary = extractCandidateBinary( archive, - ]); - binDirectory = join(prefix, "node_modules", ".bin"); - const executable = receipt.component === "openclaw" ? "openclaw" : "hermes"; - observedOutput = run(join(binDirectory, executable), ["--version"]); + artifactValue, + runtimeDirectory, + ); + await chmod(binary, 0o700); + const output = run(binary, ["--version"]); + const version = verifyObservedVersion(receipt, output); + const wrapper = join(binDirectory, ROLE_BINARY[artifactValue.role]); + await writeFile( + wrapper, + [ + "#!/bin/sh", + "set -eu", + ': "${NEMOCLAW_CANDIDATE_INVOCATION_LOG:?candidate invocation log is required}"', + `printf '%s\\t%s\\n' ${shellQuote(artifactValue.role)} "$*" >> "$NEMOCLAW_CANDIDATE_INVOCATION_LOG"`, + `exec ${shellQuote(binary)} "$@"`, + "", + ].join("\n"), + { mode: 0o500 }, + ); + observed[artifactValue.role] = { output, path: binary, version }; } - const observedVersion = verifyObservedVersion(receipt, observedOutput); return { binDirectory, component: receipt.component, - observedOutput, - observedVersion, + observed, resolutionId: receipt.resolutionId, schemaVersion: 1 as const, }; @@ -840,7 +831,9 @@ async function main(argv: string[]): Promise { githubToken: process.env.GITHUB_TOKEN, nemoclawSha: required(args, "--nemoclaw-sha"), }); - await writeFile(required(args, "--output"), `${stableJson(receipt)}\n`, { mode: 0o600 }); + await writeFile(required(args, "--output"), `${stableJson(receipt)}\n`, { + mode: 0o600, + }); return; } if (command === "plan") { @@ -851,7 +844,9 @@ async function main(argv: string[]): Promise { readFile(required(args, "--e2e-registry"), "utf8"), ]); const plan = buildCandidatePlan(component, sources.join("\n")); - await writeFile(required(args, "--output"), `${stableJson(plan)}\n`, { mode: 0o600 }); + await writeFile(required(args, "--output"), `${stableJson(plan)}\n`, { + mode: 0o600, + }); return; } if (command === "materialize") { @@ -859,18 +854,31 @@ async function main(argv: string[]): Promise { JSON.parse(await readFile(required(args, "--receipt"), "utf8")), required(args, "--resolution-id"), ); - const observed = await materializeCandidate(receipt, required(args, "--directory")); - await writeFile(required(args, "--output"), `${stableJson(observed)}\n`, { mode: 0o600 }); + const observed = await materializeCandidate( + receipt, + required(args, "--directory"), + ); + await writeFile(required(args, "--output"), `${stableJson(observed)}\n`, { + mode: 0o600, + }); const githubEnv = args.get("--github-env"); if (githubEnv) { + const invocationLog = join( + required(args, "--directory"), + "candidate-invocations.log", + ); await writeFile( githubEnv, [ `PATH=${observed.binDirectory}:${process.env.PATH ?? ""}`, `NEMOCLAW_CANDIDATE_COMPONENT=${receipt.component}`, + `NEMOCLAW_CANDIDATE_INVOCATION_LOG=${invocationLog}`, `NEMOCLAW_CANDIDATE_RECEIPT=${resolve(required(args, "--receipt"))}`, `NEMOCLAW_CANDIDATE_RESOLUTION_ID=${receipt.resolutionId}`, `NEMOCLAW_CANDIDATE_VERSION=${receipt.resolvedCandidate}`, + `NEMOCLAW_OPENSHELL_SANDBOX_BIN=${join(observed.binDirectory, "openshell-sandbox")}`, + `OPENSHELL_BIN=${join(observed.binDirectory, "openshell")}`, + `OPENSHELL_GATEWAY_BIN=${join(observed.binDirectory, "openshell-gateway")}`, "", ].join("\n"), { flag: "a" }, @@ -878,25 +886,36 @@ async function main(argv: string[]): Promise { } return; } + if (command === "verify-invocations") { + const receipt = parseCandidateReceipt( + JSON.parse(await readFile(required(args, "--receipt"), "utf8")), + required(args, "--resolution-id"), + ); + const result = verifyCandidateInvocations({ + invocationLog: await readFile(required(args, "--log"), "utf8"), + lane: required(args, "--lane"), + receipt, + }); + await writeFile(required(args, "--output"), `${stableJson(result)}\n`, { + mode: 0o600, + }); + return; + } if (command === "finalize") { const receipt = JSON.parse( await readFile(required(args, "--receipt"), "utf8"), ) as CandidateReceipt; - const plan = JSON.parse(await readFile(required(args, "--plan"), "utf8")) as CandidatePlan; + const plan = JSON.parse( + await readFile(required(args, "--plan"), "utf8"), + ) as CandidatePlan; const resultDirectory = required(args, "--results"); - const resultFiles = (await readdir(resultDirectory, { recursive: true })).filter((path) => - path.endsWith(".json"), - ); + const resultFiles = ( + await readdir(resultDirectory, { recursive: true }) + ).filter((path) => path.endsWith(".json")); const results = await Promise.all( - plan.deterministic - .filter((lane) => lane.status === "selected") - .map(async (lane) => { - const matches = resultFiles.filter((path) => basename(path) === `${lane.id}.json`); - if (matches.length !== 1) { - throw new Error(`expected exactly one result file for lane ${lane.id}`); - } - return JSON.parse(await readFile(join(resultDirectory, matches[0]!), "utf8")); - }), + resultFiles.map(async (path) => + JSON.parse(await readFile(join(resultDirectory, path), "utf8")), + ), ); const evidence = finalizeEvidence({ attempt: required(args, "--attempt"), @@ -905,15 +924,24 @@ async function main(argv: string[]): Promise { results, runId: required(args, "--run-id"), }); - await writeFile(required(args, "--output"), `${stableJson(evidence)}\n`, { mode: 0o600 }); + await writeFile(required(args, "--output"), `${stableJson(evidence)}\n`, { + mode: 0o600, + }); return; } - throw new Error("command must be resolve, plan, materialize, or finalize"); + throw new Error( + "command must be resolve, plan, materialize, verify-invocations, or finalize", + ); } -if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { main(process.argv.slice(2)).catch((error) => { - console.error(`candidate-compat: ${error instanceof Error ? error.message : String(error)}`); + console.error( + `candidate-compat: ${error instanceof Error ? error.message : String(error)}`, + ); process.exitCode = 1; }); } From 8bff537a5e5e7a2dfcf00f06016ba0436ec83c06 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Tue, 14 Jul 2026 17:18:12 -0700 Subject: [PATCH 3/4] fix(ci): bind candidate proof to tested runtime Signed-off-by: Ho Lim --- .../workflows/candidate-compatibility.yaml | 22 +- test/candidate-compat.test.ts | 76 +++- test/install-openshell-version-check.test.ts | 28 ++ tools/candidate-compat.mts | 416 +++++++----------- 4 files changed, 285 insertions(+), 257 deletions(-) diff --git a/.github/workflows/candidate-compatibility.yaml b/.github/workflows/candidate-compatibility.yaml index c857ccf81e..672bc0da86 100644 --- a/.github/workflows/candidate-compatibility.yaml +++ b/.github/workflows/candidate-compatibility.yaml @@ -184,10 +184,21 @@ jobs: set -euo pipefail exec > >(tee "candidate-lane-${LANE}.log") 2>&1 [[ "$LANE" == installer ]] || { echo "::error::untrusted lane id: $LANE"; exit 1; } - "$OPENSHELL_BIN" --version - "$OPENSHELL_GATEWAY_BIN" --version - "$NEMOCLAW_OPENSHELL_SANDBOX_BIN" --version - npx vitest run --project installer-integration + npx vitest run --project installer-integration \ + test/install-openshell-version-check.test.ts \ + --testNamePattern "executes every receipt-bound candidate component" + base_path="${PATH#*:}" + env \ + -u NEMOCLAW_CANDIDATE_COMPONENT \ + -u NEMOCLAW_CANDIDATE_INVOCATION_LOG \ + -u NEMOCLAW_CANDIDATE_RECEIPT \ + -u NEMOCLAW_CANDIDATE_RESOLUTION_ID \ + -u NEMOCLAW_CANDIDATE_VERSION \ + -u NEMOCLAW_OPENSHELL_SANDBOX_BIN \ + -u OPENSHELL_BIN \ + -u OPENSHELL_GATEWAY_BIN \ + PATH="$base_path" \ + npx vitest run --project installer-integration - name: Record receipt-bound lane result if: ${{ always() }} @@ -298,6 +309,7 @@ jobs: E2E_JOB: "1" E2E_TARGET_ID: openshell-gateway-auth-contract NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT: live:openshell-gateway-auth-contract:${{ needs.resolve.outputs.resolution_id }} NEMOCLAW_RUN_LIVE_E2E: "1" run: | set -euo pipefail @@ -384,6 +396,7 @@ jobs: - name: Finalize auditable evidence env: + RESOLUTION_ID: ${{ needs.resolve.outputs.resolution_id }} RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} shell: bash @@ -391,6 +404,7 @@ jobs: set -euo pipefail node --experimental-strip-types controller/tools/candidate-compat.mts finalize \ --receipt candidate-input/candidate-receipt.json \ + --resolution-id "$RESOLUTION_ID" \ --plan candidate-input/candidate-plan.json \ --results candidate-results \ --run-id "$RUN_ID" \ diff --git a/test/candidate-compat.test.ts b/test/candidate-compat.test.ts index 2e992a557f..762c398e02 100644 --- a/test/candidate-compat.test.ts +++ b/test/candidate-compat.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { createHash } from "node:crypto"; -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; import { spawnSync } from "node:child_process"; @@ -15,6 +15,7 @@ import { downloadCandidateArtifact, finalizeEvidence, materializeCandidate, + parseCandidatePlan, parseCandidateReceipt, resolveCandidate, verifyCandidateInvocations, @@ -267,7 +268,11 @@ describe("OpenShell candidate compatibility contract", () => { async (url) => new Response(archives.get(url)), ); const log = join(directory, "invocations.log"); - const env = { ...process.env, NEMOCLAW_CANDIDATE_INVOCATION_LOG: log }; + const env = { + ...process.env, + NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT: `installer:${candidateReceipt.resolutionId}`, + NEMOCLAW_CANDIDATE_INVOCATION_LOG: log, + }; for (const binary of ["openshell", "openshell-gateway", "openshell-sandbox"]) { expect(spawnSync(join(observed.binDirectory, binary), ["--version"], { env }).status).toBe( 0, @@ -282,11 +287,19 @@ describe("OpenShell candidate compatibility contract", () => { ).toMatchObject({ conclusion: "success", observedVersion: VERSION }); expect(() => verifyCandidateInvocations({ - invocationLog: "gateway\t--version\n", + invocationLog: `gateway\tlive:openshell-gateway-auth-contract:${candidateReceipt.resolutionId}\t--version\n`, lane: "live:openshell-gateway-auth-contract", receipt: candidateReceipt, }), ).toThrow("did not start the candidate runtime"); + expect(() => + verifyCandidateInvocations({ + invocationLog: + "cli\tpreflight\t--version\ngateway\tpreflight\t--version\nsandbox\tpreflight\t--version\n", + lane: "installer", + receipt: candidateReceipt, + }), + ).toThrow("receipt-bound cli"); } finally { rmSync(directory, { force: true, recursive: true }); } @@ -294,6 +307,7 @@ describe("OpenShell candidate compatibility contract", () => { it("selects only candidate-aware deterministic and live lanes (#6691)", () => { const plan = buildCandidatePlan("openshell", E2E_SOURCES); + expect(parseCandidatePlan(plan, "openshell")).toEqual(plan); expect( plan.deterministic.filter(({ status }) => status === "selected").map(({ id }) => id), ).toEqual(["installer"]); @@ -304,6 +318,62 @@ describe("OpenShell candidate compatibility contract", () => { status: "selected", }), ]); + expect(() => parseCandidatePlan({ ...plan, deterministic: [], live: [] }, "openshell")).toThrow( + "every deterministic lane", + ); + }); + + it("rejects an empty persisted plan through the finalize CLI (#6691)", async () => { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-candidate-finalize-")); + try { + const candidateReceipt = await resolveCandidate({ + candidate: `v${VERSION}`, + component: "openshell", + fetcher: async () => response(releaseMetadata()), + nemoclawSha: SHA, + }); + const receiptPath = join(directory, "receipt.json"); + const planPath = join(directory, "plan.json"); + const resultsPath = join(directory, "results"); + writeFileSync(receiptPath, JSON.stringify(candidateReceipt)); + writeFileSync( + planPath, + JSON.stringify({ + component: "openshell", + deterministic: [], + live: [], + schemaVersion: 1, + }), + ); + mkdirSync(resultsPath); + const result = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + resolve("tools/candidate-compat.mts"), + "finalize", + "--receipt", + receiptPath, + "--resolution-id", + candidateReceipt.resolutionId, + "--plan", + planPath, + "--results", + resultsPath, + "--run-id", + "123", + "--attempt", + "1", + "--output", + join(directory, "evidence.json"), + ], + { encoding: "utf8" }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain("every deterministic lane exactly once"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } }); it("finalizes only complete receipt-bound deterministic and live evidence (#6691)", () => { diff --git a/test/install-openshell-version-check.test.ts b/test/install-openshell-version-check.test.ts index eaaaba4ddc..5c5015c615 100644 --- a/test/install-openshell-version-check.test.ts +++ b/test/install-openshell-version-check.test.ts @@ -11,6 +11,14 @@ import credentialBoundaryManifest from "../src/lib/actions/sandbox/openshell-chi import { buildRebuildHermesChildEnv } from "./e2e/live/rebuild-hermes-env.ts"; const SCRIPT = path.join(import.meta.dirname, "..", "scripts", "install-openshell.sh"); +const CANDIDATE_RUNTIME = { + cli: process.env.OPENSHELL_BIN, + gateway: process.env.OPENSHELL_GATEWAY_BIN, + resolutionId: process.env.NEMOCLAW_CANDIDATE_RESOLUTION_ID, + sandbox: process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN, + version: process.env.NEMOCLAW_CANDIDATE_VERSION, +}; +const CANDIDATE_RUNTIME_ENABLED = Object.values(CANDIDATE_RUNTIME).every(Boolean); const PINNED_OPEN_SHELL_SHA256 = { cliDarwinArm64: "117b5354cc42d80bc4d5e070ea5ac4e341208ff6d3c29b516d8a9c80e2310f8d", cliLinuxArm64: "a5ff01a3240d73c72ec1700eda6cc6c752a86cf50c5dd1b5bdc459f544d03045", @@ -209,6 +217,26 @@ exit 0`, } describe("install-openshell.sh version check", { timeout: 15_000 }, () => { + it.runIf(CANDIDATE_RUNTIME_ENABLED)( + "executes every receipt-bound candidate component under the installer lane (#6691)", + () => { + const context = `installer:${CANDIDATE_RUNTIME.resolutionId}`; + for (const [role, binary] of Object.entries(CANDIDATE_RUNTIME).filter( + ([role]) => role === "cli" || role === "gateway" || role === "sandbox", + )) { + const result = spawnSync(binary!, ["--version"], { + encoding: "utf8", + env: { + ...process.env, + NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT: context, + }, + }); + expect(result.status, `${role}: ${result.stderr}`).toBe(0); + expect(result.stdout).toContain(CANDIDATE_RUNTIME.version); + } + }, + ); + it("exits cleanly when the required OpenShell and driver binaries are already installed", () => { const result = runWithInstalledVersion(REQUIRED_OPENSHELL_VERSION); expect(result.status).toBe(0); diff --git a/tools/candidate-compat.mts b/tools/candidate-compat.mts index 4dd86b8556..693b59c7cd 100644 --- a/tools/candidate-compat.mts +++ b/tools/candidate-compat.mts @@ -66,15 +66,8 @@ type DeterministicLane = const FULL_SHA = /^[a-f0-9]{40}$/; const VERSION = /^[0-9]+(?:\.[0-9]+){2}(?:[-+][0-9A-Za-z.-]+)?$/; const DIGEST = /^[a-f0-9]+$/; -const OFFICIAL_HOSTS = new Set([ - "api.github.com", - "codeload.github.com", - "github.com", -]); -const DOWNLOAD_HOSTS = new Set([ - ...OFFICIAL_HOSTS, - "release-assets.githubusercontent.com", -]); +const OFFICIAL_HOSTS = new Set(["api.github.com", "codeload.github.com", "github.com"]); +const DOWNLOAD_HOSTS = new Set([...OFFICIAL_HOSTS, "release-assets.githubusercontent.com"]); const MAX_DOWNLOAD_REDIRECTS = 5; const ALL_LANES: DeterministicLane[] = [ "source-unit", @@ -85,10 +78,7 @@ const ALL_LANES: DeterministicLane[] = [ "e2e-support", ]; -const SELECTED_LANES: Record< - CandidateComponent, - ReadonlySet -> = { +const SELECTED_LANES: Record> = { openshell: new Set(["installer"]), }; @@ -121,15 +111,8 @@ function assertComponent(value: string): asserts value is CandidateComponent { } } -function assertCandidate( - component: CandidateComponent, - candidate: string, -): void { - if ( - candidate.length > 128 || - candidate.includes("/") || - /[\x00-\x20\x7f]/u.test(candidate) - ) { +function assertCandidate(component: CandidateComponent, candidate: string): void { + if (candidate.length > 128 || candidate.includes("/") || /[\x00-\x20\x7f]/u.test(candidate)) { throw new Error("candidate contains unsafe characters"); } const normalized = candidate.replace(/^v/u, ""); @@ -141,14 +124,10 @@ function assertCandidate( function assertOfficialUrl(raw: string, expectedPathPrefix: string): URL { const url = new URL(raw); if (url.protocol !== "https:" || !OFFICIAL_HOSTS.has(url.hostname)) { - throw new Error( - `candidate artifact is not on an approved official HTTPS host: ${raw}`, - ); + throw new Error(`candidate artifact is not on an approved official HTTPS host: ${raw}`); } if (!url.pathname.startsWith(expectedPathPrefix)) { - throw new Error( - `candidate artifact has unexpected official-source path: ${raw}`, - ); + throw new Error(`candidate artifact has unexpected official-source path: ${raw}`); } if (url.username || url.password || url.search || url.hash) { throw new Error( @@ -158,11 +137,7 @@ function assertOfficialUrl(raw: string, expectedPathPrefix: string): URL { return url; } -async function fetchJson( - fetcher: FetchLike, - url: string, - token?: string, -): Promise { +async function fetchJson(fetcher: FetchLike, url: string, token?: string): Promise { const response = await fetcher(url, { headers: { Accept: "application/vnd.github+json, application/json", @@ -171,9 +146,7 @@ async function fetchJson( redirect: "error", }); if (!response.ok) - throw new Error( - `official metadata request failed (${response.status}): ${url}`, - ); + throw new Error(`official metadata request failed (${response.status}): ${url}`); return response.json(); } @@ -184,11 +157,7 @@ function record(value: unknown, label: string): Record { return value as Record; } -function stringField( - value: Record, - key: string, - label: string, -): string { +function stringField(value: Record, key: string, label: string): string { const field = value[key]; if (typeof field !== "string" || field.length === 0) { throw new Error(`${label}.${key} must be a non-empty string`); @@ -198,13 +167,10 @@ function stringField( function artifact(input: Artifact, expectedPathPrefix: string): Artifact { const url = assertOfficialUrl(input.url, expectedPathPrefix); - if (!DIGEST.test(input.digest)) - throw new Error(`${input.name} has an invalid digest`); + if (!DIGEST.test(input.digest)) throw new Error(`${input.name} has an invalid digest`); const expectedLength = 64; if (input.digest.length !== expectedLength) { - throw new Error( - `${input.name} has an invalid ${input.digestAlgorithm} digest`, - ); + throw new Error(`${input.name} has an invalid ${input.digestAlgorithm} digest`); } return { ...input, url: url.href }; } @@ -213,9 +179,7 @@ async function resolveOpenShell( candidate: string, fetcher: FetchLike, token?: string, -): Promise< - Omit -> { +): Promise> { const version = candidate.replace(/^v/u, ""); const tag = `v${version}`; const metadata = record( @@ -226,15 +190,11 @@ async function resolveOpenShell( ), "OpenShell release", ); - if (metadata.draft === true) - throw new Error("OpenShell candidate release is a draft"); + if (metadata.draft === true) throw new Error("OpenShell candidate release is a draft"); if (stringField(metadata, "tag_name", "OpenShell release") !== tag) { - throw new Error( - "OpenShell release tag does not match the requested candidate", - ); + throw new Error("OpenShell release tag does not match the requested candidate"); } - if (!Array.isArray(metadata.assets)) - throw new Error("OpenShell release assets must be an array"); + if (!Array.isArray(metadata.assets)) throw new Error("OpenShell release assets must be an array"); const requiredAssets = [ { name: "openshell-x86_64-unknown-linux-musl.tar.gz", @@ -249,17 +209,11 @@ async function resolveOpenShell( role: "sandbox" as const, }, ]; - const releaseAssets = metadata.assets.map((item) => - record(item, "OpenShell asset"), - ); + const releaseAssets = metadata.assets.map((item) => record(item, "OpenShell asset")); const artifacts = requiredAssets.map(({ name, role }) => { const assetMetadata = releaseAssets.find((item) => item.name === name); if (!assetMetadata) throw new Error(`OpenShell release is missing ${name}`); - const digest = stringField( - assetMetadata, - "digest", - `OpenShell asset ${name}`, - ); + const digest = stringField(assetMetadata, "digest", `OpenShell asset ${name}`); if (!digest.startsWith("sha256:")) { throw new Error(`OpenShell asset ${name} is missing SHA-256 provenance`); } @@ -270,11 +224,7 @@ async function resolveOpenShell( kind: "archive", name, role, - url: stringField( - assetMetadata, - "browser_download_url", - `OpenShell asset ${name}`, - ), + url: stringField(assetMetadata, "browser_download_url", `OpenShell asset ${name}`), }, `/NVIDIA/OpenShell/releases/download/${tag}/`, ); @@ -296,15 +246,10 @@ export async function resolveCandidate(input: { nemoclawSha: string; }): Promise { assertComponent(input.component); - if (!FULL_SHA.test(input.nemoclawSha)) - throw new Error("NemoClaw ref must resolve to a full SHA"); + if (!FULL_SHA.test(input.nemoclawSha)) throw new Error("NemoClaw ref must resolve to a full SHA"); assertCandidate(input.component, input.candidate); const fetcher = input.fetcher ?? fetch; - const resolved = await resolveOpenShell( - input.candidate, - fetcher, - input.githubToken, - ); + const resolved = await resolveOpenShell(input.candidate, fetcher, input.githubToken); const receiptWithoutId = { ...resolved, nemoclawSha: input.nemoclawSha, @@ -316,21 +261,11 @@ export async function resolveCandidate(input: { }; } -function parseArtifact( - value: unknown, - resolvedCandidate: string, - index: number, -): Artifact { +function parseArtifact(value: unknown, resolvedCandidate: string, index: number): Artifact { const input = record(value, `candidate artifact ${index}`); - const digestAlgorithm = stringField( - input, - "digestAlgorithm", - `candidate artifact ${index}`, - ); + const digestAlgorithm = stringField(input, "digestAlgorithm", `candidate artifact ${index}`); if (digestAlgorithm !== "sha256") { - throw new Error( - `candidate artifact ${index} has an invalid digest algorithm`, - ); + throw new Error(`candidate artifact ${index} has an invalid digest algorithm`); } const kind = stringField(input, "kind", `candidate artifact ${index}`); if (kind !== "archive") { @@ -356,9 +291,7 @@ function parseArtifact( }, ][index]; if (!expected || name !== expected.name || input.role !== expected.role) { - throw new Error( - `candidate artifact ${index} does not match the resolved component`, - ); + throw new Error(`candidate artifact ${index} does not match the resolved component`); } return artifact( { @@ -382,48 +315,30 @@ export function parseCandidateReceipt( throw new Error("expected candidate resolution id is invalid"); } const input = record(value, "candidate receipt"); - if (input.schemaVersion !== 1) - throw new Error("candidate receipt schema version must be 1"); + if (input.schemaVersion !== 1) throw new Error("candidate receipt schema version must be 1"); const component = stringField(input, "component", "candidate receipt"); assertComponent(component); - const requestedCandidate = stringField( - input, - "requestedCandidate", - "candidate receipt", - ); + const requestedCandidate = stringField(input, "requestedCandidate", "candidate receipt"); assertCandidate(component, requestedCandidate); - const resolvedCandidate = stringField( - input, - "resolvedCandidate", - "candidate receipt", - ); + const resolvedCandidate = stringField(input, "resolvedCandidate", "candidate receipt"); if (!VERSION.test(resolvedCandidate)) { throw new Error("candidate receipt has an invalid resolved version"); } if (requestedCandidate.replace(/^v/u, "") !== resolvedCandidate) { - throw new Error( - "candidate receipt resolved version does not match the request", - ); + throw new Error("candidate receipt resolved version does not match the request"); } const nemoclawSha = stringField(input, "nemoclawSha", "candidate receipt"); - if (!FULL_SHA.test(nemoclawSha)) - throw new Error("candidate receipt has an invalid NemoClaw SHA"); + if (!FULL_SHA.test(nemoclawSha)) throw new Error("candidate receipt has an invalid NemoClaw SHA"); if (!Array.isArray(input.artifacts)) throw new Error("candidate receipt artifacts must be an array"); const expectedArtifactCount = 3; if (input.artifacts.length !== expectedArtifactCount) { - throw new Error( - `candidate receipt for ${component} has an invalid artifact count`, - ); + throw new Error(`candidate receipt for ${component} has an invalid artifact count`); } const artifacts = input.artifacts.map((item, index) => parseArtifact(item, resolvedCandidate, index), ); - const officialSource = stringField( - input, - "officialSource", - "candidate receipt", - ); + const officialSource = stringField(input, "officialSource", "candidate receipt"); const sourcePattern = /^github:NVIDIA\/OpenShell:release:[1-9][0-9]*$/u; if (!sourcePattern.test(officialSource)) { throw new Error("candidate receipt has an invalid official source"); @@ -445,9 +360,7 @@ export function parseCandidateReceipt( stringField(input, "resolutionId", "candidate receipt") !== resolutionId || resolutionId !== expectedResolutionId ) { - throw new Error( - "candidate receipt does not match the trusted resolution id", - ); + throw new Error("candidate receipt does not match the trusted resolution id"); } return { ...receiptWithoutId, resolutionId }; } @@ -458,17 +371,13 @@ export function buildCandidatePlan( ): CandidatePlan { const selected = SELECTED_LANES[component]; const live = LIVE_SELECTORS[component].map((entry) => { - const marker = - entry.selector === "job" ? ` ${entry.id}:\n` : `id: "${entry.id}"`; + const marker = entry.selector === "job" ? ` ${entry.id}:\n` : `id: "${entry.id}"`; if (!e2eSources.includes(marker)) { - throw new Error( - `E2E source of truth does not declare ${entry.selector} ${entry.id}`, - ); + throw new Error(`E2E source of truth does not declare ${entry.selector} ${entry.id}`); } return { ...entry, - reason: - "the selected live job executes the digest-bound OpenShell gateway candidate", + reason: "the selected live job executes the digest-bound OpenShell gateway candidate", status: "selected" as const, }; }); @@ -486,30 +395,91 @@ export function buildCandidatePlan( }; } +/** Validate a persisted plan before it can select compatibility evidence. */ +export function parseCandidatePlan( + value: unknown, + expectedComponent: CandidateComponent, +): CandidatePlan { + const input = record(value, "candidate plan"); + if (input.schemaVersion !== 1) throw new Error("candidate plan schema version must be 1"); + const component = stringField(input, "component", "candidate plan"); + assertComponent(component); + if (component !== expectedComponent) { + throw new Error("candidate plan component does not match the receipt"); + } + if (!Array.isArray(input.deterministic)) { + throw new Error("candidate plan deterministic lanes must be an array"); + } + const deterministic = input.deterministic.map((value, index) => { + const lane = record(value, `candidate plan deterministic lane ${index}`); + const id = stringField(lane, "id", `candidate plan deterministic lane ${index}`); + if (!ALL_LANES.includes(id as DeterministicLane)) { + throw new Error(`candidate plan has an unsupported deterministic lane: ${id}`); + } + const statusValue = stringField(lane, "status", `candidate plan deterministic lane ${index}`); + if (statusValue !== "selected" && statusValue !== "skipped") { + throw new Error(`candidate plan deterministic lane ${id} has an invalid status`); + } + const status: "selected" | "skipped" = statusValue; + return { + id: id as DeterministicLane, + reason: stringField(lane, "reason", `candidate plan deterministic lane ${index}`), + status, + }; + }); + if ( + deterministic.length !== ALL_LANES.length || + new Set(deterministic.map(({ id }) => id)).size !== ALL_LANES.length + ) { + throw new Error("candidate plan must account for every deterministic lane exactly once"); + } + for (const lane of deterministic) { + const expectedStatus = SELECTED_LANES[component].has(lane.id) ? "selected" : "skipped"; + if (lane.status !== expectedStatus) { + throw new Error(`candidate plan has an invalid selection for ${lane.id}`); + } + } + if (!Array.isArray(input.live)) { + throw new Error("candidate plan live lanes must be an array"); + } + const expectedLive = LIVE_SELECTORS[component]; + const live = input.live.map((value, index) => { + const lane = record(value, `candidate plan live lane ${index}`); + const id = stringField(lane, "id", `candidate plan live lane ${index}`); + const selector = stringField(lane, "selector", `candidate plan live lane ${index}`); + const expected = expectedLive.find((candidate) => candidate.id === id); + if (!expected || selector !== expected.selector) { + throw new Error(`candidate plan has an unsupported live lane: ${id}`); + } + if (lane.status !== "selected") { + throw new Error(`candidate plan live lane ${id} must be selected`); + } + return { + id, + reason: stringField(lane, "reason", `candidate plan live lane ${index}`), + selector: expected.selector, + status: "selected" as const, + }; + }); + if ( + live.length !== expectedLive.length || + new Set(live.map(({ id }) => id)).size !== expectedLive.length + ) { + throw new Error("candidate plan must account for every live lane exactly once"); + } + return { component, deterministic, live, schemaVersion: 1 }; +} + export function verifyDigest(bytes: Uint8Array, artifactValue: Artifact): void { - const actual = createHash(artifactValue.digestAlgorithm) - .update(bytes) - .digest("hex"); + const actual = createHash(artifactValue.digestAlgorithm).update(bytes).digest("hex"); if (actual !== artifactValue.digest) { - throw new Error( - `${artifactValue.name} ${artifactValue.digestAlgorithm} mismatch`, - ); + throw new Error(`${artifactValue.name} ${artifactValue.digestAlgorithm} mismatch`); } } -export function verifyObservedVersion( - receipt: CandidateReceipt, - output: string, -): string { - const escaped = receipt.resolvedCandidate.replace( - /[.*+?^${}()|[\]\\]/g, - "\\$&", - ); - if ( - !new RegExp(`(^|[^0-9A-Za-z.])v?${escaped}([^0-9A-Za-z.]|$)`, "u").test( - output.trim(), - ) - ) { +export function verifyObservedVersion(receipt: CandidateReceipt, output: string): string { + const escaped = receipt.resolvedCandidate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + if (!new RegExp(`(^|[^0-9A-Za-z.])v?${escaped}([^0-9A-Za-z.]|$)`, "u").test(output.trim())) { throw new Error( `observed ${receipt.component} version does not match ${receipt.resolvedCandidate}: ${output.trim()}`, ); @@ -524,46 +494,33 @@ export function finalizeEvidence(input: { results: LaneResult[]; runId: string; }): Record { - if ( - !/^[1-9][0-9]*$/u.test(input.runId) || - !/^[1-9][0-9]*$/u.test(input.attempt) - ) { + if (!/^[1-9][0-9]*$/u.test(input.runId) || !/^[1-9][0-9]*$/u.test(input.attempt)) { throw new Error("run id and attempt must be positive integers"); } const selected = [ - ...input.plan.deterministic - .filter((lane) => lane.status === "selected") - .map((lane) => lane.id), + ...input.plan.deterministic.filter((lane) => lane.status === "selected").map((lane) => lane.id), ...input.plan.live .filter((lane) => lane.status === "selected") .map((lane) => `live:${lane.id}`), ].sort(); const actual = input.results.map((result) => result.lane).sort(); - if ( - new Set(actual).size !== actual.length || - stableJson(actual) !== stableJson(selected) - ) { + if (new Set(actual).size !== actual.length || stableJson(actual) !== stableJson(selected)) { throw new Error( "lane results do not account for every selected compatibility lane exactly once", ); } for (const result of input.results) { if (result.resolutionId !== input.receipt.resolutionId) { - throw new Error( - `lane ${result.lane} used a different candidate resolution`, - ); + throw new Error(`lane ${result.lane} used a different candidate resolution`); } if (result.conclusion !== "success" && result.conclusion !== "failure") { throw new Error(`lane ${result.lane} has an invalid conclusion`); } if ( result.conclusion === "success" && - (result.observedVersion !== input.receipt.resolvedCandidate || - !result.observedOutput) + (result.observedVersion !== input.receipt.resolvedCandidate || !result.observedOutput) ) { - throw new Error( - `lane ${result.lane} has no matching observed candidate version`, - ); + throw new Error(`lane ${result.lane} has no matching observed candidate version`); } } return { @@ -592,14 +549,8 @@ function run( stdio: ["ignore", "pipe", "pipe"], }); if (result.status !== 0) { - const detail = - result.error?.message ?? - result.stderr?.trim() ?? - result.stdout?.trim() ?? - ""; - throw new Error( - `${command} ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`, - ); + const detail = result.error?.message ?? result.stderr?.trim() ?? result.stdout?.trim() ?? ""; + throw new Error(`${command} ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`); } return result.stdout.trim(); } @@ -619,24 +570,16 @@ function assertDownloadUrl(raw: string): URL { return url; } -async function fetchDownload( - fetcher: FetchLike, - rawUrl: string, -): Promise { +async function fetchDownload(fetcher: FetchLike, rawUrl: string): Promise { let current = assertDownloadUrl(rawUrl); - for ( - let redirectCount = 0; - redirectCount <= MAX_DOWNLOAD_REDIRECTS; - redirectCount += 1 - ) { + for (let redirectCount = 0; redirectCount <= MAX_DOWNLOAD_REDIRECTS; redirectCount += 1) { const response = await fetcher(current.href, { redirect: "manual" }); if (![301, 302, 303, 307, 308].includes(response.status)) return response; if (redirectCount === MAX_DOWNLOAD_REDIRECTS) { throw new Error("candidate download exceeded the redirect limit"); } const location = response.headers.get("location"); - if (!location) - throw new Error("candidate download redirect has no location"); + if (!location) throw new Error("candidate download redirect has no location"); current = assertDownloadUrl(new URL(location, current).href); } throw new Error("candidate download redirect handling failed"); @@ -649,8 +592,7 @@ export async function downloadCandidateArtifact( fetcher: FetchLike = fetch, ): Promise { const response = await fetchDownload(fetcher, artifactValue.url); - if (!response.ok) - throw new Error(`candidate download failed (${response.status})`); + if (!response.ok) throw new Error(`candidate download failed (${response.status})`); const bytes = new Uint8Array(await response.arrayBuffer()); verifyDigest(bytes, artifactValue); const target = join(directory, `candidate-${index}.tar.gz`); @@ -676,19 +618,11 @@ function extractCandidateBinary( const binaryName = ROLE_BINARY[artifactValue.role]; const members = run("tar", ["-tzf", archive]); if (members !== binaryName) { - throw new Error( - `${artifactValue.name} must contain exactly one ${binaryName} binary`, - ); + throw new Error(`${artifactValue.name} must contain exactly one ${binaryName} binary`); } const listing = run("tar", ["-tvzf", archive]); - if ( - listing.includes("\n") || - !listing.startsWith("-") || - !listing.endsWith(` ${binaryName}`) - ) { - throw new Error( - `${artifactValue.name} must contain one regular ${binaryName} binary`, - ); + if (listing.includes("\n") || !listing.startsWith("-") || !listing.endsWith(` ${binaryName}`)) { + throw new Error(`${artifactValue.name} must contain one regular ${binaryName} binary`); } run("tar", ["-xzf", archive, "-C", runtimeDirectory, binaryName]); return join(runtimeDirectory, binaryName); @@ -703,35 +637,35 @@ export function verifyCandidateInvocations(input: { .split("\n") .filter(Boolean) .map((line) => { - const separator = line.indexOf("\t"); - if (separator <= 0) + const fields = line.split("\t"); + if (fields.length !== 3 || !fields[0] || !fields[1]) throw new Error("candidate invocation log contains a malformed entry"); return { - args: line.slice(separator + 1), - role: line.slice(0, separator), + args: fields[2] ?? "", + context: fields[1], + role: fields[0], }; }); - const requiredRoles = - input.lane === "installer" ? ["cli", "gateway", "sandbox"] : ["gateway"]; - if ( - input.lane !== "installer" && - input.lane !== "live:openshell-gateway-auth-contract" - ) { + const requiredRoles = input.lane === "installer" ? ["cli", "gateway", "sandbox"] : ["gateway"]; + if (input.lane !== "installer" && input.lane !== "live:openshell-gateway-auth-contract") { throw new Error(`unsupported candidate-aware lane: ${input.lane}`); } + const expectedContext = `${input.lane}:${input.receipt.resolutionId}`; for (const role of requiredRoles) { if ( !entries.some( - (entry) => entry.role === role && entry.args === "--version", + (entry) => + entry.role === role && entry.context === expectedContext && entry.args === "--version", ) ) { - throw new Error(`lane ${input.lane} did not invoke ${role} --version`); + throw new Error(`lane ${input.lane} did not invoke receipt-bound ${role} --version`); } } if ( input.lane === "live:openshell-gateway-auth-contract" && !entries.some( - (entry) => entry.role === "gateway" && entry.args !== "--version", + (entry) => + entry.role === "gateway" && entry.context === expectedContext && entry.args !== "--version", ) ) { throw new Error("live gateway lane did not start the candidate runtime"); @@ -740,7 +674,7 @@ export function verifyCandidateInvocations(input: { conclusion: "success", lane: input.lane, observedOutput: entries - .map((entry) => `${entry.role} ${entry.args}`) + .map((entry) => `${entry.role} ${entry.context} ${entry.args}`) .join("\n"), observedVersion: input.receipt.resolvedCandidate, resolutionId: input.receipt.resolutionId, @@ -762,19 +696,11 @@ export async function materializeCandidate( downloadCandidateArtifact(candidateArtifact, directory, index, fetcher), ), ); - const observed: Record< - string, - { output: string; path: string; version: string } - > = {}; + const observed: Record = {}; for (const [index, artifactValue] of receipt.artifacts.entries()) { const archive = archives[index]; - if (!archive) - throw new Error(`candidate artifact ${index} was not downloaded`); - const binary = extractCandidateBinary( - archive, - artifactValue, - runtimeDirectory, - ); + if (!archive) throw new Error(`candidate artifact ${index} was not downloaded`); + const binary = extractCandidateBinary(archive, artifactValue, runtimeDirectory); await chmod(binary, 0o700); const output = run(binary, ["--version"]); const version = verifyObservedVersion(receipt, output); @@ -785,7 +711,8 @@ export async function materializeCandidate( "#!/bin/sh", "set -eu", ': "${NEMOCLAW_CANDIDATE_INVOCATION_LOG:?candidate invocation log is required}"', - `printf '%s\\t%s\\n' ${shellQuote(artifactValue.role)} "$*" >> "$NEMOCLAW_CANDIDATE_INVOCATION_LOG"`, + ': "${NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT:?candidate invocation context is required}"', + `printf '%s\\t%s\\t%s\\n' ${shellQuote(artifactValue.role)} "$NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT" "$*" >> "$NEMOCLAW_CANDIDATE_INVOCATION_LOG"`, `exec ${shellQuote(binary)} "$@"`, "", ].join("\n"), @@ -854,19 +781,13 @@ async function main(argv: string[]): Promise { JSON.parse(await readFile(required(args, "--receipt"), "utf8")), required(args, "--resolution-id"), ); - const observed = await materializeCandidate( - receipt, - required(args, "--directory"), - ); + const observed = await materializeCandidate(receipt, required(args, "--directory")); await writeFile(required(args, "--output"), `${stableJson(observed)}\n`, { mode: 0o600, }); const githubEnv = args.get("--github-env"); if (githubEnv) { - const invocationLog = join( - required(args, "--directory"), - "candidate-invocations.log", - ); + const invocationLog = join(required(args, "--directory"), "candidate-invocations.log"); await writeFile( githubEnv, [ @@ -902,16 +823,18 @@ async function main(argv: string[]): Promise { return; } if (command === "finalize") { - const receipt = JSON.parse( - await readFile(required(args, "--receipt"), "utf8"), - ) as CandidateReceipt; - const plan = JSON.parse( - await readFile(required(args, "--plan"), "utf8"), - ) as CandidatePlan; + const receipt = parseCandidateReceipt( + JSON.parse(await readFile(required(args, "--receipt"), "utf8")), + required(args, "--resolution-id"), + ); + const plan = parseCandidatePlan( + JSON.parse(await readFile(required(args, "--plan"), "utf8")), + receipt.component, + ); const resultDirectory = required(args, "--results"); - const resultFiles = ( - await readdir(resultDirectory, { recursive: true }) - ).filter((path) => path.endsWith(".json")); + const resultFiles = (await readdir(resultDirectory, { recursive: true })).filter((path) => + path.endsWith(".json"), + ); const results = await Promise.all( resultFiles.map(async (path) => JSON.parse(await readFile(join(resultDirectory, path), "utf8")), @@ -929,19 +852,12 @@ async function main(argv: string[]): Promise { }); return; } - throw new Error( - "command must be resolve, plan, materialize, verify-invocations, or finalize", - ); + throw new Error("command must be resolve, plan, materialize, verify-invocations, or finalize"); } -if ( - process.argv[1] && - resolve(process.argv[1]) === fileURLToPath(import.meta.url) -) { +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { main(process.argv.slice(2)).catch((error) => { - console.error( - `candidate-compat: ${error instanceof Error ? error.message : String(error)}`, - ); + console.error(`candidate-compat: ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; }); } From 982a722ced2d87282d05a937cb9c04d6add54ee8 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Wed, 15 Jul 2026 01:16:15 -0700 Subject: [PATCH 4/4] fix(ci): exercise candidate through installer path Signed-off-by: Ho Lim --- .../workflows/candidate-compatibility.yaml | 2 +- test/candidate-compat.test.ts | 33 ++++++++++++++----- test/install-openshell-version-check.test.ts | 30 +++++++++-------- tools/candidate-compat.mts | 19 +++++++++++ 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/.github/workflows/candidate-compatibility.yaml b/.github/workflows/candidate-compatibility.yaml index 672bc0da86..6c7d14814b 100644 --- a/.github/workflows/candidate-compatibility.yaml +++ b/.github/workflows/candidate-compatibility.yaml @@ -186,7 +186,7 @@ jobs: [[ "$LANE" == installer ]] || { echo "::error::untrusted lane id: $LANE"; exit 1; } npx vitest run --project installer-integration \ test/install-openshell-version-check.test.ts \ - --testNamePattern "executes every receipt-bound candidate component" + --testNamePattern "validates the receipt-bound candidate through the installer path" base_path="${PATH#*:}" env \ -u NEMOCLAW_CANDIDATE_COMPONENT \ diff --git a/test/candidate-compat.test.ts b/test/candidate-compat.test.ts index 762c398e02..a971ab6622 100644 --- a/test/candidate-compat.test.ts +++ b/test/candidate-compat.test.ts @@ -1,17 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; -import { spawnSync } from "node:child_process"; import { describe, expect, it } from "vitest"; import { parse as parseYaml } from "yaml"; import { buildCandidatePlan, + type CandidateReceipt, downloadCandidateArtifact, finalizeEvidence, materializeCandidate, @@ -21,7 +22,6 @@ import { verifyCandidateInvocations, verifyDigest, verifyObservedVersion, - type CandidateReceipt, } from "../tools/candidate-compat.mts"; const SHA = "a".repeat(40); @@ -244,7 +244,16 @@ describe("OpenShell candidate compatibility contract", () => { const assets = receipt().artifacts.map((artifact) => { const binaryName = artifact.role === "cli" ? "openshell" : `openshell-${artifact.role}`; const source = join(directory, binaryName); - writeFileSync(source, `#!/bin/sh\nprintf '%s\\n' '${binaryName} ${VERSION}'\n`); + const features = + artifact.role === "cli" + ? "# request-body-credential-rewrite websocket-credential-rewrite" + : artifact.role === "sandbox" + ? "# allow_all_known_mcp_methods" + : ""; + writeFileSync( + source, + `#!/bin/sh\n${features}\nprintf '%s\\n' '${binaryName} ${VERSION}'\n`, + ); chmodSync(source, 0o700); const archive = join(directory, `${artifact.role}-exact.tar.gz`); expect(spawnSync("tar", ["-czf", archive, "-C", directory, binaryName]).status).toBe(0); @@ -272,12 +281,20 @@ describe("OpenShell candidate compatibility contract", () => { ...process.env, NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT: `installer:${candidateReceipt.resolutionId}`, NEMOCLAW_CANDIDATE_INVOCATION_LOG: log, + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + NEMOCLAW_OPENSHELL_GATEWAY_BIN: join(observed.binDirectory, "openshell-gateway"), + NEMOCLAW_OPENSHELL_MAX_VERSION: VERSION, + NEMOCLAW_OPENSHELL_MIN_VERSION: VERSION, + NEMOCLAW_OPENSHELL_PIN_VERSION: VERSION, + NEMOCLAW_OPENSHELL_SANDBOX_BIN: join(observed.binDirectory, "openshell-sandbox"), + PATH: `${observed.binDirectory}:/usr/bin:/bin`, }; - for (const binary of ["openshell", "openshell-gateway", "openshell-sandbox"]) { - expect(spawnSync(join(observed.binDirectory, binary), ["--version"], { env }).status).toBe( - 0, - ); - } + const installer = spawnSync("bash", [resolve("scripts/install-openshell.sh")], { + encoding: "utf8", + env, + }); + expect(installer.status, `${installer.stdout}\n${installer.stderr}`).toBe(0); + expect(installer.stdout).toContain(`openshell already installed: ${VERSION}`); expect( verifyCandidateInvocations({ invocationLog: readFileSync(log, "utf8"), diff --git a/test/install-openshell-version-check.test.ts b/test/install-openshell-version-check.test.ts index 5c5015c615..b86a169f71 100644 --- a/test/install-openshell-version-check.test.ts +++ b/test/install-openshell-version-check.test.ts @@ -218,22 +218,24 @@ exit 0`, describe("install-openshell.sh version check", { timeout: 15_000 }, () => { it.runIf(CANDIDATE_RUNTIME_ENABLED)( - "executes every receipt-bound candidate component under the installer lane (#6691)", + "validates the receipt-bound candidate through the installer path (#6691)", () => { const context = `installer:${CANDIDATE_RUNTIME.resolutionId}`; - for (const [role, binary] of Object.entries(CANDIDATE_RUNTIME).filter( - ([role]) => role === "cli" || role === "gateway" || role === "sandbox", - )) { - const result = spawnSync(binary!, ["--version"], { - encoding: "utf8", - env: { - ...process.env, - NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT: context, - }, - }); - expect(result.status, `${role}: ${result.stderr}`).toBe(0); - expect(result.stdout).toContain(CANDIDATE_RUNTIME.version); - } + const result = spawnSync("bash", [SCRIPT], { + encoding: "utf8", + env: { + ...process.env, + NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT: context, + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + NEMOCLAW_OPENSHELL_GATEWAY_BIN: CANDIDATE_RUNTIME.gateway, + NEMOCLAW_OPENSHELL_MAX_VERSION: CANDIDATE_RUNTIME.version, + NEMOCLAW_OPENSHELL_MIN_VERSION: CANDIDATE_RUNTIME.version, + NEMOCLAW_OPENSHELL_PIN_VERSION: CANDIDATE_RUNTIME.version, + NEMOCLAW_OPENSHELL_SANDBOX_BIN: CANDIDATE_RUNTIME.sandbox, + }, + }); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain(`openshell already installed: ${CANDIDATE_RUNTIME.version}`); }, ); diff --git a/tools/candidate-compat.mts b/tools/candidate-compat.mts index 693b59c7cd..ef8e93770a 100644 --- a/tools/candidate-compat.mts +++ b/tools/candidate-compat.mts @@ -606,10 +606,25 @@ const ROLE_BINARY: Record = { sandbox: "openshell-sandbox", }; +const INSTALLER_FEATURE_MARKERS = [ + "request-body-credential-rewrite", + "websocket-credential-rewrite", + "allow_all_known_mcp_methods", +] as const; + function shellQuote(value: string): string { return `'${value.replaceAll("'", `'"'"'`)}'`; } +function binaryContains(binary: string, marker: string): boolean { + const result = spawnSync("grep", ["-aFq", "--", marker, binary], { + stdio: "ignore", + }); + if (result.status === 0) return true; + if (result.status === 1) return false; + throw new Error(`failed to inspect ${basename(binary)} for installer feature markers`); +} + function extractCandidateBinary( archive: string, artifactValue: Artifact, @@ -705,11 +720,15 @@ export async function materializeCandidate( const output = run(binary, ["--version"]); const version = verifyObservedVersion(receipt, output); const wrapper = join(binDirectory, ROLE_BINARY[artifactValue.role]); + const installerFeatureMarkers = INSTALLER_FEATURE_MARKERS.filter((marker) => + binaryContains(binary, marker), + ); await writeFile( wrapper, [ "#!/bin/sh", "set -eu", + ...installerFeatureMarkers.map((marker) => `# receipt-bound feature: ${marker}`), ': "${NEMOCLAW_CANDIDATE_INVOCATION_LOG:?candidate invocation log is required}"', ': "${NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT:?candidate invocation context is required}"', `printf '%s\\t%s\\t%s\\n' ${shellQuote(artifactValue.role)} "$NEMOCLAW_CANDIDATE_INVOCATION_CONTEXT" "$*" >> "$NEMOCLAW_CANDIDATE_INVOCATION_LOG"`,