Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ All notable changes to Kova are documented in this file.

### Fixed

- Stop dependent scenario phases after a Gateway readiness deadline and preserve the failing checkpoint in the report.

- Keep skipped required proof incomplete across evidence ledgers and JSON or terminal reports while preserving optional skips. Thanks @llbohacll. (#268)

- Preserve the scoped OCM executable during Gateway resource lookup so login-shell PATH changes cannot hide a running Gateway or mark startup coverage incomplete.
Expand Down
4 changes: 3 additions & 1 deletion docs/REPORT_SCHEMA.md
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,9 @@ fields; readers should use the scoped health object directly.

Startup readiness polls `/readyz` and requires both HTTP success and JSON
`ready: true`. A live `/health` response or a Control UI fallback page does not
admit scenario traffic. Post-ready health samples continue to use `/health`.
admit scenario traffic. A product readiness checkpoint that reaches its deadline
without admission ends the scenario at that phase and runs normal teardown.
Post-ready health samples continue to use `/health`.
Runtimes without the readiness contract remain unready until the deadline.

When a threshold policy includes `postReadyHealthP95Ms`, Kova derives zero
Expand Down
8 changes: 8 additions & 0 deletions src/runner.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { attachEvidenceLedger } from "./evidence-ledger.mjs";
import { executeScenarioPhase } from "./run/scenario-phase.mjs";
import { runWithShutdownScope } from "./commands.mjs";
import { commandFailureRecordStatus } from "./command-results.mjs";
import { measuredProductPhase } from "./measurement-contract.mjs";
import {
executeStateLifecycleSteps,
executeStateSetupAfterPhase
Expand Down Expand Up @@ -170,6 +171,13 @@ async function executeScenarioInScope(scenario, context, signal) {
signal
);
record.phases.push(completedPhase);
// A successful service command can precede Gateway admission. Preserve
// the failed readiness checkpoint and let teardown collect its evidence.
if (!scenarioFailed && measuredProductPhase(completedPhase) &&
phase.healthScope === "readiness" && completedPhase.metrics?.readiness?.ready === false) {
record.status = "FAIL";
break;
}

const authSetupPhase = shouldApplyAuthAfterPhase(phase, authPolicy, record)
? await executeAuthPhase(buildAuthSetupPhase(authPolicy, envName, artifactDir), context, envName, artifactDir, authPolicy)
Expand Down
3 changes: 2 additions & 1 deletion tests/startup-resources.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -238,5 +238,6 @@ test('complete startup scenario runs with isolated path initialization', { skip:
encoding: 'utf8', timeout: 30000, env
});
assert.equal(result.status, 0, result.stdout + result.stderr);
assert.match(result.stdout, /# pass 1\b/);
assert.match(result.stdout, /^ok \d+ - complete scenario records independent startup windows across a warm restart$/m);
assert.match(result.stdout, /^ok \d+ - failed readiness preserves the startup failure and skips dependent commands$/m);
});
87 changes: 70 additions & 17 deletions tests/startup-runner.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,23 @@ const { evaluateRecord } = await import("../src/evaluator.mjs");
if (previousHome === undefined) delete process.env.KOVA_HOME;
else process.env.KOVA_HOME = previousHome;

test("complete scenario records independent startup windows across a warm restart", {
skip: process.platform !== "linux", timeout: 30000
}, async () => {
const servicePath = join(root, "service.json");
const birthsPath = join(root, "births.jsonl");
const callsPath = join(root, "calls.jsonl");
const gatewayPath = join(root, "gateway.mjs");
async function createOcmFixture(readyDelayMs) {
const fixtureDir = await mkdtemp(join(root, "ocm-"));
const servicePath = join(fixtureDir, "service.json");
const birthsPath = join(fixtureDir, "births.jsonl");
const callsPath = join(fixtureDir, "calls.jsonl");
const gatewayPath = join(fixtureDir, "gateway.mjs");
await writeFile(gatewayPath, [
"import { createServer } from 'node:http';",
"import { writeFileSync, appendFileSync } from 'node:fs';",
"const born = Date.now(); const readyAt = born + 2400;",
"const born = Date.now(); const readyAt = born + " + readyDelayMs + ";",
"const server = createServer((req, res) => { const ready = Date.now() >= readyAt; res.writeHead(req.url === '/readyz' && !ready ? 503 : 200); res.end(JSON.stringify({ ready })); });",
"server.listen(0, '127.0.0.1', () => {",
"const state = { childPid: process.pid, gatewayPort: server.address().port, running: true, gatewayState: 'running', desiredRunning: true, born, readyAt };",
"writeFileSync(" + JSON.stringify(servicePath) + ", JSON.stringify(state));",
"appendFileSync(" + JSON.stringify(birthsPath) + ", JSON.stringify(state) + '\\n'); });"
].join("\n"));
await writeFile(join(root, "ocm"), [
await writeFile(join(fixtureDir, "ocm"), [
"#!" + process.execPath,
"import { spawn } from 'node:child_process';",
"import { appendFileSync, existsSync, readFileSync, unlinkSync } from 'node:fs';",
Expand All @@ -56,7 +55,26 @@ test("complete scenario records independent startup windows across a warm restar
"/* No log or diagnostic artifacts in this synthetic runtime. */",
"} else { console.error('unexpected fixture OCM command', args); process.exit(2); }"
].join("\n"), { mode: 0o755 });
const env = { PATH: root + delimiter + process.env.PATH, KOVA_OCM_TRANSPORT_JSON: undefined };
const env = { PATH: fixtureDir + delimiter + process.env.PATH, KOVA_OCM_TRANSPORT_JSON: undefined };
const cleanup = async () => {
try {
const births = (await readFile(birthsPath, "utf8")).trim().split("\n").map(JSON.parse);
for (const { childPid } of births) {
try { process.kill(childPid, "SIGTERM"); } catch (error) { if (error.code !== "ESRCH") throw error; }
}
} catch (error) {
if (error.code !== "ENOENT") throw error;
} finally {
await rm(fixtureDir, { recursive: true, force: true });
}
};
return { birthsPath, callsPath, env, cleanup };
}

test("complete scenario records independent startup windows across a warm restart", {
skip: process.platform !== "linux", timeout: 30000
}, async () => {
const { birthsPath, callsPath, env, cleanup } = await createOcmFixture(2400);
const scenario = {
id: "startup-runner", title: "Synthetic warm restart", surface: "gateway", auth: { mode: "none" },
thresholds: { gatewayReadyMs: 5000, warmRestartReadyMs: 5000 },
Expand Down Expand Up @@ -122,12 +140,47 @@ test("complete scenario records independent startup windows across a warm restar
const calls = (await readFile(callsPath, "utf8")).trim().split("\n").map(JSON.parse);
assert.equal(calls.filter(args => args[0] === "env" && args[1] === "destroy").length, 1);
} finally {
try {
const births = (await readFile(birthsPath, "utf8")).trim().split("\n").map(JSON.parse);
for (const { childPid } of births) {
try { process.kill(childPid, "SIGTERM"); } catch (error) { if (error.code !== "ESRCH") throw error; }
}
} catch (error) { if (error.code !== "ENOENT") throw error; }
await rm(root, { recursive: true, force: true });
await cleanup();
}
});

test("failed readiness preserves the startup failure and skips dependent commands", {
skip: process.platform !== "linux", timeout: 15000
}, async () => {
const { callsPath, env, cleanup } = await createOcmFixture(60000);
const scenario = {
id: "startup-unready", title: "Synthetic unready Gateway", surface: "gateway", auth: { mode: "none" },
thresholds: { gatewayReadyMs: 100, gatewayReadyHardTimeoutMs: 200 },
phases: [
{ id: "source-runtime", title: "Source runtime", healthScope: "readiness", measurementScope: "harness",
commands: ["ocm service start {env} --json"] },
{ id: "gateway-start", title: "Start", healthScope: "readiness", measurementScope: "product",
commands: ["ocm service status {env} --json"] },
{ id: "cold-turn", title: "Cold turn", healthScope: "post-ready",
commands: ["ocm @{env} -- unexpected-turn"] }
]
};
try {
const record = await runWithCommandEnv(env, () => executeScenario(scenario, {
runId: "startup-unready", target: "fixture", targetPlan: { kind: "fixture", value: "fixture" },
commandEnv: env, timeoutMs: 10000, readinessIntervalMs: 20,
healthSamples: 1, healthIntervalMs: 0, resourceSampling: false, openclawDiagnostics: false
}));
const sourceRuntime = record.phases.find((phase) => phase.id === "source-runtime");
assert.equal(sourceRuntime.metrics.readiness.ready, false);
const startup = record.phases.find((phase) => phase.id === "gateway-start");
assert.deepEqual(startup.results.map((result) => result.status), [0]);
assert.equal(startup.metrics.readiness.listeningReady, true);
assert.equal(startup.metrics.readiness.ready, false);
assert.equal(startup.metrics.readiness.health.status, 503);
assert.equal(startup.metrics.readiness.classification.state, "unhealthy");
assert.equal(record.phases.some((phase) => phase.id === "cold-turn"), false);
assert.equal(record.status, "FAIL");
assert.equal(record.cleanup, "destroyed");
const calls = (await readFile(callsPath, "utf8")).trim().split("\n").map(JSON.parse);
assert.equal(calls.some((args) => args.includes("unexpected-turn")), false);
assert.equal(calls.filter((args) => args[0] === "env" && args[1] === "destroy").length, 1);
} finally {
await cleanup();
}
});
Loading