Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ All notable changes to Kova are documented in this file.

### Fixed

- Keep skipped required proof incomplete across evidence ledgers and JSON or terminal reports while preserving optional skips. Thanks @llbohacll. (#268)

- Preserve the scoped OCM executable during Gateway resource lookup so login-shell PATH changes cannot hide a running Gateway or mark startup coverage incomplete.

- Keep official plugin install proof incomplete when required-plugin or failed-required counts are missing or malformed, instead of defaulting missing failures to zero.
Expand Down
9 changes: 6 additions & 3 deletions docs/REPORT_SCHEMA.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,9 +180,12 @@ Important fields:
- `phases`: commands, results, and metrics by phase

`evidenceLedger.completeness` is `not-evaluated` for dry-run records,
`complete` when required ledger entries are present, and `incomplete` when one
or more required ledger entries are missing. A passing execution record with
missing required ledger evidence is downgraded to `INCOMPLETE`.
`complete` when required ledger entries are present and not skipped, and
`incomplete` when one or more required ledger entries are missing or skipped.
A passing execution record with missing or skipped required ledger evidence is
downgraded to `INCOMPLETE`. Skipped required entries count toward `requiredMissing`
while retaining their original `skipped` status and `byStatus.skipped` count.
Optional skips and whole-record `DRY-RUN`/`SKIPPED` outcomes remain unchanged.

OpenClaw state snapshots use `kova.openclawStateSnapshot.v1`. They summarize
known config, auth/model shape, plugin indexes, and plugin manifests with
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"plan": "node bin/kova.mjs plan",
"test:cpu-accounting": "node --test tests/linux-cpu-accounting.mjs tests/resource-command-accounting.mjs tests/resource-process-attribution.mjs",
"test:ocm-transport": "node tests/ocm-transport.mjs",
"check": "node --test tests/catalogue.mjs tests/persona.mjs tests/readiness.mjs && npm run test:cli-flags && npm run test:cpu-accounting && npm run test:assert-command-output && npm run test:selfcheck-isolation && npm run test:ocm-transport && node bin/kova.mjs self-check",
"check": "node --test tests/catalogue.mjs tests/persona.mjs tests/readiness.mjs tests/report-required-evidence.mjs && npm run test:cli-flags && npm run test:cpu-accounting && npm run test:assert-command-output && npm run test:selfcheck-isolation && npm run test:ocm-transport && node bin/kova.mjs self-check",
"test:cli-flags": "node --test tests/cli-flags.mjs",
"check:full": "npm run check && npm run test:snapshots && npm run check:web-payload && npm run test:release-contracts",
"check:web-payload": "node scripts/check-web-payload.mjs",
Expand Down
19 changes: 12 additions & 7 deletions src/evidence-ledger.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export function applyEvidenceLedgerGating(record) {
return record;
}

const missing = ledger.entries.filter((entry) => entry.required && entry.status === "missing");
const missing = ledger.entries.filter((entry) => entry.required && isUnfulfilledEvidenceStatus(entry.status));
const failed = ledger.entries.filter((entry) => entry.required && entry.status === "failed");
const failedCommands = failed.filter((entry) => entry.category === "command");
const failedInvariants = failed.filter((entry) => entry.category === "invariant");
Expand Down Expand Up @@ -77,8 +77,8 @@ export function buildEvidenceLedger(record) {

return {
schemaVersion: EVIDENCE_LEDGER_SCHEMA,
completeness: record.status === RECORD_STATUS.DRY_RUN ? "not-evaluated" : completenessForEntries(entries),
summary: summarizeEntries(entries),
completeness: record.status === RECORD_STATUS.DRY_RUN ? "not-evaluated" : completenessForEntries(entries, record.status),
summary: summarizeEntries(entries, record.status),
entries
};
}
Expand Down Expand Up @@ -193,10 +193,15 @@ function isBehaviorFailureCategory(category) {
return category === "command" || category === "invariant" || category === "channel-capability";
}

function completenessForEntries(entries) {
export function isUnfulfilledEvidenceStatus(status, recordStatus) {
return status === "missing" || (status === "skipped" &&
recordStatus !== RECORD_STATUS.DRY_RUN && recordStatus !== RECORD_STATUS.SKIPPED);
}

function completenessForEntries(entries, recordStatus) {
return entries.some((entry) =>
entry.required &&
(entry.status === "missing" || (entry.status === "failed" && !isBehaviorFailureCategory(entry.category)))
(isUnfulfilledEvidenceStatus(entry.status, recordStatus) || (entry.status === "failed" && !isBehaviorFailureCategory(entry.category)))
)
? "incomplete"
: "complete";
Expand Down Expand Up @@ -272,7 +277,7 @@ function firstFailedCommandInPhase(phase, results) {
return null;
}

function summarizeEntries(entries) {
function summarizeEntries(entries, recordStatus) {
const byStatus = {};
const byCategory = {};
let required = 0;
Expand All @@ -283,7 +288,7 @@ function summarizeEntries(entries) {
byCategory[entry.category] = (byCategory[entry.category] ?? 0) + 1;
if (entry.required) {
required += 1;
if (entry.status === "missing") {
if (isUnfulfilledEvidenceStatus(entry.status, recordStatus)) {
requiredMissing += 1;
}
if (entry.status === "failed") {
Expand Down
11 changes: 6 additions & 5 deletions src/reporting/summary.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { isUnfulfilledEvidenceStatus } from "../evidence-ledger.mjs";
import { healthTotalFailures } from "../health.mjs";
import { RECORD_STATUS, findingSeverityForStatus } from "../statuses.mjs";
import {
Expand Down Expand Up @@ -254,11 +255,11 @@ function ledgerFindings(record, index, state) {
continue;
}
const status = normalizedEvidenceStatus(record, entry);
if (status === "missing") {
if (isUnfulfilledEvidenceStatus(status, record.status)) {
findings.push(ledgerFinding(record, index, state, entry, {
severity: "incomplete",
kind: "evidence",
actual: "missing proof"
actual: `${status} proof`
}));
} else if (status === "failed" && entry.category === "invariant") {
findings.push(ledgerFinding(record, index, state, entry, {
Expand Down Expand Up @@ -368,7 +369,7 @@ function summarizeProofCompleteness(records) {
}
const requiredEntries = (ledger.entries ?? []).filter((entry) => entry.required);
const normalizedStatuses = requiredEntries.map((entry) => normalizedEvidenceStatus(record, entry));
const requiredMissing = normalizedStatuses.filter((status) => status === "missing").length;
const requiredMissing = normalizedStatuses.filter((status) => isUnfulfilledEvidenceStatus(status, record.status)).length;
const requiredFailed = normalizedStatuses.filter((status) => status === "failed").length;
proof.completeness[requiredMissing > 0 ? "incomplete" : "complete"] = (proof.completeness[requiredMissing > 0 ? "incomplete" : "complete"] ?? 0) + 1;
proof.requiredTotal += requiredEntries.length;
Expand All @@ -390,7 +391,7 @@ function summarizeProofCompleteness(records) {
reason: entry.reason ?? null,
artifactPath: entry.artifactPath ?? null
};
if (status === "missing") {
if (isUnfulfilledEvidenceStatus(status, record.status)) {
proof.missingRequired.push(item);
} else if (status === "failed") {
proof.failedRequired.push(item);
Expand Down Expand Up @@ -436,7 +437,7 @@ function summarizeChannelCapabilityProof(records) {
applyChannelCapabilityCounts(channel, entry);
byChannel.set(channelId, channel);

if (entry.required && (entry.status === "failed" || entry.status === "missing")) {
if (entry.required && (entry.status === "failed" || isUnfulfilledEvidenceStatus(entry.status, record.status))) {
const item = {
scenario: record.scenario ?? "unknown",
state: record.state?.id ?? null,
Expand Down
47 changes: 47 additions & 0 deletions src/selfcheck/evidence-ledger.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,53 @@ export function evidenceLedgerGatingCheck() {
assertEqual(record.evidenceLedger.summary.requiredMissing, 1, "ledger missing count");
assertEqual(record.incompleteEvidence?.[0], "command:post-upgrade:2", "incomplete evidence id");

const skippedEvidence = [
["command", (required) => ({ phases: [{ id: "proof", commands: ["inspect"],
results: [{ status: 0, evidenceStatus: "skipped", evidenceRequired: required }] }] })],
["snapshot", (required) => ({ phases: [{ id: "proof", commands: ["snapshot"],
results: [{ status: 0, evidenceKind: "snapshot", evidenceStatus: "skipped", evidenceRequired: required }] }] })],
["invariant", (required) => ({ evidenceInvariants: [{ id: "proof", status: "skipped", required }] })],
["artifact", (required) => ({ evidenceArtifacts: [{ id: "proof", status: "skipped", required }] })],
["cleanup", (required) => ({ cleanupEvidence: [{ id: "proof", status: "skipped", required }] })],
["channel", (required) => ({ channelCapabilityEvidence: [{ channelId: "fixture", group: "proof",
capabilityId: "receive", status: "skipped", required }] })]
];
for (const [kind, evidence] of skippedEvidence) {
for (const required of [true, false]) {
const skipped = { ...record, status: "PASS", phases: [], ...evidence(required) };
attachEvidenceLedger(skipped);
applyEvidenceLedgerGating(skipped);
assertEqual(skipped.status, required ? "INCOMPLETE" : "PASS", `${kind} skipped requirement gates pass`);
assertEqual(skipped.evidenceLedger.completeness, required ? "incomplete" : "complete", `${kind} skipped completeness`);
assertEqual(skipped.evidenceLedger.summary.requiredMissing, required ? 1 : 0, `${kind} skipped required proof count`);
assertEqual(skipped.evidenceLedger.summary.byStatus.skipped, 1, `${kind} retains skipped observation`);
const summary = buildReportSummary({ records: [skipped], summary: summarizeRecords([skipped]) });
assertEqual(summary.proof.requiredMissing, required ? 1 : 0, `${kind} report missing count`);
assertEqual(summary.proof.completeness[required ? "incomplete" : "complete"], 1, `${kind} report completeness`);
assertEqual(summary.proof.missingRequired.length, required ? 1 : 0, `${kind} report missing obligations`);
assertEqual(summary.decision.ok, !required, `${kind} report decision`);
if (kind === "channel") {
assertEqual(summary.channelCapabilities.skipped, 1, "channel skipped observation count");
assertEqual(summary.channelCapabilities.missing, 0, "channel raw missing count stays distinct");
assertEqual(summary.channelCapabilities.missingRequired.length, required ? 1 : 0, "channel required skipped obligation");
}
if (required) {
assertEqual(summary.proof.missingRequired[0].status, "skipped", `${kind} report preserves skipped observation`);
assertEqual(summary.findings.some((finding) => finding.actual === "skipped proof"), true, `${kind} report explains skipped proof`);
}
}
}
for (const status of ["DRY-RUN", "SKIPPED"]) {
const unexecuted = { ...record, status, ...skippedEvidence[0][1](true) };
attachEvidenceLedger(unexecuted);
applyEvidenceLedgerGating(unexecuted);
assertEqual(unexecuted.status, status, `${status} record remains unexecuted`);
assertEqual(unexecuted.evidenceLedger.summary.requiredMissing, 0, `${status} skips are not missing proof`);
const summary = buildReportSummary({ records: [unexecuted], summary: summarizeRecords([unexecuted]) });
assertEqual(summary.decision.blockingFindingCount, 0, `${status} skips do not create blocking findings`);
assertEqual(summary.proof.requiredMissing, 0, `${status} report has no missing proof`);
}

const failedRecord = {
...record,
status: "PASS",
Expand Down
59 changes: 59 additions & 0 deletions tests/report-required-evidence.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { test } from "node:test";
import { attachEvidenceLedger, applyEvidenceLedgerGating } from "../src/evidence-ledger.mjs";
import { summarizeRecords } from "../src/reporting/report.mjs";

const root = fileURLToPath(new URL("../", import.meta.url));

for (const recordStatus of ["PASS", "DRY-RUN", "SKIPPED"]) for (const required of [true, false]) {
test(`report CLI preserves ${required ? "required" : "optional"} skipped evidence in ${recordStatus}`, () => {
const dir = mkdtempSync(join(tmpdir(), "kova-report-evidence-"));
try {
const report = JSON.parse(readFileSync(new URL("fixtures/reports/pass.json", import.meta.url), "utf8"));
const record = report.records[0];
record.status = recordStatus;
const missing = recordStatus === "PASS" && required;
const completeness = recordStatus === "DRY-RUN" ? "not-evaluated" : missing ? "incomplete" : "complete";
record.phases = [{ id: "proof", commands: ["inspect"], results: [{
status: 0, evidenceStatus: "skipped", evidenceRequired: required
}] }];
record.finalMetrics = {
error: null, service: { gatewayState: "running" },
health: { ok: true, durationMs: 5 }, healthSamples: [{ ok: true, durationMs: 5 }],
healthSummary: { count: 1, failureCount: 0 }
};
attachEvidenceLedger(record);
applyEvidenceLedgerGating(record);
report.records = [record];
report.summary = summarizeRecords(report.records);
const path = join(dir, "report.json");
writeFileSync(path, JSON.stringify(report));
const run = (args) => {
const result = spawnSync(process.execPath, ["bin/kova.mjs", "report", ...args], {
cwd: root, encoding: "utf8", timeout: 30000,
env: { ...process.env, KOVA_HOME: dir, NO_COLOR: "1" }
});
assert.ifError(result.error);
assert.equal(result.status, 0, result.stderr);
return result.stdout;
};
const summary = JSON.parse(run([path, "--json"]));
assert.equal(summary.decision.verdict, missing ? "INCOMPLETE" : recordStatus === "DRY-RUN" ? "DRY-RUN" : "PASS");
assert.equal(summary.decision.blockingFindingCount, missing ? 1 : 0);
assert.equal(summary.proof.requiredMissing, missing ? 1 : 0);
assert.equal(summary.proof.completeness[completeness], 1);
assert.equal(summary.proof.missingRequired.length, missing ? 1 : 0);
if (missing) assert.equal(summary.proof.missingRequired[0].status, "skipped");
const plain = run(["summarize", path, "--plain"]);
assert.match(plain, new RegExp(`Proof: ${completeness}=1; required \\d+, missing ${missing ? 1 : 0}`));
assert.match(plain, new RegExp(`- ${missing ? "INCOMPLETE" : recordStatus}: 1`));
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
}
Loading