diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b3f5436..ca671906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ All notable changes to Kova are documented in this file. ### Fixed +- Keep skipped required proof incomplete across evidence ledgers and JSON or terminal reports while preserving optional skips. Thanks @llbohacll. (#268) + - Preserve the scoped OCM executable during Gateway resource lookup so login-shell PATH changes cannot hide a running Gateway or mark startup coverage incomplete. - Keep official plugin install proof incomplete when required-plugin or failed-required counts are missing or malformed, instead of defaulting missing failures to zero. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 0008d033..b4959a11 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -180,9 +180,12 @@ Important fields: - `phases`: commands, results, and metrics by phase `evidenceLedger.completeness` is `not-evaluated` for dry-run records, -`complete` when required ledger entries are present, and `incomplete` when one -or more required ledger entries are missing. A passing execution record with -missing required ledger evidence is downgraded to `INCOMPLETE`. +`complete` when required ledger entries are present and not skipped, and +`incomplete` when one or more required ledger entries are missing or skipped. +A passing execution record with missing or skipped required ledger evidence is +downgraded to `INCOMPLETE`. Skipped required entries count toward `requiredMissing` +while retaining their original `skipped` status and `byStatus.skipped` count. +Optional skips and whole-record `DRY-RUN`/`SKIPPED` outcomes remain unchanged. OpenClaw state snapshots use `kova.openclawStateSnapshot.v1`. They summarize known config, auth/model shape, plugin indexes, and plugin manifests with diff --git a/package.json b/package.json index b1849533..8a43e3a3 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "plan": "node bin/kova.mjs plan", "test:cpu-accounting": "node --test tests/linux-cpu-accounting.mjs tests/resource-command-accounting.mjs tests/resource-process-attribution.mjs", "test:ocm-transport": "node tests/ocm-transport.mjs", - "check": "node --test tests/catalogue.mjs tests/persona.mjs tests/readiness.mjs && npm run test:cli-flags && npm run test:cpu-accounting && npm run test:assert-command-output && npm run test:selfcheck-isolation && npm run test:ocm-transport && node bin/kova.mjs self-check", + "check": "node --test tests/catalogue.mjs tests/persona.mjs tests/readiness.mjs tests/report-required-evidence.mjs && npm run test:cli-flags && npm run test:cpu-accounting && npm run test:assert-command-output && npm run test:selfcheck-isolation && npm run test:ocm-transport && node bin/kova.mjs self-check", "test:cli-flags": "node --test tests/cli-flags.mjs", "check:full": "npm run check && npm run test:snapshots && npm run check:web-payload && npm run test:release-contracts", "check:web-payload": "node scripts/check-web-payload.mjs", diff --git a/src/evidence-ledger.mjs b/src/evidence-ledger.mjs index 9db29086..b74ac0db 100644 --- a/src/evidence-ledger.mjs +++ b/src/evidence-ledger.mjs @@ -25,7 +25,7 @@ export function applyEvidenceLedgerGating(record) { return record; } - const missing = ledger.entries.filter((entry) => entry.required && entry.status === "missing"); + const missing = ledger.entries.filter((entry) => entry.required && isUnfulfilledEvidenceStatus(entry.status)); const failed = ledger.entries.filter((entry) => entry.required && entry.status === "failed"); const failedCommands = failed.filter((entry) => entry.category === "command"); const failedInvariants = failed.filter((entry) => entry.category === "invariant"); @@ -77,8 +77,8 @@ export function buildEvidenceLedger(record) { return { schemaVersion: EVIDENCE_LEDGER_SCHEMA, - completeness: record.status === RECORD_STATUS.DRY_RUN ? "not-evaluated" : completenessForEntries(entries), - summary: summarizeEntries(entries), + completeness: record.status === RECORD_STATUS.DRY_RUN ? "not-evaluated" : completenessForEntries(entries, record.status), + summary: summarizeEntries(entries, record.status), entries }; } @@ -193,10 +193,15 @@ function isBehaviorFailureCategory(category) { return category === "command" || category === "invariant" || category === "channel-capability"; } -function completenessForEntries(entries) { +export function isUnfulfilledEvidenceStatus(status, recordStatus) { + return status === "missing" || (status === "skipped" && + recordStatus !== RECORD_STATUS.DRY_RUN && recordStatus !== RECORD_STATUS.SKIPPED); +} + +function completenessForEntries(entries, recordStatus) { return entries.some((entry) => entry.required && - (entry.status === "missing" || (entry.status === "failed" && !isBehaviorFailureCategory(entry.category))) + (isUnfulfilledEvidenceStatus(entry.status, recordStatus) || (entry.status === "failed" && !isBehaviorFailureCategory(entry.category))) ) ? "incomplete" : "complete"; @@ -272,7 +277,7 @@ function firstFailedCommandInPhase(phase, results) { return null; } -function summarizeEntries(entries) { +function summarizeEntries(entries, recordStatus) { const byStatus = {}; const byCategory = {}; let required = 0; @@ -283,7 +288,7 @@ function summarizeEntries(entries) { byCategory[entry.category] = (byCategory[entry.category] ?? 0) + 1; if (entry.required) { required += 1; - if (entry.status === "missing") { + if (isUnfulfilledEvidenceStatus(entry.status, recordStatus)) { requiredMissing += 1; } if (entry.status === "failed") { diff --git a/src/reporting/summary.mjs b/src/reporting/summary.mjs index f7995673..8fadbe05 100644 --- a/src/reporting/summary.mjs +++ b/src/reporting/summary.mjs @@ -1,3 +1,4 @@ +import { isUnfulfilledEvidenceStatus } from "../evidence-ledger.mjs"; import { healthTotalFailures } from "../health.mjs"; import { RECORD_STATUS, findingSeverityForStatus } from "../statuses.mjs"; import { @@ -254,11 +255,11 @@ function ledgerFindings(record, index, state) { continue; } const status = normalizedEvidenceStatus(record, entry); - if (status === "missing") { + if (isUnfulfilledEvidenceStatus(status, record.status)) { findings.push(ledgerFinding(record, index, state, entry, { severity: "incomplete", kind: "evidence", - actual: "missing proof" + actual: `${status} proof` })); } else if (status === "failed" && entry.category === "invariant") { findings.push(ledgerFinding(record, index, state, entry, { @@ -368,7 +369,7 @@ function summarizeProofCompleteness(records) { } const requiredEntries = (ledger.entries ?? []).filter((entry) => entry.required); const normalizedStatuses = requiredEntries.map((entry) => normalizedEvidenceStatus(record, entry)); - const requiredMissing = normalizedStatuses.filter((status) => status === "missing").length; + const requiredMissing = normalizedStatuses.filter((status) => isUnfulfilledEvidenceStatus(status, record.status)).length; const requiredFailed = normalizedStatuses.filter((status) => status === "failed").length; proof.completeness[requiredMissing > 0 ? "incomplete" : "complete"] = (proof.completeness[requiredMissing > 0 ? "incomplete" : "complete"] ?? 0) + 1; proof.requiredTotal += requiredEntries.length; @@ -390,7 +391,7 @@ function summarizeProofCompleteness(records) { reason: entry.reason ?? null, artifactPath: entry.artifactPath ?? null }; - if (status === "missing") { + if (isUnfulfilledEvidenceStatus(status, record.status)) { proof.missingRequired.push(item); } else if (status === "failed") { proof.failedRequired.push(item); @@ -436,7 +437,7 @@ function summarizeChannelCapabilityProof(records) { applyChannelCapabilityCounts(channel, entry); byChannel.set(channelId, channel); - if (entry.required && (entry.status === "failed" || entry.status === "missing")) { + if (entry.required && (entry.status === "failed" || isUnfulfilledEvidenceStatus(entry.status, record.status))) { const item = { scenario: record.scenario ?? "unknown", state: record.state?.id ?? null, diff --git a/src/selfcheck/evidence-ledger.mjs b/src/selfcheck/evidence-ledger.mjs index bf709e56..eece2570 100644 --- a/src/selfcheck/evidence-ledger.mjs +++ b/src/selfcheck/evidence-ledger.mjs @@ -46,6 +46,53 @@ export function evidenceLedgerGatingCheck() { assertEqual(record.evidenceLedger.summary.requiredMissing, 1, "ledger missing count"); assertEqual(record.incompleteEvidence?.[0], "command:post-upgrade:2", "incomplete evidence id"); + const skippedEvidence = [ + ["command", (required) => ({ phases: [{ id: "proof", commands: ["inspect"], + results: [{ status: 0, evidenceStatus: "skipped", evidenceRequired: required }] }] })], + ["snapshot", (required) => ({ phases: [{ id: "proof", commands: ["snapshot"], + results: [{ status: 0, evidenceKind: "snapshot", evidenceStatus: "skipped", evidenceRequired: required }] }] })], + ["invariant", (required) => ({ evidenceInvariants: [{ id: "proof", status: "skipped", required }] })], + ["artifact", (required) => ({ evidenceArtifacts: [{ id: "proof", status: "skipped", required }] })], + ["cleanup", (required) => ({ cleanupEvidence: [{ id: "proof", status: "skipped", required }] })], + ["channel", (required) => ({ channelCapabilityEvidence: [{ channelId: "fixture", group: "proof", + capabilityId: "receive", status: "skipped", required }] })] + ]; + for (const [kind, evidence] of skippedEvidence) { + for (const required of [true, false]) { + const skipped = { ...record, status: "PASS", phases: [], ...evidence(required) }; + attachEvidenceLedger(skipped); + applyEvidenceLedgerGating(skipped); + assertEqual(skipped.status, required ? "INCOMPLETE" : "PASS", `${kind} skipped requirement gates pass`); + assertEqual(skipped.evidenceLedger.completeness, required ? "incomplete" : "complete", `${kind} skipped completeness`); + assertEqual(skipped.evidenceLedger.summary.requiredMissing, required ? 1 : 0, `${kind} skipped required proof count`); + assertEqual(skipped.evidenceLedger.summary.byStatus.skipped, 1, `${kind} retains skipped observation`); + const summary = buildReportSummary({ records: [skipped], summary: summarizeRecords([skipped]) }); + assertEqual(summary.proof.requiredMissing, required ? 1 : 0, `${kind} report missing count`); + assertEqual(summary.proof.completeness[required ? "incomplete" : "complete"], 1, `${kind} report completeness`); + assertEqual(summary.proof.missingRequired.length, required ? 1 : 0, `${kind} report missing obligations`); + assertEqual(summary.decision.ok, !required, `${kind} report decision`); + if (kind === "channel") { + assertEqual(summary.channelCapabilities.skipped, 1, "channel skipped observation count"); + assertEqual(summary.channelCapabilities.missing, 0, "channel raw missing count stays distinct"); + assertEqual(summary.channelCapabilities.missingRequired.length, required ? 1 : 0, "channel required skipped obligation"); + } + if (required) { + assertEqual(summary.proof.missingRequired[0].status, "skipped", `${kind} report preserves skipped observation`); + assertEqual(summary.findings.some((finding) => finding.actual === "skipped proof"), true, `${kind} report explains skipped proof`); + } + } + } + for (const status of ["DRY-RUN", "SKIPPED"]) { + const unexecuted = { ...record, status, ...skippedEvidence[0][1](true) }; + attachEvidenceLedger(unexecuted); + applyEvidenceLedgerGating(unexecuted); + assertEqual(unexecuted.status, status, `${status} record remains unexecuted`); + assertEqual(unexecuted.evidenceLedger.summary.requiredMissing, 0, `${status} skips are not missing proof`); + const summary = buildReportSummary({ records: [unexecuted], summary: summarizeRecords([unexecuted]) }); + assertEqual(summary.decision.blockingFindingCount, 0, `${status} skips do not create blocking findings`); + assertEqual(summary.proof.requiredMissing, 0, `${status} report has no missing proof`); + } + const failedRecord = { ...record, status: "PASS", diff --git a/tests/report-required-evidence.mjs b/tests/report-required-evidence.mjs new file mode 100644 index 00000000..6f58c0a6 --- /dev/null +++ b/tests/report-required-evidence.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; +import { attachEvidenceLedger, applyEvidenceLedgerGating } from "../src/evidence-ledger.mjs"; +import { summarizeRecords } from "../src/reporting/report.mjs"; + +const root = fileURLToPath(new URL("../", import.meta.url)); + +for (const recordStatus of ["PASS", "DRY-RUN", "SKIPPED"]) for (const required of [true, false]) { + test(`report CLI preserves ${required ? "required" : "optional"} skipped evidence in ${recordStatus}`, () => { + const dir = mkdtempSync(join(tmpdir(), "kova-report-evidence-")); + try { + const report = JSON.parse(readFileSync(new URL("fixtures/reports/pass.json", import.meta.url), "utf8")); + const record = report.records[0]; + record.status = recordStatus; + const missing = recordStatus === "PASS" && required; + const completeness = recordStatus === "DRY-RUN" ? "not-evaluated" : missing ? "incomplete" : "complete"; + record.phases = [{ id: "proof", commands: ["inspect"], results: [{ + status: 0, evidenceStatus: "skipped", evidenceRequired: required + }] }]; + record.finalMetrics = { + error: null, service: { gatewayState: "running" }, + health: { ok: true, durationMs: 5 }, healthSamples: [{ ok: true, durationMs: 5 }], + healthSummary: { count: 1, failureCount: 0 } + }; + attachEvidenceLedger(record); + applyEvidenceLedgerGating(record); + report.records = [record]; + report.summary = summarizeRecords(report.records); + const path = join(dir, "report.json"); + writeFileSync(path, JSON.stringify(report)); + const run = (args) => { + const result = spawnSync(process.execPath, ["bin/kova.mjs", "report", ...args], { + cwd: root, encoding: "utf8", timeout: 30000, + env: { ...process.env, KOVA_HOME: dir, NO_COLOR: "1" } + }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); + return result.stdout; + }; + const summary = JSON.parse(run([path, "--json"])); + assert.equal(summary.decision.verdict, missing ? "INCOMPLETE" : recordStatus === "DRY-RUN" ? "DRY-RUN" : "PASS"); + assert.equal(summary.decision.blockingFindingCount, missing ? 1 : 0); + assert.equal(summary.proof.requiredMissing, missing ? 1 : 0); + assert.equal(summary.proof.completeness[completeness], 1); + assert.equal(summary.proof.missingRequired.length, missing ? 1 : 0); + if (missing) assert.equal(summary.proof.missingRequired[0].status, "skipped"); + const plain = run(["summarize", path, "--plain"]); + assert.match(plain, new RegExp(`Proof: ${completeness}=1; required \\d+, missing ${missing ? 1 : 0}`)); + assert.match(plain, new RegExp(`- ${missing ? "INCOMPLETE" : recordStatus}: 1`)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}