Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ All notable changes to Kova are documented in this file.

### Changed

- Let the Gateway session benchmark record the first streamed answer delta when requested.

- Report total Gateway process-tree RSS 120 seconds after cold-start readiness on Linux.

- Remove the always-empty `commandReceiptLocks` field from repeated-work inventory output.
Expand Down
5 changes: 5 additions & 0 deletions docs/SCENARIO_HIERARCHY.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ If another turn causes their input to queue and the Gateway omits an explicit
reply association, they now report `BLOCKED` instead of accepting an inferred
queued reply. Direct callers of the support script have the same behavior.

Direct callers can pass `--observe-answer-delta true` against a Gateway that
supports session message subscriptions. The JSON report then includes
`firstAnswerDeltaMs`, measured from send to the first matching streamed answer
text on the submitted run. A `null` value means no matching delta was observed.

This scenario owns:

- cold and warm Gateway session turn timing
Expand Down
1 change: 1 addition & 0 deletions src/evaluation/agent-turns.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ function extractGatewaySessionTurn(result) {
sessionCreateDurationMs: numberOrNull(payload.sessionCreateDurationMs),
sendDurationMs: numberOrNull(payload.sendDurationMs),
timeToFirstAssistantMs: numberOrNull(payload.timeToFirstAssistantMs),
...(Object.hasOwn(payload, "firstAnswerDeltaMs") ? { firstAnswerDeltaMs: numberOrNull(payload.firstAnswerDeltaMs) } : {}),
timeToMatchedAssistantMs: numberOrNull(payload.timeToMatchedAssistantMs),
assistantMessageCount: numberOrNull(payload.assistantMessageCount),
historyPollCount: numberOrNull(payload.historyPollCount),
Expand Down
14 changes: 10 additions & 4 deletions support/openclaw-runtime.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ export function runOcmJson(args, { env } = {}) {
}
}

export async function openDirectGatewayRpcClient(runtimeContext, { env = process.env } = {}) {
export async function openDirectGatewayRpcClient(runtimeContext, { env = process.env, caps, onEvent } = {}) {
if (typeof WebSocket !== "function") {
throw new Error("direct Gateway RPC requires WebSocket support");
}
Expand All @@ -129,7 +129,9 @@ export async function openDirectGatewayRpcClient(runtimeContext, { env = process

const client = new DirectGatewayRpcClient({
url: gateway.url,
token
token,
caps,
onEvent
});
await client.connect();
return {
Expand Down Expand Up @@ -163,9 +165,11 @@ export async function waitForGatewayMethodOk(client, method, {
}

export class DirectGatewayRpcClient {
constructor({ url, token }) {
constructor({ url, token, caps = [], onEvent }) {
this.url = url;
this.token = token;
this.caps = caps;
this.onEvent = onEvent;
this.ws = null;
this.pending = new Map();
this.connectStarted = false;
Expand Down Expand Up @@ -249,7 +253,7 @@ export class DirectGatewayRpcClient {
mode: GATEWAY_RPC_CLIENT_MODE,
instanceId: `kova-${randomUUID()}`
},
caps: [],
caps: this.caps,
role: "operator",
scopes: GATEWAY_OPERATOR_SCOPES,
auth: {
Expand Down Expand Up @@ -281,9 +285,11 @@ export class DirectGatewayRpcClient {
}

async handleMessage(event) {
const receivedAtMs = this.onEvent && performance.now();
const raw = await readWebSocketData(event.data);
const frame = JSON.parse(raw);
if (frame?.type !== "res") {
if (frame?.type === "event") this.onEvent?.(frame, receivedAtMs);
return frame;
}
const pending = this.pending.get(frame.id);
Expand Down
32 changes: 30 additions & 2 deletions support/run-gateway-session-send-turn.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,32 @@ class RunAssociationUnavailableError extends Error {}

try {
const args = parseSupportArgs(process.argv.slice(2));
rejectUnsupportedArgs(args, ["env", "message", "expected-text", "timeout", "session-key", "create-session", "min-assistant-count"]);
rejectUnsupportedArgs(args, ["env", "message", "expected-text", "timeout", "session-key", "create-session", "min-assistant-count", "observe-answer-delta"]);
const runtimeContext = prepareOpenClawRuntimeFromOcmEnv(args.env);
const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only.";
const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK";
const expectedAnswerText = expectedText.trim();
const timeoutMs = readTimeoutMs(args.timeout, 120000);
const sessionKey = args["session-key"] ?? `kova-gateway-session-${randomUUID()}`;
const createSession = readBoolean(args["create-session"], true);
const minAssistantCount = readPositiveInteger(args["min-assistant-count"], 1);
const gatewayTransport = await openDirectGatewayRpcClient(runtimeContext);
const observeAnswerDelta = readBoolean(args["observe-answer-delta"], false);
let streamMeasurement;
const gatewayTransport = await openDirectGatewayRpcClient(runtimeContext, observeAnswerDelta ? {
caps: ["chat-queue-status"],
onEvent: ({ event, payload }, receivedAtMs) => {
if (!streamMeasurement || streamMeasurement.firstAnswerDeltaMs !== null ||
event !== "chat" || payload?.state !== "delta" ||
payload.sessionKey !== streamMeasurement.sessionKey || payload.runId !== streamMeasurement.runId) return;
const text = payload.deltaText ?? (payload.message?.role === "assistant"
? extractAssistantVisibleText(payload.message)
: "");
const prefix = typeof text === "string" ? text.trim() : "";
if (prefix && expectedAnswerText.startsWith(prefix)) {
streamMeasurement.firstAnswerDeltaMs = receivedAtMs - streamMeasurement.sendStartedAtMs;
}
}
} : undefined);

try {
let created = null;
Expand All @@ -45,6 +62,16 @@ try {
}
const canonicalKey = created?.key ?? sessionKey;
const idempotencyKey = `kova-gateway-session-${randomUUID()}`;
if (observeAnswerDelta) {
const subscription = await gatewayCall(gatewayTransport, "sessions.messages.subscribe", {
key: canonicalKey
}, Math.min(timeoutMs, 60000));
if (subscription?.subscribed !== true || typeof subscription.key !== "string" || !subscription.key.trim()) {
throw new RunAssociationUnavailableError("Gateway did not confirm the canonical session stream subscription");
}
// Arm before sending, since the first delta can arrive before the send reply.
streamMeasurement = { sessionKey: subscription.key, runId: idempotencyKey, sendStartedAtMs: performance.now(), firstAnswerDeltaMs: null };
}
const sendStartedAtEpochMs = Date.now();
const sent = await gatewayCall(gatewayTransport, "sessions.send", {
key: canonicalKey,
Expand Down Expand Up @@ -122,6 +149,7 @@ try {
assistantFirstSeenAtEpochMs: history.assistantFirstSeenAtEpochMs,
assistantMatchedAtEpochMs: history.assistantMatchedAtEpochMs,
timeToFirstAssistantMs: history.assistantFirstSeenAtEpochMs === null ? null : history.assistantFirstSeenAtEpochMs - sendStartedAtEpochMs,
...(streamMeasurement ? { firstAnswerDeltaMs: streamMeasurement.firstAnswerDeltaMs } : {}),
timeToMatchedAssistantMs: history.assistantMatchedAtEpochMs === null ? null : history.assistantMatchedAtEpochMs - sendStartedAtEpochMs,
historyPollCount: history.pollCount,
historyErrorCount: history.errorCount,
Expand Down
47 changes: 42 additions & 5 deletions tests/gateway-session-send-turn.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ const queuedRunId = "queued-execution";
const foreignRunId = "foreign-execution";

export function runGatewaySessionSendTurnChecks() {
for (const scenario of ["stale-ack", "prefixed-error", "overflow", "early-error", "success", "new-session", "early-reply", "injected", "blank-reply", "wrong-reply", "min-count", "missing-run", "wrong-wait-run", "queued", "queued-paged", "queued-session-change", "queued-retry-success", "queued-retry-error", "collected", "queued-error", "queued-other-input", "queued-forwarded", "queued-cron", "queued-hidden-input", "queued-hidden-completion", "queued-explicit", "queued-submitted-error", "direct-paged", "queued-old-reply", "timeout"]) {
for (const scenario of ["stale-ack", "prefixed-error", "overflow", "early-error", "success", "new-session", "early-reply", "injected", "stream", "stream-legacy", "stream-missing", "blank-reply", "wrong-reply", "min-count", "missing-run", "wrong-wait-run", "queued", "queued-paged", "queued-session-change", "queued-retry-success", "queued-retry-error", "collected", "queued-error", "queued-other-input", "queued-forwarded", "queued-cron", "queued-hidden-input", "queued-hidden-completion", "queued-explicit", "queued-submitted-error", "direct-paged", "queued-old-reply", "timeout"]) {
const observeStream = scenario.startsWith("stream");
const result = spawnSync(process.execPath, [
"--import", fileURLToPath(import.meta.url), helper,
"--env", "fixture", "--session-key", "existing-session",
"--expected-text", scenario === "blank-reply" ? "" : "KOVA_AGENT_OK",
"--expected-text", scenario === "blank-reply" ? "" : scenario === "stream" ? " KOVA_AGENT_OK " : "KOVA_AGENT_OK",
"--create-session", scenario === "new-session" ? "true" : "false",
"--min-assistant-count", scenario === "new-session" || scenario === "direct-paged" || (scenario === "queued-paged" || scenario === "queued-session-change") ? "1" : scenario === "min-count" ? "6" : "2", "--timeout", "1500"
"--min-assistant-count", scenario === "new-session" || scenario === "direct-paged" || (scenario === "queued-paged" || scenario === "queued-session-change") ? "1" : scenario === "min-count" ? "6" : "2", "--timeout", "1500",
...(observeStream ? ["--observe-answer-delta", "true"] : [])
], {
encoding: "utf8", timeout: 10000,
env: {
Expand All @@ -29,7 +31,7 @@ export function runGatewaySessionSendTurnChecks() {
});
assert.ifError(result.error);
const receipt = JSON.parse(result.stdout);
if (["success", "queued-explicit", "queued-retry-success", "direct-paged", "new-session", "early-reply", "injected"].includes(scenario)) {
if (observeStream || ["success", "queued-explicit", "queued-retry-success", "direct-paged", "new-session", "early-reply", "injected"].includes(scenario)) {
assert.equal(result.status, 0, `${scenario}: ${receipt.error}`);
assert.equal(receipt.ok, true);
assert.equal(receipt.recordStatus, "PASS");
Expand All @@ -42,6 +44,14 @@ export function runGatewaySessionSendTurnChecks() {
assert.ok(receipt.assistantFirstSeenAtEpochMs < receipt.activeFinishedAtEpochMs);
assert.ok(receipt.assistantMatchedAtEpochMs < receipt.activeFinishedAtEpochMs);
}
if (observeStream && scenario !== "stream-missing") {
assert.ok(receipt.firstAnswerDeltaMs >= 20, "ignore earlier unrelated frames");
assert.ok(receipt.firstAnswerDeltaMs < receipt.sendDurationMs, "observe a delta received before the send reply");
} else if (observeStream) {
assert.equal(receipt.firstAnswerDeltaMs, null, "a history match never supplies a stream measurement");
} else {
assert.equal(Object.hasOwn(receipt, "firstAnswerDeltaMs"), false);
}
} else if ((scenario.startsWith("queued") && !["queued-retry-error", "queued-submitted-error"].includes(scenario)) || scenario === "collected") {
assert.equal(result.status, 1, scenario);
assert.equal(receipt.ok, false);
Expand Down Expand Up @@ -101,11 +111,38 @@ if (process.env.KOVA_SEND_TURN_FIXTURE) {
role: "assistant", content: [{ type: "text", text }],
__openclaw: { runId }, timestamp: 1
});
if (method === "connect") return queueMicrotask(() => reply({}));
if (method === "connect") {
assert.deepEqual(params.caps, scenario.startsWith("stream") ? ["chat-queue-status"] : []);
return queueMicrotask(() => reply({}));
}
if (method === "sessions.create") return queueMicrotask(() => reply({ key: "canonical-session" }));
if (method === "sessions.messages.subscribe") {
assert.ok(scenario.startsWith("stream"));
assert.equal(params.key, "existing-session");
this.subscribed = true;
return queueMicrotask(() => reply({ subscribed: true, key: "canonical-session" }));
}
if (method === "sessions.send") {
assert.equal(params.key, scenario === "new-session" ? "canonical-session" : "existing-session");
submittedRunId = params.idempotencyKey;
if (scenario.startsWith("stream")) {
assert.equal(this.subscribed, true);
const chat = (payload) => this.frame({ type: "event", event: "chat", payload: {
state: "delta", sessionKey: "canonical-session", runId: submittedRunId, deltaText: "KOVA", ...payload
} });
return queueMicrotask(() => {
chat({ sessionKey: "other-session" });
chat({ runId: foreignRunId });
chat({ state: "final" });
chat({ deltaText: "queued" });
chat({ deltaText: "" });
if (scenario === "stream") setTimeout(() => chat({ deltaText: "KOVA_AGENT_OK\n" }), 40);
if (scenario === "stream-legacy") {
setTimeout(() => chat({ deltaText: undefined, message: assistant(submittedRunId, "KOVA") }), 40);
}
setTimeout(() => reply({ runId: submittedRunId }), 80);
});
}
return queueMicrotask(() => reply(scenario === "missing-run" ? {} : { runId: scenario === "stale-ack" ? "historical-0" : submittedRunId }));
}
if (method === "agent.wait") {
Expand Down
Loading