diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 220fff4b..da4305ba 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -5,7 +5,8 @@ export type { KernelBrowser } from "./translator/translator"; export { InternalComputerTranslator } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; export { BrowserExecutor } from "./translator/browser"; -export type { BrowserFindCandidate, BrowserRefState } from "./translator/browser"; +export type { BrowserFindCandidate } from "./translator/browser"; +export type { BrowserRefState } from "./translator/browser-ref-lifecycle"; export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { diff --git a/packages/agent/src/translator/browser-document-reconciliation.ts b/packages/agent/src/translator/browser-document-reconciliation.ts new file mode 100644 index 00000000..6778df43 --- /dev/null +++ b/packages/agent/src/translator/browser-document-reconciliation.ts @@ -0,0 +1,162 @@ +import type { CdpConnection } from "./cdp"; +import type { RefGenerationLifecycle } from "./browser-ref-lifecycle"; + +/** How long to wait for a pending OOPIF session before treating the frame as gone. */ +const FRAME_ATTACH_WAIT_MS = 500; +/** Backoff while an attached OOPIF's Page domain becomes ready. */ +const FRAME_TREE_RETRY_MS = 20; + +/** Minimal shape returned by CDP's `Page.getFrameTree`. */ +export interface FrameTreeNode { + frame?: { id?: string; loaderId?: string }; + childFrames?: FrameTreeNode[]; +} + +/** + * Captures and reconciles durable loader identities for ref-owning documents. + * Main and same-process frames are read from the page frame tree; OOPIFs are + * read through their own auto-attached sessions. + */ +export class BrowserDocumentReconciler { + private readonly frameSessionWaiters = new Map void>>(); + + constructor( + private readonly cdp: CdpConnection, + private readonly lifecycle: RefGenerationLifecycle, + private readonly frameSessions: ReadonlyMap, + ) {} + + /** Wake document reconciliation when an OOPIF session auto-attaches. */ + frameSessionAttached(frameKey: string): void { + const waiters = this.frameSessionWaiters.get(frameKey); + if (waiters) for (const wake of [...waiters]) wake(); + } + + /** Read the page frame tree and record its main-frame document identity. */ + async capturePage(targetId: string, pageSession: string): Promise { + const pageTree = await this.pageFrameTree(pageSession); + const loaderId = pageTree?.frame?.loaderId; + if (loaderId !== undefined) this.lifecycle.recordDocument(targetId, targetId, loaderId); + return pageTree; + } + + /** Record a child frame's identity at ref-mint time. */ + async captureFrame( + frameId: string, + targetId: string, + pageSession: string, + pageTree: FrameTreeNode | undefined, + ): Promise { + const frameSession = this.frameSessions.get(frameId); + const loaderId = frameSession + ? (await this.pageFrameTree(frameSession))?.frame?.loaderId + : frameLoaderIn(pageTree, frameId); + if (loaderId !== undefined) this.lifecycle.recordDocument(frameId, targetId, loaderId); + } + + /** + * Reconcile every imported ref-owning frame before any of its refs resolve. + * Changed or unverifiable children stale selectively; a changed main frame + * invalidates the target. Each pending expectation is consumed once. + */ + async reconcile(targetId: string, pageSession: string): Promise { + if (!this.lifecycle.hasPendingDocument(targetId)) return; + const verifiable: string[] = []; + for (const { frameKey, verifiable: ok } of this.lifecycle.pendingDocuments(targetId)) { + if (ok) verifiable.push(frameKey); + else this.lifecycle.reconcileDocument(frameKey, targetId, undefined); + } + if (verifiable.length === 0) return; + + const pageTree = await this.pageFrameTree(pageSession); + const oopifCandidates: string[] = []; + for (const frameKey of verifiable) { + if (frameKey === targetId) { + this.lifecycle.reconcileDocument(frameKey, targetId, pageTree?.frame?.loaderId); + continue; + } + const attached = this.frameSessions.get(frameKey); + if (attached) { + this.lifecycle.reconcileDocument(frameKey, targetId, await this.frameSessionLoaderId(attached)); + continue; + } + const sameProcess = frameLoaderIn(pageTree, frameKey); + if (sameProcess !== undefined) this.lifecycle.reconcileDocument(frameKey, targetId, sameProcess); + else oopifCandidates.push(frameKey); + } + + for (const frameKey of oopifCandidates) { + const frameSession = await this.waitForFrameSession(frameKey); + const loaderId = frameSession ? await this.frameSessionLoaderId(frameSession) : undefined; + this.lifecycle.reconcileDocument(frameKey, targetId, loaderId); + } + } + + private async frameSessionLoaderId(session: string): Promise { + for (let attempt = 0; ; attempt += 1) { + const loaderId = (await this.pageFrameTree(session))?.frame?.loaderId; + if (loaderId !== undefined || attempt === 2) return loaderId; + await delay(FRAME_TREE_RETRY_MS); + } + } + + private async pageFrameTree(session: string): Promise { + try { + const { frameTree } = await this.cdp.send<{ frameTree?: FrameTreeNode }>("Page.getFrameTree", {}, session); + return frameTree; + } catch { + return undefined; + } + } + + private async waitForFrameSession(frameKey: string): Promise { + const existing = this.frameSessions.get(frameKey); + if (existing) return existing; + return new Promise((resolve) => { + const wake = (): void => { + clearTimeout(timer); + const waiters = this.frameSessionWaiters.get(frameKey); + if (waiters) { + const index = waiters.indexOf(wake); + if (index >= 0) waiters.splice(index, 1); + if (waiters.length === 0) this.frameSessionWaiters.delete(frameKey); + } + resolve(this.frameSessions.get(frameKey)); + }; + const timer = setTimeout(wake, FRAME_ATTACH_WAIT_MS); + const waiters = this.frameSessionWaiters.get(frameKey) ?? []; + waiters.push(wake); + this.frameSessionWaiters.set(frameKey, waiters); + }); + } +} + +/** Translate an OOPIF-local point into the page viewport coordinate space. */ +export async function oopifFrameOffset( + cdp: CdpConnection, + frameId: string, + pageSession: string, +): Promise<{ x: number; y: number }> { + const { backendNodeId } = await cdp.send<{ backendNodeId: number }>("DOM.getFrameOwner", { frameId }, pageSession); + const { model } = await cdp.send<{ model: { content: number[] } }>( + "DOM.getBoxModel", + { backendNodeId }, + pageSession, + ); + return { x: model.content[0]!, y: model.content[1]! }; +} + +/** Find a same-process frame's loader identity in a page frame tree. */ +function frameLoaderIn(tree: FrameTreeNode | undefined, frameId: string): string | undefined { + if (!tree) return undefined; + if (tree.frame?.id === frameId) return tree.frame.loaderId; + for (const child of tree.childFrames ?? []) { + const found = frameLoaderIn(child, frameId); + if (found !== undefined) return found; + } + return undefined; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/packages/agent/src/translator/browser-frame-collection.ts b/packages/agent/src/translator/browser-frame-collection.ts new file mode 100644 index 00000000..620d1504 --- /dev/null +++ b/packages/agent/src/translator/browser-frame-collection.ts @@ -0,0 +1,38 @@ +import { CdpProtocolError } from "./cdp"; + +/** Unexpected iframe collection failure with frame and collection-stage context. */ +export class FrameCollectionError extends Error { + constructor(message: string, cause: unknown) { + super(message, { cause }); + this.name = "FrameCollectionError"; + } +} + +/** Whether a CDP error is a known transient frame-disappearance race. */ +export function isExpectedFrameCollectionError( + error: unknown, + method: "DOM.describeNode" | "Accessibility.getFullAXTree", +): error is CdpProtocolError { + if (!(error instanceof CdpProtocolError) || error.method !== method) return false; + const message = error.protocolMessage.trim(); + if (method === "DOM.describeNode") { + return /^(?:Could not find node with given id|No node with given id found)\.?$/i.test(message); + } + return /^(?:Frame with the given id was not found|No frame for given id found|Session with given id not found|Target session terminated)\.?$/i.test( + message, + ); +} + +/** Wrap an unexpected iframe collection failure with actionable context. */ +export function frameCollectionError( + backendNodeId: number, + frameId: string | undefined, + stage: string, + cause: unknown, +): FrameCollectionError { + const detail = cause instanceof Error ? cause.message : String(cause); + return new FrameCollectionError( + `Failed to collect iframe ${frameId ?? "with unknown frame id"} at backend node ${backendNodeId} during ${stage}: ${detail}`, + cause, + ); +} diff --git a/packages/agent/src/translator/browser-observation.ts b/packages/agent/src/translator/browser-observation.ts new file mode 100644 index 00000000..7ac9bc2f --- /dev/null +++ b/packages/agent/src/translator/browser-observation.ts @@ -0,0 +1,243 @@ +/** Minimal CDP accessibility node used by browser observations. */ +export interface AXNode { + readonly nodeId: string; + readonly ignored?: boolean; + readonly role?: { readonly value?: string }; + readonly name?: { readonly value?: string }; + readonly value?: { readonly value?: unknown }; + readonly properties?: readonly { readonly name: string; readonly value?: { readonly value?: unknown } }[]; + readonly backendDOMNodeId?: number; + readonly parentId?: string; + readonly childIds?: readonly string[]; +} + +/** Role/name cohort positions used when minting and healing refs. */ +export interface NthIndex { + readonly index: ReadonlyMap; + readonly cohorts: ReadonlyMap; +} + +/** Immutable frame and generation metadata required to mint a ref. */ +export interface RenderContext { + readonly targetId: string; + readonly frameKey: string; + readonly sessionId: string; + readonly generation: number; + readonly nthIndex: NthIndex; + readonly cursorIds?: ReadonlySet; +} + +/** One normalized accessibility line before its ref is minted. */ +export interface ObservationLine { + readonly text: string; + readonly refNode?: AXNode; + readonly ctx: RenderContext; +} + +/** Accessibility tree collected for one frame. */ +export interface FrameStitch { + readonly byId: ReadonlyMap; + readonly roots: readonly string[]; + readonly ctx: RenderContext; +} + +/** A child frame that could not be collected because it detached or became inaccessible. */ +export interface IncompleteFrame { + readonly backendNodeId: number; + readonly frameId?: string; + readonly stage: "describe" | "resolve" | "accessibility"; + readonly reason: string; +} + +/** Accessibility node paired with its frame context. */ +export interface ObservedNode { + readonly node: AXNode; + readonly ctx: RenderContext; +} + +/** Stable structured browser state collected before presentation filtering. */ +export interface BrowserObservation { + readonly targetId: string; + readonly tree: FrameStitch; + readonly stitches: ReadonlyMap; + readonly incompleteFrames: readonly IncompleteFrame[]; + /** Target topology revision used only for observation/cache fencing, not ref validity. */ + readonly revision: number; + readonly generations: ReadonlyMap; +} + +/** Render-ready projection of one structured browser observation. */ +export interface BrowserPresentation { + readonly observation: BrowserObservation; + readonly cacheKey: string; + readonly lines: readonly ObservationLine[]; + readonly shape: string; +} + +/** Signals that browser state changed while an observation was collected. */ +export class ObservationChangedError extends Error { + constructor(message = "Browser observation changed during collection") { + super(message); + this.name = "ObservationChangedError"; + } +} + +/** Signals that a scoped ref could not be verified in an incompletely collected frame. */ +export class IncompleteObservationError extends Error { + constructor(message: string) { + super(message); + this.name = "IncompleteObservationError"; + } +} + +export const REF_PLACEHOLDER = "\u0000"; + +export const INTERACTIVE_ROLES: ReadonlySet = new Set([ + "button", + "link", + "textbox", + "searchbox", + "checkbox", + "radio", + "combobox", + "listbox", + "option", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "slider", + "spinbutton", + "switch", + "tab", + "treeitem", +]); + +export const FRAME_ROLES: ReadonlySet = new Set(["Iframe", "IframePresentational"]); + +const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); + +/** Non-interactive roles that get refs when named, so scroll_to / ref-scoped snapshots can target them. */ +const CONTENT_ROLES: ReadonlySet = new Set([ + "heading", + "cell", + "gridcell", + "columnheader", + "rowheader", + "row", + "listitem", + "article", + "region", + "main", + "navigation", + "banner", + "contentinfo", + "complementary", + "tabpanel", + "figure", + "image", +]); + +/** Index each ref-healing candidate by its position among nodes with the same role and name, in tree order. */ +export function buildNthIndex(nodes: readonly AXNode[]): NthIndex { + const cohorts = new Map(); + const index = new Map(); + for (const node of nodes) { + if (node.ignored || node.backendDOMNodeId === undefined) continue; + const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); + const nth = cohorts.get(key) ?? 0; + cohorts.set(key, nth + 1); + index.set(node.nodeId, nth); + } + return { index, cohorts }; +} + +/** Build the stable key for a role/name ref-healing cohort. */ +export function cohortKey(role: string, name: string): string { + return `${role}\u0000${name}`; +} + +/** Iterate an observation without eagerly allocating a wrapper for every AX node. */ +export function* observedNodes(observation: BrowserObservation): IterableIterator { + for (const node of observation.tree.byId.values()) yield { node, ctx: observation.tree.ctx }; + for (const stitch of observation.stitches.values()) { + for (const node of stitch.byId.values()) yield { node, ctx: stitch.ctx }; + } +} + +/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ +export function staticTextRun( + tree: ReadonlyMap, + childIds: readonly string[], + start: number, +): { node: AXNode; end: number } | undefined { + let end = start; + const parts: string[] = []; + while (end < childIds.length) { + const node = tree.get(childIds[end]!); + if (!node || node.ignored || node.role?.value !== "StaticText") break; + const text = node.name?.value ?? ""; + if (text) parts.push(text); + end += 1; + } + if (end - start < 2) return undefined; + const first = tree.get(childIds[start]!)!; + return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; +} + +export function renderObservationNode( + node: AXNode, + depth: number, + parentName: string, + ctx: RenderContext, + interactiveOnly: boolean, +): { text: string; refNode?: AXNode } | undefined { + const role = node.role?.value ?? ""; + const name = node.name?.value ?? ""; + const interactive = INTERACTIVE_ROLES.has(role); + const pointer = node.backendDOMNodeId !== undefined && (ctx.cursorIds?.has(node.backendDOMNodeId) ?? false); + if (interactiveOnly && !interactive && !pointer) return undefined; + if (role === "StaticText" && name === parentName) return undefined; + if (!interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; + let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; + let refNode: AXNode | undefined; + const refWorthy = interactive || pointer || FRAME_ROLES.has(role) || (name !== "" && CONTENT_ROLES.has(role)); + if (node.backendDOMNodeId !== undefined && refWorthy) { + line += ` [${REF_PLACEHOLDER}]`; + refNode = node; + } + const states = collectStates(node); + if (pointer && !interactive) states.push("cursor:pointer"); + if (states.length > 0) line += ` [${states.join(", ")}]`; + return { text: line, refNode }; +} + +function collectStates(node: AXNode): string[] { + const states: string[] = []; + for (const property of node.properties ?? []) { + const value = property.value?.value; + switch (property.name) { + case "checked": + case "pressed": + case "expanded": + // False is meaningful here: it distinguishes an unchecked checkbox or + // collapsed disclosure from an element without the state at all. + if (value === true || value === "true") states.push(property.name); + else if (value === false || value === "false") states.push(`${property.name}=false`); + else if (value === "mixed") states.push(`${property.name}=mixed`); + break; + case "disabled": + case "selected": + case "required": + if (value === true || value === "true") states.push(property.name); + break; + case "level": + if (typeof value === "number") states.push(`level=${value}`); + break; + } + } + const value = node.value?.value; + if (value !== undefined && value !== "" && String(value) !== (node.name?.value ?? "")) { + states.push(`value=${JSON.stringify(String(value))}`); + } + return states; +} diff --git a/packages/agent/src/translator/browser-ref-lifecycle.ts b/packages/agent/src/translator/browser-ref-lifecycle.ts new file mode 100644 index 00000000..0407b15e --- /dev/null +++ b/packages/agent/src/translator/browser-ref-lifecycle.ts @@ -0,0 +1,437 @@ +import { ObservationChangedError } from "./browser-observation"; + +const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; + +export interface RefEntry { + backendNodeId: number; + targetId: string; + /** Generation key: the owning page target id for main-frame refs, the frame id for iframe refs. */ + frameId: string; + /** Session to route DOM/Input calls through: the frame's own session for OOPIFs, the page session otherwise. */ + sessionId: string; + generation: number; + role: string; + name: string; + nth: number; + /** Size of the (role, name) cohort in the tree the ref was minted from. */ + cohort: number; +} + +interface TargetGenerationState { + readonly targetId: string; + generation: number; + observationRevision: number; + /** + * Durable identity of the main-frame document the current generation's refs + * were minted against: Chrome's main-frame `loaderId`, which changes on every + * cross-document load and is stable across same-document (pushState) ones. + * Unlike {@link generation} (process-local, reset by import) this survives a + * process restart, so a later invocation can tell whether the document a + * ref describes is still the one loaded in the browser. + */ + document?: string; +} + +interface FrameGenerationState { + readonly targetId: string; + readonly frameId: string; + generation: number; + captures: number; + refs: number; + /** + * Durable identity of the document this frame's current generation refs were + * minted against: the frame's own `loaderId` (an OOPIF's from its session, a + * same-process child's from the page frame tree). The per-frame analogue of + * {@link TargetGenerationState.document}; it lets a later process detect that + * *this* frame's document changed even when the main-frame loaderId did not. + */ + document?: string; +} + +/** Serialized ref state format understood by {@link RefGenerationLifecycle.importState}. */ +export const REF_STATE_VERSION = 1; + +export interface GenerationCapture { + readonly targetId: string; + readonly frameKey: string; + readonly generation: number; + readonly targetGeneration: number; + readonly observationRevision: number; + readonly targetState: TargetGenerationState; + readonly frameState?: FrameGenerationState; +} + +/** + * Serializable ref state, so refs minted in one process (e.g. a `cua + * snapshot` invocation) can be resolved in a later one against the same + * browser. Session ids are process-local and deliberately not exported; + * imported refs rebind lazily. Backend node ids stay valid for the life of + * the document, and the usual generation/self-heal machinery covers pages + * that changed in between. + * + * `documents` carries the `loaderId` of every ref-owning frame (keyed exactly + * like a ref's {@link RefEntry.frameId}: the page target id for main-frame + * refs, the child frame id for same-process iframes and OOPIFs) at mint time, + * so the next process can reconcile each frame against the live browser: a + * frame whose document changed (reload/navigation — including a click-induced + * one that raced this process's exit, or an OOPIF-only navigation that left the + * main-frame loaderId untouched) stales only that frame's imported refs instead + * of silently resolving them against a different document whose backend node + * ids may have been reused. + * + * It is optional only for structural back-compat. A ref-owning frame with no + * usable identity here (legacy state predating this field, or a malformed/ + * partial entry) is treated as *unverifiable* and its refs are staled on first + * attach — never resolved by process-local generation alone, which cannot + * detect a cross-process document change and would risk a silent mis-target. + */ +export interface BrowserRefState { + /** Absent or {@link REF_STATE_VERSION} for the current shape; a newer value is rejected on import. */ + version?: number; + refCounter: number; + activeTargetId?: string; + generations: Array<[string, number]>; + documents?: Array<[string, string]>; + refs: Array<[string, Omit]>; +} + +/** + * Owns generation identity and ref retention. Target documents and child + * frames have distinct records even though CDP represents both ids as strings. + * Frame records live only while an observation is collecting or a ref needs + * them, so transient and rotating frames cannot grow retained state. + */ +export class RefGenerationLifecycle { + private readonly targets = new Map(); + private readonly frames = new Map(); + /** Imported, verifiable per-frame loaderIds awaiting reconciliation against the live browser; see {@link reconcileDocument}. */ + private readonly expectedDocuments = new Map(); + /** Owning target of every imported ref-owning frame still awaiting reconciliation (verifiable and unverifiable alike). */ + private readonly documentOwners = new Map(); + /** Imported ref-owning frames whose document identity is missing/malformed; staled on first attach rather than resolved. */ + private readonly unverifiedDocuments = new Set(); + + constructor(private readonly refs: Map) {} + + captureTarget(targetId: string): GenerationCapture { + const targetState = this.ensureTarget(targetId); + return { + targetId, + frameKey: targetId, + generation: targetState.generation, + targetGeneration: targetState.generation, + observationRevision: targetState.observationRevision, + targetState, + }; + } + + captureFrame(targetId: string, frameId: string): GenerationCapture { + const targetState = this.ensureTarget(targetId); + let frameState = this.frames.get(frameId); + if (frameState && frameState.targetId !== targetId) { + throw new Error(`frame ${frameId} changed owner from ${frameState.targetId} to ${targetId}`); + } + frameState ??= { targetId, frameId, generation: 0, captures: 0, refs: 0 }; + this.frames.set(frameId, frameState); + frameState.captures += 1; + return { + targetId, + frameKey: frameId, + generation: frameState.generation, + targetGeneration: targetState.generation, + observationRevision: targetState.observationRevision, + targetState, + frameState, + }; + } + + isCurrent(capture: GenerationCapture): boolean { + if ( + this.targets.get(capture.targetId) !== capture.targetState || + capture.targetState.generation !== capture.targetGeneration || + capture.targetState.observationRevision !== capture.observationRevision + ) { + return false; + } + return ( + !capture.frameState || + (this.frames.get(capture.frameKey) === capture.frameState && capture.frameState.generation === capture.generation) + ); + } + + release(captures: readonly GenerationCapture[]): void { + for (const capture of captures) { + if (!capture.frameState) continue; + capture.frameState.captures = Math.max(0, capture.frameState.captures - 1); + this.deleteUnusedFrame(capture.frameState); + } + } + + retainRef(ref: string, entry: RefEntry): void { + this.deleteRef(ref); + if (entry.frameId === entry.targetId) { + const target = this.targets.get(entry.targetId); + if (!target || target.generation !== entry.generation) throw new ObservationChangedError(); + } else { + const frame = this.frames.get(entry.frameId); + if (!frame || frame.targetId !== entry.targetId || frame.generation !== entry.generation) { + throw new ObservationChangedError(); + } + frame.refs += 1; + } + this.refs.set(ref, entry); + } + + deleteRef(ref: string): void { + const entry = this.refs.get(ref); + if (!entry) return; + this.refs.delete(ref); + if (entry.frameId === entry.targetId) return; + const frame = this.frames.get(entry.frameId); + if (!frame || frame.targetId !== entry.targetId) return; + frame.refs = Math.max(0, frame.refs - 1); + this.deleteUnusedFrame(frame); + } + + isRefCurrent(entry: RefEntry): boolean { + if (entry.frameId === entry.targetId) return this.targets.get(entry.targetId)?.generation === entry.generation; + const frame = this.frames.get(entry.frameId); + return frame?.targetId === entry.targetId && frame.generation === entry.generation; + } + + /** + * Record the document identity (loaderId) a ref-owning frame's current + * generation was observed at, keyed like a ref's frameId: the target for a + * main-frame key, else the child frame. Persisted and reconciled later. + */ + recordDocument(frameKey: string, ownerTargetId: string, loaderId: string): void { + if (frameKey === ownerTargetId) { + this.ensureTarget(frameKey).document = loaderId; + return; + } + const frame = this.frames.get(frameKey); + if (frame && frame.targetId === ownerTargetId) frame.document = loaderId; + } + + /** The document identity recorded for a frame's current generation, if known. */ + documentOf(frameKey: string, ownerTargetId: string): string | undefined { + return frameKey === ownerTargetId ? this.targets.get(frameKey)?.document : this.frames.get(frameKey)?.document; + } + + /** Whether any imported ref-owning frame under this target still awaits reconciliation. */ + hasPendingDocument(targetId: string): boolean { + for (const owner of this.documentOwners.values()) if (owner === targetId) return true; + return false; + } + + /** + * The imported ref-owning frames under this target still awaiting + * reconciliation, each flagged `verifiable` when a usable mint-time loaderId + * exists to compare against the live document. Unverifiable frames (legacy or + * malformed identity) carry no loaderId and must fail safe. + */ + pendingDocuments(targetId: string): Array<{ frameKey: string; verifiable: boolean }> { + const pending: Array<{ frameKey: string; verifiable: boolean }> = []; + for (const [frameKey, owner] of this.documentOwners) { + if (owner === targetId) pending.push({ frameKey, verifiable: this.expectedDocuments.has(frameKey) }); + } + return pending; + } + + /** + * Reconcile one imported ref-owning frame against the document loaded now. + * A verifiable frame whose live loaderId matches its mint-time identity keeps + * its refs (unchanged across the process boundary); a mismatch, an unknown + * live document, or an unverifiable (legacy/malformed) identity invalidates + * *only that frame* — the target for a main-frame key, else the child frame, + * leaving the main frame, siblings, and unchanged frames intact. One-shot: + * the pending entry is consumed either way. A blank expected or live loaderId + * is always treated as a non-match so empty identities can never mis-verify. + */ + reconcileDocument(frameKey: string, ownerTargetId: string, liveLoaderId: string | undefined): void { + if (!this.documentOwners.has(frameKey)) return; + this.documentOwners.delete(frameKey); + const expected = this.expectedDocuments.get(frameKey); + this.expectedDocuments.delete(frameKey); + const unverifiable = this.unverifiedDocuments.delete(frameKey); + if (!unverifiable && !!expected && !!liveLoaderId && liveLoaderId === expected) { + this.recordDocument(frameKey, ownerTargetId, liveLoaderId); + } else if (frameKey === ownerTargetId) { + this.invalidateTarget(frameKey); + } else { + this.invalidateFrame(ownerTargetId, frameKey); + } + } + + invalidateTarget(targetId: string): void { + const target = this.ensureTarget(targetId); + target.generation += 1; + target.observationRevision += 1; + // The old document's refs are gone; its identity no longer describes the + // live page, so drop it until the next observation records the new one. + target.document = undefined; + this.deleteRefs((entry) => entry.targetId === targetId); + for (const frame of [...this.frames.values()]) { + if (frame.targetId !== targetId) continue; + frame.generation += 1; + this.deleteUnusedFrame(frame); + } + } + + invalidateFrame(targetId: string, frameId: string): void { + const target = this.targets.get(targetId); + if (target) target.observationRevision += 1; + const frame = this.frames.get(frameId); + if (!frame || frame.targetId !== targetId) return; + frame.generation += 1; + // The old document's refs are gone; its identity no longer describes the + // live frame, so drop it until the next observation records the new one. + frame.document = undefined; + this.deleteRefs((entry) => entry.targetId === targetId && entry.frameId === frameId); + this.deleteUnusedFrame(frame); + } + + removeFrame(targetId: string, frameId: string): void { + const target = this.targets.get(targetId); + if (target) target.observationRevision += 1; + const frame = this.frames.get(frameId); + if (!frame || frame.targetId !== targetId) return; + this.deleteRefs((entry) => entry.targetId === targetId && entry.frameId === frameId); + this.frames.delete(frameId); + } + + dropTarget(targetId: string): void { + this.deleteRefs((entry) => entry.targetId === targetId); + this.targets.delete(targetId); + for (const [frameId, frame] of this.frames) { + if (frame.targetId === targetId) this.frames.delete(frameId); + } + } + + exportGenerations(): Array<[string, number]> { + return [ + ...[...this.targets.values()].map((state): [string, number] => [state.targetId, state.generation]), + ...[...this.frames.values()].map((state): [string, number] => [state.frameId, state.generation]), + ]; + } + + /** + * Document identities to persist for the given ref-owning frame keys: the + * recorded (live-verified or mint-time) loaderId, or a still-pending imported + * one for a frame this process never re-attached, so the identity survives + * intermediate invocations that never touched the frame. An unverifiable + * imported frame contributes nothing — identity is never fabricated on + * re-export, so legacy/partial state stays honestly unverifiable downstream. + */ + exportDocuments(refFrames: ReadonlySet): Array<[string, string]> { + const out: Array<[string, string]> = []; + for (const frameKey of refFrames) { + const loaderId = + this.frames.get(frameKey)?.document ?? this.targets.get(frameKey)?.document ?? this.expectedDocuments.get(frameKey); + if (loaderId !== undefined) out.push([frameKey, loaderId]); + } + return out; + } + + importState( + generations: readonly (readonly [string, number])[], + entries: readonly (readonly [string, Omit])[], + activeTargetId?: string, + documents?: readonly (readonly [string, string])[], + ): void { + const importedGenerations = new Map(generations); + const validDocuments = validateImportedDocuments(documents); + this.expectedDocuments.clear(); + this.documentOwners.clear(); + this.unverifiedDocuments.clear(); + const targetIds = new Set(entries.map(([, entry]) => entry.targetId)); + if (activeTargetId) targetIds.add(activeTargetId); + for (const targetId of targetIds) { + this.targets.set(targetId, { + targetId, + generation: importedGenerations.get(targetId) ?? 0, + observationRevision: 0, + }); + } + for (const [ref, imported] of entries) { + this.deleteRef(ref); + const entry: RefEntry = { ...imported, sessionId: "" }; + // Classify each ref-owning frame's document identity once, keyed like the + // ref itself: a usable loaderId is verifiable and reconciled against the + // live browser; anything else fails safe (staled) rather than resolving. + if (!this.documentOwners.has(entry.frameId)) { + this.documentOwners.set(entry.frameId, entry.targetId); + const loaderId = validDocuments.get(entry.frameId); + if (loaderId !== undefined) this.expectedDocuments.set(entry.frameId, loaderId); + else this.unverifiedDocuments.add(entry.frameId); + } + if (entry.frameId !== entry.targetId) { + let frame = this.frames.get(entry.frameId); + if (!frame) { + frame = { + targetId: entry.targetId, + frameId: entry.frameId, + generation: importedGenerations.get(entry.frameId) ?? entry.generation, + captures: 0, + refs: 0, + }; + this.frames.set(entry.frameId, frame); + } + frame.refs += 1; + } + this.refs.set(ref, entry); + } + } + + private ensureTarget(targetId: string): TargetGenerationState { + let target = this.targets.get(targetId); + if (!target) { + target = { targetId, generation: 0, observationRevision: 0 }; + this.targets.set(targetId, target); + } + return target; + } + + private deleteRefs(predicate: (entry: RefEntry) => boolean): void { + for (const [ref, entry] of this.refs) { + if (predicate(entry)) this.deleteRef(ref); + } + } + + private deleteUnusedFrame(frame: FrameGenerationState): void { + if (frame.captures === 0 && frame.refs === 0 && this.frames.get(frame.frameId) === frame) { + this.frames.delete(frame.frameId); + } + } +} + +export function staleRefError(ref: string, cause?: unknown): Error { + return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); +} + +/** + * Distil an untrusted, disk-sourced `documents` array into the frame keys that + * carry a single, non-blank loaderId usable for reconciliation. Everything + * dubious degrades to "unverifiable for that frame" (dropped here, so the frame + * fails safe on import) rather than throwing: a non-array input, non-tuple or + * non-string entries, a blank loaderId (which must never match a blank live + * one), and duplicate keys with conflicting loaderIds (ambiguous identity). + */ +function validateImportedDocuments(documents: unknown): Map { + const valid = new Map(); + const ambiguous = new Set(); + if (!Array.isArray(documents)) return valid; + for (const entry of documents) { + if (!Array.isArray(entry) || entry.length < 2) continue; + const [frameKey, loaderId] = entry as [unknown, unknown]; + if (typeof frameKey !== "string" || frameKey === "" || typeof loaderId !== "string" || loaderId === "") continue; + if (ambiguous.has(frameKey)) continue; + const existing = valid.get(frameKey); + if (existing === undefined) valid.set(frameKey, loaderId); + else if (existing !== loaderId) { + valid.delete(frameKey); + ambiguous.add(frameKey); + } + } + return valid; +} diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 9cb21a14..0f407aed 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -13,6 +13,39 @@ import { type CuaBrowserAction, } from "@onkernel/cua-ai"; import { CdpConnection, type CdpEventMessage } from "./cdp"; +import { + BrowserDocumentReconciler, + oopifFrameOffset, + type FrameTreeNode, +} from "./browser-document-reconciliation"; +import { FrameCollectionError, frameCollectionError, isExpectedFrameCollectionError } from "./browser-frame-collection"; +import { + FRAME_ROLES, + INTERACTIVE_ROLES, + REF_PLACEHOLDER, + IncompleteObservationError, + ObservationChangedError, + buildNthIndex, + cohortKey, + observedNodes, + renderObservationNode, + staticTextRun, + type AXNode, + type BrowserObservation, + type BrowserPresentation, + type FrameStitch, + type IncompleteFrame, + type ObservationLine, + type RenderContext, +} from "./browser-observation"; +import { + REF_STATE_VERSION, + RefGenerationLifecycle, + staleRefError, + type BrowserRefState, + type GenerationCapture, + type RefEntry, +} from "./browser-ref-lifecycle"; import type { BatchReadResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; @@ -21,62 +54,11 @@ const FIND_MATCH_LIMIT = 20; const REF_LIMIT_PER_TARGET = 1000; const SCROLL_NOTCH_PX = 120; -const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; -const REF_PLACEHOLDER = "\u0000"; const UNCHANGED_SNAPSHOT = "Page unchanged since the last snapshot; previous element refs are still valid."; -interface AXNode { - nodeId: string; - ignored?: boolean; - role?: { value?: string }; - name?: { value?: string }; - value?: { value?: unknown }; - properties?: Array<{ name: string; value?: { value?: unknown } }>; - backendDOMNodeId?: number; - parentId?: string; - childIds?: string[]; -} - -interface RefEntry { - backendNodeId: number; - targetId: string; - /** Generation key: the owning page target id for main-frame refs, the frame id for iframe refs. */ - frameId: string; - /** Session to route DOM/Input calls through: the frame's own session for OOPIFs, the page session otherwise. */ - sessionId: string; - generation: number; - role: string; - name: string; - nth: number; - /** Size of the (role, name) cohort in the tree the ref was minted from. */ - cohort: number; -} - -interface NthIndex { - index: Map; - cohorts: Map; -} - -interface RenderContext { - targetId: string; - frameKey: string; - sessionId: string; - generation: number; - interactiveOnly: boolean; - nthIndex: NthIndex; - cursorIds?: ReadonlySet; -} - -interface RenderedLine { - text: string; - refNode?: AXNode; - ctx: RenderContext; -} - -interface FrameStitch { - byId: Map; - roots: string[]; - ctx: RenderContext; +interface CollectedObservation { + readonly observation: BrowserObservation; + readonly captures: readonly GenerationCapture[]; } export interface BrowserFindCandidate { @@ -86,21 +68,6 @@ export interface BrowserFindCandidate { score: number; } -/** - * Serializable ref state, so refs minted in one process (e.g. a `cua - * snapshot` invocation) can be resolved in a later one against the same - * browser. Session ids are process-local and deliberately not exported; - * imported refs rebind lazily. Backend node ids stay valid for the life of - * the document, and the usual generation/self-heal machinery covers pages - * that changed in between. - */ -export interface BrowserRefState { - refCounter: number; - activeTargetId?: string; - generations: Array<[string, number]>; - refs: Array<[string, Omit]>; -} - /** * Executes browser-plane canonical actions over CDP. * @@ -122,7 +89,22 @@ export interface BrowserRefState { * session so actions resolve through the right one, and each frame's refs * are invalidated independently when that frame navigates. Re-snapshotting * an unchanged page with the same params returns a short unchanged notice - * instead of the full tree. + * instead of the full tree. Clicking/hovering an out-of-process iframe ref + * dispatches on the page session (where the Input domain lives) but shifts + * the frame-local box-model quads by the iframe owner's offset first, so the + * event lands on the intended element rather than the frame's origin. + * + * Refs persisted across invocations (see {@link exportRefState}) carry the + * `loaderId` of every ref-owning frame — the page target for main-frame refs, + * the child frame for same-process iframes and OOPIFs. On the next process the + * imported set is reconciled per frame against the live browser before any ref + * resolves: an unchanged frame keeps its refs, while a reload/navigation + * between invocations stales only the frames whose document changed — including + * an OOPIF-only navigation that left the main-frame loaderId untouched — since + * generation alone is process-local and cannot detect a change that committed + * after export. A ref-owning frame whose imported identity is missing or + * unusable (legacy or partial state) is staled rather than trusted, so it can + * never silently resolve against a different document with reused node ids. * * Native JavaScript dialogs are auto-handled so they never wedge the CDP * session: alert and beforeunload dialogs are accepted (so navigation can @@ -131,10 +113,12 @@ export interface BrowserRefState { */ export class BrowserExecutor { private readonly refs = new Map(); - private readonly generations = new Map(); + private readonly lifecycle = new RefGenerationLifecycle(this.refs); private readonly targetsBySession = new Map(); private readonly frameSessions = new Map(); + private readonly frameOwners = new Map(); private readonly frameTargets = new Set(); + private readonly documents: BrowserDocumentReconciler; private readonly lastSnapshots = new Map(); private readonly selfNavigations = new Set(); private readonly dialogNotes: string[] = []; @@ -144,28 +128,51 @@ export class BrowserExecutor { constructor(cdp: string | CdpConnection) { this.cdp = typeof cdp === "string" ? new CdpConnection(cdp) : cdp; + this.documents = new BrowserDocumentReconciler(this.cdp, this.lifecycle, this.frameSessions); this.cdp.onEvent((event) => this.handleCdpEvent(event)); } private handleCdpEvent(event: CdpEventMessage): void { switch (event.method) { case "Page.frameNavigated": { - const frame = event.params.frame as { id?: string; parentId?: string } | undefined; + const frame = event.params.frame as { id?: string; parentId?: string; loaderId?: string } | undefined; if (!event.sessionId || !frame) return; - const targetId = this.targetsBySession.get(event.sessionId); - if (!targetId) return; - if (this.frameTargets.has(targetId)) { - // Refs from a frame target's tree (its root and any same-process - // subframes inlined in it) are all minted against the target's key, - // so any navigation observed in its session stales them. - this.invalidateFrame(targetId); + const sessionTargetId = this.targetsBySession.get(event.sessionId); + if (!sessionTargetId) return; + if (this.frameTargets.has(sessionTargetId)) { + // The OOPIF tree and any same-process descendants fetched through + // its session share the OOPIF generation key. + const owner = this.ownerForFrameTarget(sessionTargetId); + if (owner) { + this.lifecycle.invalidateFrame(owner, sessionTargetId); + if (frame.loaderId) this.lifecycle.recordDocument(sessionTargetId, owner, frame.loaderId); + } return; } if (frame.parentId) { - if (frame.id) this.invalidateFrame(frame.id); + if (frame.id) { + this.lifecycle.invalidateFrame(sessionTargetId, frame.id); + if (frame.loaderId) this.lifecycle.recordDocument(frame.id, sessionTargetId, frame.loaderId); + } return; } - if (!this.selfNavigations.delete(targetId)) this.invalidateRefs(targetId); + if (!this.selfNavigations.delete(sessionTargetId)) this.lifecycle.invalidateTarget(sessionTargetId); + // Record the committed document so it reflects the current generation, + // whether the navigation was ours or page-initiated. + if (frame.loaderId) this.lifecycle.recordDocument(sessionTargetId, sessionTargetId, frame.loaderId); + return; + } + case "Page.frameDetached": { + if (!event.sessionId) return; + const { frameId } = event.params as { frameId?: string; reason?: "remove" | "swap" }; + const sessionTargetId = this.targetsBySession.get(event.sessionId); + if (!frameId || !sessionTargetId) return; + if (this.frameTargets.has(sessionTargetId)) { + const owner = this.ownerForFrameTarget(sessionTargetId); + if (owner) this.lifecycle.invalidateFrame(owner, sessionTargetId); + } else { + this.lifecycle.removeFrame(sessionTargetId, frameId); + } return; } case "Page.navigatedWithinDocument": { @@ -180,9 +187,13 @@ export class BrowserExecutor { case "Target.attachedToTarget": { const { sessionId, targetInfo } = event.params as { sessionId?: string; targetInfo?: { targetId?: string; type?: string } }; if (!sessionId || !targetInfo?.targetId || targetInfo.type !== "iframe") return; + const parentTarget = event.sessionId ? this.targetsBySession.get(event.sessionId) : undefined; + const owner = parentTarget ? (this.frameOwners.get(parentTarget) ?? parentTarget) : undefined; this.frameSessions.set(targetInfo.targetId, sessionId); + if (owner) this.frameOwners.set(targetInfo.targetId, owner); this.frameTargets.add(targetInfo.targetId); this.targetsBySession.set(sessionId, targetInfo.targetId); + this.documents.frameSessionAttached(targetInfo.targetId); void this.cdp.send("Page.enable", {}, sessionId).catch(() => {}); return; } @@ -205,7 +216,16 @@ export class BrowserExecutor { if (typeof sessionId !== "string") return; const targetId = this.targetsBySession.get(sessionId); this.targetsBySession.delete(sessionId); - if (targetId) this.dropTarget(targetId); + if (!targetId) return; + if (this.frameTargets.has(targetId)) { + const owner = this.ownerForFrameTarget(targetId); + if (owner) this.lifecycle.removeFrame(owner, targetId); + this.frameSessions.delete(targetId); + this.frameOwners.delete(targetId); + this.frameTargets.delete(targetId); + } else { + this.dropTarget(targetId); + } return; } } @@ -218,20 +238,27 @@ export class BrowserExecutor { /** Snapshot the ref table for persistence across invocations; see {@link BrowserRefState}. */ exportRefState(): BrowserRefState { + const refFrames = new Set([...this.refs.values()].map((entry) => entry.frameId)); + const documents = this.lifecycle.exportDocuments(refFrames); return { refCounter: this.refCounter, ...(this.activeTargetId ? { activeTargetId: this.activeTargetId } : {}), - generations: [...this.generations], + generations: this.lifecycle.exportGenerations(), + ...(documents.length ? { documents } : {}), refs: [...this.refs].map(([ref, { sessionId: _sessionId, ...entry }]) => [ref, entry]), }; } /** Restore a ref table exported by a previous invocation against the same browser. */ importRefState(state: BrowserRefState): void { + if (state.version !== undefined && state.version > REF_STATE_VERSION) { + // A newer identity scheme we can't verify against; reject loudly rather + // than silently mis-parse it into trusted-looking refs. + throw new Error(`unsupported browser ref state version ${state.version}; this build understands up to ${REF_STATE_VERSION}`); + } this.refCounter = Math.max(this.refCounter, state.refCounter); this.activeTargetId = state.activeTargetId ?? this.activeTargetId; - for (const [frameId, generation] of state.generations) this.generations.set(frameId, generation); - for (const [ref, entry] of state.refs) this.refs.set(ref, { ...entry, sessionId: "" }); + this.lifecycle.importState(state.generations, state.refs, state.activeTargetId, state.documents); } async execute(action: CuaBrowserAction): Promise { @@ -306,72 +333,145 @@ export class BrowserExecutor { } private async snapshot(action: CuaActionBrowserSnapshot): Promise { - const targetId = await this.resolveTarget(action.tab_id); + return this.withObservation(action.tab_id, true, (observation) => + this.renderObservation(this.presentObservation(observation, action)), + ); + } + + private async withObservation( + tabId: string | undefined, + includeCursor: boolean, + consume: (observation: BrowserObservation) => T, + ): Promise { + for (let attempt = 0; ; attempt += 1) { + let collected: CollectedObservation | undefined; + try { + collected = await this.collectObservation(tabId, includeCursor); + return consume(collected.observation); + } catch (error) { + const retryable = error instanceof ObservationChangedError || error instanceof IncompleteObservationError; + if (!retryable || attempt === 2) throw error; + } finally { + if (collected) this.lifecycle.release(collected.captures); + } + } + } + + private async collectObservation(tabId: string | undefined, includeCursor: boolean): Promise { + const targetId = await this.resolveTarget(tabId); const pageSession = await this.attach(targetId); - const refEntry = action.ref ? this.resolveRef(action.ref, targetId) : undefined; - const frameKey = refEntry?.frameId ?? targetId; - const { nodes, sessionId } = await this.frameAxTree(frameKey, targetId, pageSession); - const byId = new Map(nodes.map((node) => [node.nodeId, node])); - let rootIds = nodes.filter((node) => !node.parentId).map((node) => node.nodeId); + const pageTree = await this.documents.capturePage(targetId, pageSession); + const before = (await this.cdp.pageTargets()).find((target) => target.targetId === targetId); + if (!before) throw new ObservationChangedError("Browser target disappeared during observation"); + + const captures: GenerationCapture[] = [this.lifecycle.captureTarget(targetId)]; + try { + const rootCapture = captures[0]!; + const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); + const rootCtx: RenderContext = { + targetId, + frameKey: targetId, + sessionId, + generation: rootCapture.generation, + nthIndex: buildNthIndex(nodes), + ...(includeCursor ? { cursorIds: await this.cursorPointerIds(pageSession) } : {}), + }; + const tree = this.frameStitch(nodes, rootCtx); + const { stitches, incompleteFrames } = await this.stitchFrames(nodes, targetId, pageSession, captures, pageTree); + const after = (await this.cdp.pageTargets()).find((target) => target.targetId === targetId); + if (!after || before.url !== after.url || before.title !== after.title) { + throw new ObservationChangedError("Browser target metadata changed during observation"); + } + if (captures.some((capture) => !this.lifecycle.isCurrent(capture))) throw new ObservationChangedError(); + return { + observation: { + targetId, + tree, + stitches, + incompleteFrames, + revision: rootCapture.observationRevision, + generations: new Map(captures.map((capture) => [capture.frameKey, capture.generation])), + }, + captures, + }; + } catch (error) { + this.lifecycle.release(captures); + throw error; + } + } + + private presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation { + const refEntry = action.ref ? this.resolveRef(action.ref, observation.targetId) : undefined; + let tree = observation.tree; + let rootIds = tree.roots; if (action.ref && refEntry) { + if (refEntry.frameId === observation.targetId) { + tree = observation.tree; + } else { + const frameTree = [...observation.stitches.values()].find((stitch) => stitch.ctx.frameKey === refEntry.frameId); + if (!frameTree) { + const details = observation.incompleteFrames + .map((frame) => `${frame.frameId ?? `backend node ${frame.backendNodeId}`} (${frame.reason})`) + .join(", "); + throw new IncompleteObservationError( + `Could not verify ref ${action.ref}: owning frame ${refEntry.frameId} was not collected${details ? `; incomplete frames: ${details}` : ""}`, + ); + } + tree = frameTree; + } + const treeNodes = [...tree.byId.values()]; const rootNode = - nodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, nodes); + treeNodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, treeNodes); rootIds = [rootNode.nodeId]; } - const interactiveOnly = action.filter === "interactive"; - const ctx: RenderContext = { - targetId, - frameKey, - sessionId, - generation: this.generation(frameKey), - interactiveOnly, - nthIndex: buildNthIndex(nodes), - cursorIds: frameKey === targetId ? await this.cursorPointerIds(pageSession) : undefined, - }; - const stitches = frameKey === targetId ? await this.stitchFrames(nodes, targetId, pageSession, interactiveOnly) : new Map(); - const lines: RenderedLine[] = []; + const lines: ObservationLine[] = []; const maxDepth = action.depth ?? DEFAULT_SNAPSHOT_DEPTH; - const walk = (tree: Map, treeCtx: RenderContext, nodeId: string, depth: number, parentName: string): void => { - const node = tree.get(nodeId); + const walk = (current: FrameStitch, nodeId: string, depth: number, parentName: string): void => { + const node = current.byId.get(nodeId); if (!node) return; + const ctx = current.ctx; let childDepth = depth; if (!node.ignored) { - const rendered = this.renderNode(node, depth, parentName, treeCtx); + const rendered = renderObservationNode(node, depth, parentName, ctx, interactiveOnly); if (rendered) { - lines.push({ ...rendered, ctx: treeCtx }); + lines.push({ ...rendered, ctx }); childDepth = depth + 1; } } if (childDepth > maxDepth) return; - const stitch = treeCtx === ctx && node.backendDOMNodeId !== undefined ? stitches.get(node.backendDOMNodeId) : undefined; + const stitch = current === observation.tree && node.backendDOMNodeId !== undefined ? observation.stitches.get(node.backendDOMNodeId) : undefined; if (stitch) { - for (const frameRootId of stitch.roots) walk(stitch.byId, stitch.ctx, frameRootId, childDepth, ""); + for (const frameRootId of stitch.roots) walk(stitch, frameRootId, childDepth, ""); return; } - const name = node.name?.value ?? ""; - const childName = name || parentName; + const childName = node.name?.value || parentName; const childIds = node.childIds ?? []; for (let i = 0; i < childIds.length; i += 1) { - const run = staticTextRun(tree, childIds, i); + const run = staticTextRun(current.byId, childIds, i); if (run) { - const rendered = this.renderNode(run.node, childDepth, childName, treeCtx); - if (rendered) lines.push({ ...rendered, ctx: treeCtx }); + const rendered = renderObservationNode(run.node, childDepth, childName, ctx, interactiveOnly); + if (rendered) lines.push({ ...rendered, ctx }); i = run.end; - continue; - } - walk(tree, treeCtx, childIds[i]!, childDepth, childName); + } else walk(current, childIds[i]!, childDepth, childName); } }; - for (const rootId of rootIds) walk(byId, ctx, rootId, 0, ""); - + for (const rootId of rootIds) walk(tree, rootId, 0, ""); const shape = lines.map((line) => line.text).join("\n"); - const frameGenerations = [...stitches.values()].map((stitch) => `${stitch.ctx.frameKey}:${stitch.ctx.generation}`); - const key = [action.ref ?? "", action.depth ?? "", action.filter ?? "", `${frameKey}:${ctx.generation}`, ...frameGenerations].join("|"); - const cached = this.lastSnapshots.get(targetId); - this.lastSnapshots.set(targetId, { key, shape }); - if (cached && cached.key === key && cached.shape === shape) return UNCHANGED_SNAPSHOT; + const generationKey = [...observation.generations].map(([frameKey, generation]) => `${frameKey}:${generation}`).join("|"); + return { + observation, + cacheKey: [action.ref ?? "", action.depth ?? "", action.filter ?? "", `revision:${observation.revision}`, generationKey].join("|"), + lines, + shape, + }; + } + private renderObservation(presentation: BrowserPresentation): string { + const { observation, cacheKey, lines, shape } = presentation; + const cached = this.lastSnapshots.get(observation.targetId); + this.lastSnapshots.set(observation.targetId, { key: cacheKey, shape }); + if (cached?.key === cacheKey && cached.shape === shape) return UNCHANGED_SNAPSHOT; let text = ""; for (const line of lines) { if (text.length > SNAPSHOT_CHAR_LIMIT) break; @@ -381,31 +481,10 @@ export class BrowserExecutor { if (text.length > SNAPSHOT_CHAR_LIMIT) { text = `${text.slice(0, SNAPSHOT_CHAR_LIMIT)}\n… truncated at ${SNAPSHOT_CHAR_LIMIT} characters. Re-request with a smaller depth, filter: "interactive", or a ref to narrow the subtree.`; } - this.pruneRefs(targetId); + this.pruneRefs(observation.targetId); return text || "(empty accessibility tree)"; } - private renderNode(node: AXNode, depth: number, parentName: string, ctx: RenderContext): { text: string; refNode?: AXNode } | undefined { - const role = node.role?.value ?? ""; - const name = node.name?.value ?? ""; - const interactive = INTERACTIVE_ROLES.has(role); - const pointer = node.backendDOMNodeId !== undefined && (ctx.cursorIds?.has(node.backendDOMNodeId) ?? false); - if (ctx.interactiveOnly && !interactive && !pointer) return undefined; - if (role === "StaticText" && name === parentName) return undefined; - if (!ctx.interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; - let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; - let refNode: AXNode | undefined; - const refWorthy = interactive || pointer || FRAME_ROLES.has(role) || (name !== "" && CONTENT_ROLES.has(role)); - if (node.backendDOMNodeId !== undefined && refWorthy) { - line += ` [${REF_PLACEHOLDER}]`; - refNode = node; - } - const states = collectStates(node); - if (pointer && !interactive) states.push("cursor:pointer"); - if (states.length > 0) line += ` [${states.join(", ")}]`; - return { text: line, refNode }; - } - /** Fetch a frame's AX tree: OOPIFs through their own session, same-process frames through the page session with a frameId. */ private async frameAxTree(frameKey: string, targetId: string, pageSession: string): Promise<{ nodes: AXNode[]; sessionId: string }> { const frameSession = this.frameSessions.get(frameKey); @@ -418,42 +497,86 @@ export class BrowserExecutor { return { nodes, sessionId: pageSession }; } + private frameStitch(nodes: AXNode[], ctx: RenderContext): FrameStitch { + return { + byId: new Map(nodes.map((node) => [node.nodeId, node])), + roots: nodes.filter((node) => !node.parentId).map((node) => node.nodeId), + ctx, + }; + } + /** Resolve each iframe node's child frame and fetch its AX tree for stitching. One nesting level only. */ private async stitchFrames( - nodes: AXNode[], + nodes: readonly AXNode[], targetId: string, pageSession: string, - interactiveOnly: boolean, - ): Promise> { + captures: GenerationCapture[], + pageTree: FrameTreeNode | undefined, + ): Promise<{ stitches: ReadonlyMap; incompleteFrames: readonly IncompleteFrame[] }> { const stitches = new Map(); + const incompleteFrames: IncompleteFrame[] = []; for (const node of nodes) { if (node.ignored || !FRAME_ROLES.has(node.role?.value ?? "") || node.backendDOMNodeId === undefined) continue; + const backendNodeId = node.backendDOMNodeId; + let dom: { frameId?: string; contentDocument?: { frameId?: string } }; try { - const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>( + const described = await this.cdp.send<{ node?: typeof dom }>( "DOM.describeNode", - { backendNodeId: node.backendDOMNodeId, depth: 1 }, + { backendNodeId, depth: 1 }, pageSession, ); - const frameId = dom.contentDocument?.frameId ?? dom.frameId; - if (!frameId || frameId === targetId) continue; + if (!described.node) throw new Error("response did not include a node"); + dom = described.node; + } catch (error) { + if (!isExpectedFrameCollectionError(error, "DOM.describeNode")) { + throw frameCollectionError(backendNodeId, undefined, "DOM.describeNode", error); + } + incompleteFrames.push({ backendNodeId, stage: "describe", reason: error.message }); + continue; + } + const frameId = dom.contentDocument?.frameId ?? dom.frameId; + if (!frameId || frameId === targetId) { + incompleteFrames.push({ backendNodeId, stage: "resolve", reason: "iframe did not expose a child frame id" }); + continue; + } + + if (this.frameSessions.has(frameId)) this.frameOwners.set(frameId, targetId); + let capture: GenerationCapture; + try { + capture = this.lifecycle.captureFrame(targetId, frameId); + } catch (error) { + throw frameCollectionError(backendNodeId, frameId, "capturing the frame generation", error); + } + let retained = false; + try { const { nodes: frameNodes, sessionId } = await this.frameAxTree(frameId, targetId, pageSession); - stitches.set(node.backendDOMNodeId, { - byId: new Map(frameNodes.map((frameNode) => [frameNode.nodeId, frameNode])), - roots: frameNodes.filter((frameNode) => !frameNode.parentId).map((frameNode) => frameNode.nodeId), - ctx: { + let stitch: FrameStitch; + try { + stitch = this.frameStitch(frameNodes, { targetId, frameKey: frameId, sessionId, - generation: this.generation(frameId), - interactiveOnly, + generation: capture.generation, nthIndex: buildNthIndex(frameNodes), - }, - }); - } catch { - // Cross-origin or already-detached frames can refuse the fetch; the iframe renders without children. + }); + } catch (error) { + throw frameCollectionError(backendNodeId, frameId, "building the accessibility index", error); + } + stitches.set(backendNodeId, stitch); + await this.documents.captureFrame(frameId, targetId, pageSession, pageTree); + captures.push(capture); + retained = true; + } catch (error) { + if (error instanceof FrameCollectionError) throw error; + if (!isExpectedFrameCollectionError(error, "Accessibility.getFullAXTree")) { + throw frameCollectionError(backendNodeId, frameId, "Accessibility.getFullAXTree", error); + } + incompleteFrames.push({ backendNodeId, frameId, stage: "accessibility", reason: error.message }); + } finally { + if (!retained) this.lifecycle.release([capture]); } } - return stitches; + return { stitches, incompleteFrames }; } /** Resolve backend node ids for elements whose own computed cursor is "pointer", without touching the DOM. */ @@ -499,46 +622,32 @@ export class BrowserExecutor { * matches, best first. Structured counterpart of the `browser_find` action. */ async findCandidates(query: string, tabId?: string, roles?: ReadonlySet): Promise { - const targetId = await this.resolveTarget(tabId); - const session = await this.attach(targetId); - const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); - const pools: Array<{ nodes: AXNode[]; ctx: RenderContext }> = [ - { - nodes, - ctx: { - targetId, - frameKey: targetId, - sessionId: session, - generation: this.generation(targetId), - interactiveOnly: false, - nthIndex: buildNthIndex(nodes), - }, - }, - ]; - // Search stitched frames too so find sees everything snapshot renders. - for (const stitch of (await this.stitchFrames(nodes, targetId, session, false)).values()) { - pools.push({ nodes: [...stitch.byId.values()], ctx: stitch.ctx }); - } - const queryTokens = tokenize(query); - const scored = pools - .flatMap(({ nodes: poolNodes, ctx }) => - poolNodes - .filter( - (node) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), - ) - .map((node) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })), - ) - .filter((entry) => entry.score > 0 && (!roles || roles.has(entry.node.role?.value ?? ""))) - .sort((a, b) => b.score - a.score) - .slice(0, FIND_MATCH_LIMIT); - const candidates = scored.map(({ node, ctx, score }) => ({ - ref: this.mintRef(node, ctx), - role: node.role?.value ?? "", - name: node.name?.value ?? "", - score, - })); - this.pruneRefs(targetId); - return candidates; + return this.withObservation(tabId, false, (observation) => { + const queryTokens = tokenize(query); + const scored = [...observedNodes(observation)] + .filter( + ({ node }) => + !node.ignored && + node.backendDOMNodeId !== undefined && + (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), + ) + .map(({ node, ctx }) => ({ + node, + ctx, + score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)), + })) + .filter((entry) => entry.score > 0 && (!roles || roles.has(entry.node.role?.value ?? ""))) + .sort((a, b) => b.score - a.score) + .slice(0, FIND_MATCH_LIMIT); + const candidates = scored.map(({ node, ctx, score }) => ({ + ref: this.mintRef(node, ctx), + role: node.role?.value ?? "", + name: node.name?.value ?? "", + score, + })); + this.pruneRefs(observation.targetId); + return candidates; + }); } private async click(action: CuaActionBrowserClick): Promise { @@ -582,6 +691,10 @@ export class BrowserExecutor { private async fill(action: CuaActionBrowserFill): Promise { const targetId = await this.resolveTarget(action.tab_id); + // Attach before resolving the ref (like click/hover) so any imported document + // identity is reconciled first; otherwise a document that changed across the + // process boundary would resolve against a reused backend node id. + await this.attach(targetId); const entry = this.resolveRef(action.ref, targetId); const session = await this.refSession(entry); const objectId = await this.resolveObject(entry, action.ref, session); @@ -603,6 +716,8 @@ export class BrowserExecutor { private async scrollTo(action: CuaActionBrowserScrollTo): Promise { const targetId = await this.resolveTarget(action.tab_id); + // Reconcile the imported document before resolving the ref; see fill(). + await this.attach(targetId); const entry = this.resolveRef(action.ref, targetId); await this.scrollIntoView(entry, action.ref, await this.refSession(entry)); } @@ -651,7 +766,7 @@ export class BrowserExecutor { const entry = history.entries[history.currentIndex + (direction === "back" ? -1 : 1)]; if (!entry) throw new Error(`cannot go ${direction}: no history entry`); await this.selfNavigate(targetId, () => this.cdp.send("Page.navigateToHistoryEntry", { entryId: entry.id }, session)); - this.invalidateRefs(targetId); + this.lifecycle.invalidateTarget(targetId); return `Navigated ${direction}.\n${await this.tabContext(targetId)}`; } const url = normalizeGotoUrl(action.url); @@ -663,7 +778,7 @@ export class BrowserExecutor { this.selfNavigations.delete(targetId); throw new Error(`navigation to ${url} failed: ${errorText}`); } - this.invalidateRefs(targetId); + this.lifecycle.invalidateTarget(targetId); return `Navigated to ${url}.\n${await this.tabContext(targetId)}`; } @@ -738,9 +853,20 @@ export class BrowserExecutor { refSession, ); const quad = model.content; - // Box-model quads are main-viewport coordinates even through an OOPIF's - // session, so input always dispatches on the page target's session. - return { x: (quad[0]! + quad[4]!) / 2, y: (quad[1]! + quad[5]!) / 2, session }; + let x = (quad[0]! + quad[4]!) / 2; + let y = (quad[1]! + quad[5]!) / 2; + // Input.dispatchMouseEvent lives on the tab target and hit-tests in + // top-level viewport coordinates, so it always dispatches on the page + // session. A cross-process OOPIF lays out in its own renderer, so its + // box-model quads are frame-local; shift them by the iframe owner's + // content-box origin before dispatching. Same-process and main-frame + // refs read through the page session and are already top-level. + if (refSession !== session) { + const offset = await oopifFrameOffset(this.cdp, entry.frameId, session); + x += offset.x; + y += offset.y; + } + return { x, y, session }; } if (typeof action.x === "number" && typeof action.y === "number") return { x: action.x, y: action.y, session }; throw new Error("page target required: pass a ref or viewport coordinates"); @@ -806,7 +932,7 @@ export class BrowserExecutor { const name = node.name?.value ?? ""; this.refCounter += 1; const ref = `e${this.refCounter}`; - this.refs.set(ref, { + this.lifecycle.retainRef(ref, { backendNodeId: node.backendDOMNodeId!, targetId: ctx.targetId, frameId: ctx.frameKey, @@ -837,47 +963,32 @@ export class BrowserExecutor { const entry = this.refs.get(ref); // Entries are deleted eagerly on invalidation; the generation check only // guards refs resolved while a navigation event is still in flight. - if (!entry || entry.targetId !== targetId || entry.generation !== this.generation(entry.frameId)) { - throw staleRefError(ref); - } + if (!entry || entry.targetId !== targetId || !this.lifecycle.isRefCurrent(entry)) throw staleRefError(ref); return entry; } - private generation(targetId: string): number { - return this.generations.get(targetId) ?? 0; - } - - private invalidateRefs(targetId: string): void { - this.generations.set(targetId, this.generation(targetId) + 1); - for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId) this.refs.delete(ref); - } - } - - private invalidateFrame(frameKey: string): void { - let tracked = this.generations.has(frameKey) || this.frameSessions.has(frameKey); - for (const [ref, entry] of this.refs) { - if (entry.frameId === frameKey) { - this.refs.delete(ref); - tracked = true; - } + private ownerForFrameTarget(frameTargetId: string): string | undefined { + const mapped = this.frameOwners.get(frameTargetId); + if (mapped) return mapped; + for (const entry of this.refs.values()) { + if (entry.frameId !== frameTargetId) continue; + this.frameOwners.set(frameTargetId, entry.targetId); + return entry.targetId; } - // Frames we never referenced don't get a generation entry, or pages with - // rotating ad iframes would grow the map without bound. - if (tracked) this.generations.set(frameKey, this.generation(frameKey) + 1); + return undefined; } private dropTarget(targetId: string): void { - this.generations.delete(targetId); + this.lifecycle.dropTarget(targetId); this.selfNavigations.delete(targetId); this.lastSnapshots.delete(targetId); - this.frameSessions.delete(targetId); - this.frameTargets.delete(targetId); - for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId || entry.frameId === targetId) { - if (entry.frameId !== targetId) this.generations.delete(entry.frameId); - this.refs.delete(ref); - } + for (const [frameId, owner] of this.frameOwners) { + if (owner !== targetId) continue; + const sessionId = this.frameSessions.get(frameId); + if (sessionId) this.targetsBySession.delete(sessionId); + this.frameSessions.delete(frameId); + this.frameOwners.delete(frameId); + this.frameTargets.delete(frameId); } } @@ -887,7 +998,7 @@ export class BrowserExecutor { for (const [ref, entry] of this.refs) { if (entry.targetId === targetId) owned.push(ref); } - for (const ref of owned.slice(0, Math.max(0, owned.length - REF_LIMIT_PER_TARGET))) this.refs.delete(ref); + for (const ref of owned.slice(0, Math.max(0, owned.length - REF_LIMIT_PER_TARGET))) this.lifecycle.deleteRef(ref); } private drainDialogNotes(): string | undefined { @@ -906,8 +1017,11 @@ export class BrowserExecutor { if (!this.targetsBySession.has(session)) { this.targetsBySession.set(session, targetId); await this.cdp.send("Page.enable", {}, session); + // setAutoAttach must run before reconcile so an imported OOPIF's session + // surfaces (via attachedToTarget) in time to read its own document. await this.cdp.send("Target.setAutoAttach", { autoAttach: true, flatten: true, waitForDebuggerOnStart: false }, session); } + await this.documents.reconcile(targetId, session); return session; } @@ -932,75 +1046,6 @@ function tabOf(action: { tab_id?: string }): string | undefined { return action.tab_id; } -function staleRefError(ref: string, cause?: unknown): Error { - return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); -} - -/** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ -function buildNthIndex(nodes: AXNode[]): NthIndex { - const cohorts = new Map(); - const index = new Map(); - for (const node of nodes) { - if (node.ignored || node.backendDOMNodeId === undefined) continue; - const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); - const nth = cohorts.get(key) ?? 0; - cohorts.set(key, nth + 1); - index.set(node.nodeId, nth); - } - return { index, cohorts }; -} - -function cohortKey(role: string, name: string): string { - return `${role}\u0000${name}`; -} - -/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ -function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { - let end = start; - const parts: string[] = []; - while (end < childIds.length) { - const node = tree.get(childIds[end]!); - if (!node || node.ignored || node.role?.value !== "StaticText") break; - const text = node.name?.value ?? ""; - if (text) parts.push(text); - end += 1; - } - if (end - start < 2) return undefined; - const first = tree.get(childIds[start]!)!; - return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; -} - -function collectStates(node: AXNode): string[] { - const states: string[] = []; - for (const property of node.properties ?? []) { - const value = property.value?.value; - switch (property.name) { - case "checked": - case "pressed": - case "expanded": - // False is meaningful here: it distinguishes an unchecked checkbox or - // collapsed disclosure from an element without the state at all. - if (value === true || value === "true") states.push(property.name); - else if (value === false || value === "false") states.push(`${property.name}=false`); - else if (value === "mixed") states.push(`${property.name}=mixed`); - break; - case "disabled": - case "selected": - case "required": - if (value === true || value === "true") states.push(property.name); - break; - case "level": - if (typeof value === "number") states.push(`level=${value}`); - break; - } - } - const value = node.value?.value; - if (value !== undefined && value !== "" && String(value) !== (node.name?.value ?? "")) { - states.push(`value=${JSON.stringify(String(value))}`); - } - return states; -} - function shortTabId(targetId: string): string { return targetId.slice(0, 10).toUpperCase(); } @@ -1113,51 +1158,6 @@ const FILL_FUNCTION = `function(value) { el.dispatchEvent(new Event("change", { bubbles: true })); }`; -const INTERACTIVE_ROLES: ReadonlySet = new Set([ - "button", - "link", - "textbox", - "searchbox", - "checkbox", - "radio", - "combobox", - "listbox", - "option", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "slider", - "spinbutton", - "switch", - "tab", - "treeitem", -]); - -const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); - -const FRAME_ROLES: ReadonlySet = new Set(["Iframe", "IframePresentational"]); - -/** Non-interactive roles that get refs when named, so scroll_to / ref-scoped snapshots can target them. */ -const CONTENT_ROLES: ReadonlySet = new Set([ - "heading", - "cell", - "gridcell", - "columnheader", - "rowheader", - "row", - "listitem", - "article", - "region", - "main", - "navigation", - "banner", - "contentinfo", - "complementary", - "tabpanel", - "figure", - "image", -]); - const CURSOR_SCAN_GROUP = "cua-cursor-scan"; const CURSOR_POINTER_SCAN = `(() => { diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts index c38604ae..7408f0fa 100644 --- a/packages/agent/src/translator/cdp.ts +++ b/packages/agent/src/translator/cdp.ts @@ -8,6 +8,7 @@ */ interface PendingCommand { + readonly method: string; resolve(result: unknown): void; reject(error: Error): void; } @@ -28,6 +29,19 @@ export interface CdpTargetInfo { export type CdpEventListener = (event: CdpEventMessage) => void; +/** A command error returned by the CDP JSON-RPC endpoint. */ +export class CdpProtocolError extends Error { + constructor( + readonly method: string, + readonly code: number, + readonly protocolMessage: string, + readonly data?: string, + ) { + super(protocolMessage); + this.name = "CdpProtocolError"; + } +} + export class CdpConnection { private socket?: WebSocket; private opening?: Promise; @@ -47,7 +61,7 @@ export class CdpConnection { const id = this.nextId++; const message = JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }); return new Promise((resolve, reject) => { - this.pending.set(id, { resolve: resolve as (result: unknown) => void, reject }); + this.pending.set(id, { method, resolve: resolve as (result: unknown) => void, reject }); // A non-OPEN socket silently discards send() per the WebSocket spec, // which would leave this command pending forever. if (socket.readyState !== WebSocket.OPEN) { @@ -123,7 +137,14 @@ export class CdpConnection { } private handleMessage(data: string): void { - let message: { id?: number; result?: unknown; error?: { message?: string }; method?: string; params?: Record; sessionId?: string }; + let message: { + id?: number; + result?: unknown; + error?: { code?: number; message?: string; data?: string }; + method?: string; + params?: Record; + sessionId?: string; + }; try { message = JSON.parse(data); } catch { @@ -133,8 +154,16 @@ export class CdpConnection { const pending = this.pending.get(message.id); if (!pending) return; this.pending.delete(message.id); - if (message.error) pending.reject(new Error(message.error.message ?? "CDP command failed")); - else pending.resolve(message.result ?? {}); + if (message.error) { + pending.reject( + new CdpProtocolError( + pending.method, + message.error.code ?? -1, + message.error.message ?? "CDP command failed", + message.error.data, + ), + ); + } else pending.resolve(message.result ?? {}); return; } if (typeof message.method === "string") { diff --git a/packages/agent/test/browser-cross-process.live.test.ts b/packages/agent/test/browser-cross-process.live.test.ts new file mode 100644 index 00000000..977737b7 --- /dev/null +++ b/packages/agent/test/browser-cross-process.live.test.ts @@ -0,0 +1,88 @@ +import Kernel from "@onkernel/sdk"; +import { describe, expect, it } from "vitest"; +import type { CuaBrowserAction } from "@onkernel/cua-ai"; +import { BrowserExecutor } from "../src/translator/browser"; +import type { BatchReadResult } from "../src/translator/types"; + +/** + * Live regression for cross-process named-session ref staleness (Defect 2). + * + * Faithfully exercises the process boundary that unit tests can only fake: + * each BrowserExecutor opens its own CDP connection to the same Kernel + * browser, so exported ref state is reconciled against real `loaderId`s and + * real navigations. Gated on CUA_E2E_LIVE=1 + KERNEL_API_KEY so it never + * provisions a browser in a normal test run. + */ +const LIVE = process.env.CUA_E2E_LIVE === "1"; +const KERNEL_API_KEY = process.env.KERNEL_API_KEY; + +const PAGE_A = "https://example.com/"; +const PAGE_B = "https://example.org/"; + +function textOf(reads: BatchReadResult[]): string { + return reads + .filter((read): read is Extract => read.type === "browser_text") + .map((read) => read.text) + .join("\n"); +} + +async function withLiveBrowser(run: (cdpWsUrl: string) => Promise): Promise { + const client = new Kernel({ apiKey: KERNEL_API_KEY! }); + const browser = await client.browsers.create({ stealth: true }); + try { + const cdpWsUrl = browser.cdp_ws_url; + if (!cdpWsUrl) throw new Error("browser has no cdp_ws_url"); + await run(cdpWsUrl); + } finally { + await client.browsers.deleteByID(browser.session_id).catch(() => {}); + } +} + +describe.skipIf(!(LIVE && KERNEL_API_KEY))("BrowserExecutor cross-process document identity (live)", () => { + it( + "reuses a ref across processes on the unchanged document but stales it after a navigation", + async () => { + await withLiveBrowser(async (cdpWsUrl) => { + // Process 1: land on page A, mint a ref, export ref state. + const first = new BrowserExecutor(cdpWsUrl); + await first.execute({ type: "browser_navigate", url: PAGE_A }); + const snapshot = textOf(await first.execute({ type: "browser_snapshot", filter: "interactive" } as CuaBrowserAction)); + const ref = /\[(e\d+)\]/.exec(snapshot)?.[1]; + expect(ref, `expected a minted ref in:\n${snapshot}`).toBeTruthy(); + const state = first.exportRefState(); + expect(state.documents?.length ?? 0).toBeGreaterThan(0); + first.close(); + + // Process 2: same browser, document unchanged -> imported ref resolves. + const second = new BrowserExecutor(cdpWsUrl); + second.importRefState(state); + const scoped = textOf(await second.execute({ type: "browser_snapshot", ref } as CuaBrowserAction)); + expect(scoped.length).toBeGreaterThan(0); + second.close(); + + // Legacy fail-safe: state serialized without document identity cannot be + // verified, so its refs stale even though the document is unchanged — never + // resolving by process-local generation against a possibly-reused node id. + const legacy = new BrowserExecutor(cdpWsUrl); + legacy.importRefState({ ...state, documents: undefined }); + await expect(legacy.execute({ type: "browser_click", ref } as CuaBrowserAction)).rejects.toThrow(/stale/); + legacy.close(); + + // A navigation changes the document after the ref was minted, standing + // in for a click-induced navigation that raced the exporting process. + const navigator = new BrowserExecutor(cdpWsUrl); + await navigator.execute({ type: "browser_navigate", url: PAGE_B }); + navigator.close(); + + // Process 3: imports the pre-navigation state; reconcile against the live + // document detects the change and stales the ref rather than resolving it + // against a different document with possibly-reused backend node ids. + const third = new BrowserExecutor(cdpWsUrl); + third.importRefState(state); + await expect(third.execute({ type: "browser_click", ref } as CuaBrowserAction)).rejects.toThrow(/stale/); + third.close(); + }); + }, + 120_000, + ); +}); diff --git a/packages/agent/test/browser-frame-collection.test.ts b/packages/agent/test/browser-frame-collection.test.ts new file mode 100644 index 00000000..030d3c07 --- /dev/null +++ b/packages/agent/test/browser-frame-collection.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { isExpectedFrameCollectionError } from "../src/translator/browser-frame-collection"; +import { CdpProtocolError } from "../src/translator/cdp"; + +// The allow-list is the sole boundary between a transiently-inaccessible frame +// (retried, then omitted) and a hard, loud FrameCollectionError. Pin every +// accepted protocol string per method so a future edit can't silently tighten +// (dropping a variant would start staling live frames) or loosen it (swallowing +// a genuine protocol bug as "incomplete frame"). Fencing behavior is unchanged +// by this suite — it only documents the accepted set. +describe("isExpectedFrameCollectionError allow-list", () => { + const cases: Array<[string, "DOM.describeNode" | "Accessibility.getFullAXTree", boolean]> = [ + ["Could not find node with given id", "DOM.describeNode", true], + ["No node with given id found", "DOM.describeNode", true], + ["No node with given id found.", "DOM.describeNode", true], + ["Frame with the given id was not found", "Accessibility.getFullAXTree", true], + ["Frame with the given id was not found.", "Accessibility.getFullAXTree", true], + ["No frame for given id found", "Accessibility.getFullAXTree", true], + ["Session with given id not found", "Accessibility.getFullAXTree", true], + ["Target session terminated", "Accessibility.getFullAXTree", true], + // A -32000 error whose message is not on the allow-list must stay unexpected + // so it propagates as a FrameCollectionError instead of being swallowed. + ["Some other protocol failure", "Accessibility.getFullAXTree", false], + ]; + + it.each(cases)("classifies %j on %s as expected=%s", (message, method, expected) => { + const error = new CdpProtocolError(method, -32000, message); + expect(isExpectedFrameCollectionError(error, method)).toBe(expected); + }); + + it("does not accept an allow-listed message reported for a different method", () => { + const describeMessage = new CdpProtocolError("Accessibility.getFullAXTree", -32000, "Could not find node with given id"); + expect(isExpectedFrameCollectionError(describeMessage, "Accessibility.getFullAXTree")).toBe(false); + + const axMessage = new CdpProtocolError("DOM.describeNode", -32000, "Frame with the given id was not found"); + expect(isExpectedFrameCollectionError(axMessage, "DOM.describeNode")).toBe(false); + }); + + it("does not accept a plain Error even when its message is on the allow-list", () => { + expect(isExpectedFrameCollectionError(new Error("No node with given id found"), "DOM.describeNode")).toBe(false); + }); +}); diff --git a/packages/agent/test/browser-ref-lifecycle.test.ts b/packages/agent/test/browser-ref-lifecycle.test.ts new file mode 100644 index 00000000..08cdd681 --- /dev/null +++ b/packages/agent/test/browser-ref-lifecycle.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "vitest"; +import { type RefEntry, RefGenerationLifecycle } from "../src/translator/browser-ref-lifecycle"; + +function frameRef(overrides: Partial = {}): RefEntry { + return { + backendNodeId: 70, + targetId: "TARGET-A", + frameId: "FRAME-X", + sessionId: "session-a", + generation: 0, + role: "button", + name: "Pay", + nth: 0, + cohort: 1, + ...overrides, + }; +} + +// A frame id is expected to belong to exactly one owning target within a +// process. captureFrame throws on a genuine owner change rather than silently +// re-keying the frame under a new target, which would cross-wire refs. This +// pins the diagnostic and confirms the throw leaves the existing frame state +// intact (no partial reassignment). It documents current behavior only. +describe("RefGenerationLifecycle.captureFrame owner invariant", () => { + it("throws when a still-referenced frame is captured under a different target and preserves prior state", () => { + const refs = new Map(); + const lifecycle = new RefGenerationLifecycle(refs); + + const capture = lifecycle.captureFrame("TARGET-A", "FRAME-X"); + lifecycle.retainRef("e1", frameRef({ generation: capture.generation })); + + expect(() => lifecycle.captureFrame("TARGET-B", "FRAME-X")).toThrow( + "frame FRAME-X changed owner from TARGET-A to TARGET-B", + ); + + // The frame stays owned by TARGET-A: the retained ref is still current and + // a re-capture on the original target succeeds, so the throw did not + // re-key the frame or bump its generation. + expect(lifecycle.isRefCurrent(refs.get("e1")!)).toBe(true); + expect(lifecycle.isCurrent(lifecycle.captureFrame("TARGET-A", "FRAME-X"))).toBe(true); + expect(refs.size).toBe(1); + expect(lifecycle.exportGenerations().filter(([key]) => key === "FRAME-X")).toEqual([["FRAME-X", 0]]); + }); +}); + +function mainEntry(targetId: string, generation = 0): Omit { + return { backendNodeId: 10, targetId, frameId: targetId, generation, role: "button", name: "Main", nth: 0, cohort: 1 }; +} + +function childEntry(targetId: string, frameId: string, backendNodeId: number, generation = 0): Omit { + return { backendNodeId, targetId, frameId, generation, role: "button", name: frameId, nth: 0, cohort: 1 }; +} + +// Per-frame document reconciliation is what lets a child-frame or OOPIF change +// stale only that frame's imported refs. These pin the invalidation scope and +// the fail-safe classification of missing/malformed identities directly on the +// lifecycle, independent of any CDP wiring. +describe("RefGenerationLifecycle document reconciliation", () => { + it("stales only the mismatched child frame, preserving the main frame and sibling frames", () => { + const refs = new Map(); + const lifecycle = new RefGenerationLifecycle(refs); + lifecycle.importState( + [ + ["T", 0], + ["FRAME-A", 0], + ["FRAME-B", 0], + ], + [ + ["e1", mainEntry("T")], + ["e2", childEntry("T", "FRAME-A", 21)], + ["e3", childEntry("T", "FRAME-B", 22)], + ], + "T", + [ + ["T", "M0"], + ["FRAME-A", "A0"], + ["FRAME-B", "B0"], + ], + ); + + // FRAME-A's document changed: invalidateFrame, not invalidateTarget. + lifecycle.reconcileDocument("FRAME-A", "T", "A1"); + expect(refs.has("e2")).toBe(false); + // The main frame and the untouched sibling frame keep their refs — proof the + // whole target was not invalidated. + expect(lifecycle.isRefCurrent(refs.get("e1")!)).toBe(true); + expect(lifecycle.isRefCurrent(refs.get("e3")!)).toBe(true); + }); + + it("keeps a child frame whose live document matches its mint-time identity", () => { + const refs = new Map(); + const lifecycle = new RefGenerationLifecycle(refs); + lifecycle.importState( + [["T", 0], ["FRAME-A", 0]], + [["e1", mainEntry("T")], ["e2", childEntry("T", "FRAME-A", 21)]], + "T", + [["T", "M0"], ["FRAME-A", "A0"]], + ); + lifecycle.reconcileDocument("FRAME-A", "T", "A0"); + expect(lifecycle.isRefCurrent(refs.get("e2")!)).toBe(true); + expect(lifecycle.documentOf("FRAME-A", "T")).toBe("A0"); + }); + + it("treats a blank imported or live loaderId as a non-match and fails safe", () => { + const refs = new Map(); + const lifecycle = new RefGenerationLifecycle(refs); + lifecycle.importState([["T", 0]], [["e1", mainEntry("T")]], "T", [["T", ""]]); + // A blank loaderId is unusable, so the frame is unverifiable. + expect(lifecycle.pendingDocuments("T")).toEqual([{ frameKey: "T", verifiable: false }]); + // Even a blank *live* loaderId must never be accepted as a match. + lifecycle.reconcileDocument("T", "T", ""); + expect(refs.has("e1")).toBe(false); + }); + + it("treats duplicate conflicting identities as ambiguous and fails safe", () => { + const refs = new Map(); + const lifecycle = new RefGenerationLifecycle(refs); + lifecycle.importState([["T", 0]], [["e1", mainEntry("T")]], "T", [["T", "M0"], ["T", "M9"]]); + expect(lifecycle.pendingDocuments("T")).toEqual([{ frameKey: "T", verifiable: false }]); + // Matching one of the two conflicting identities must not resolve the ref. + lifecycle.reconcileDocument("T", "T", "M0"); + expect(refs.has("e1")).toBe(false); + }); + + it("ignores malformed document entries, leaving those frames unverifiable", () => { + const refs = new Map(); + const lifecycle = new RefGenerationLifecycle(refs); + const malformed = [["T"], "nope", ["T", 5], [42, "x"]] as unknown as Array<[string, string]>; + lifecycle.importState([["T", 0]], [["e1", mainEntry("T")]], "T", malformed); + expect(lifecycle.pendingDocuments("T")).toEqual([{ frameKey: "T", verifiable: false }]); + lifecycle.reconcileDocument("T", "T", "whatever"); + expect(refs.has("e1")).toBe(false); + }); + + it("never fabricates identity on re-export but upgrades once a live document is recorded", () => { + const refs = new Map(); + const lifecycle = new RefGenerationLifecycle(refs); + lifecycle.importState([["T", 0]], [["e1", mainEntry("T")]], "T", undefined); + // Legacy state stays honestly legacy: nothing to export for an unverifiable frame. + expect(lifecycle.exportDocuments(new Set(["T"]))).toEqual([]); + lifecycle.recordDocument("T", "T", "M0"); + expect(lifecycle.exportDocuments(new Set(["T"]))).toEqual([["T", "M0"]]); + }); +}); diff --git a/packages/agent/test/cdp.test.ts b/packages/agent/test/cdp.test.ts index 6fe84fd8..8bc1edff 100644 --- a/packages/agent/test/cdp.test.ts +++ b/packages/agent/test/cdp.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { CdpConnection } from "../src/translator/cdp"; +import { CdpConnection, CdpProtocolError } from "../src/translator/cdp"; class FakeSocket { static instances: FakeSocket[] = []; @@ -66,4 +66,23 @@ describe("CdpConnection send", () => { FakeSocket.instances[0]!.close(); await expect(pending).rejects.toThrow(/closed/); }); + + it("preserves command and protocol error metadata", async () => { + vi.stubGlobal("WebSocket", Object.assign(FakeSocket, { OPEN: 1 })); + const cdp = new CdpConnection("wss://fake.test/cdp"); + const pending = cdp.send("Accessibility.getFullAXTree", { frameId: "F1" }, "session-1"); + await vi.waitFor(() => expect(FakeSocket.instances[0]!.sent).toHaveLength(1)); + FakeSocket.instances[0]!.emit("message", { + data: JSON.stringify({ id: 1, error: { code: -32000, message: "Frame with the given id was not found.", data: "gone" } }), + }); + + const error = await pending.catch((cause: unknown) => cause); + expect(error).toBeInstanceOf(CdpProtocolError); + expect(error).toMatchObject({ + method: "Accessibility.getFullAXTree", + code: -32000, + protocolMessage: "Frame with the given id was not found.", + data: "gone", + }); + }); }); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index b345b62f..0a14d4e8 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -3,7 +3,8 @@ import sharp from "sharp"; import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; import { BrowserExecutor } from "../src/translator/browser"; -import type { CdpConnection } from "../src/translator/cdp"; +import type { BrowserRefState } from "../src/translator/browser-ref-lifecycle"; +import { CdpProtocolError, type CdpConnection } from "../src/translator/cdp"; import { buildCuaComputerTools } from "../src/tools"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; import type { BatchReadResult } from "../src/translator/types"; @@ -106,11 +107,29 @@ function createFakeCdp(initialNodes: unknown[] = []) { const listeners: Array<(event: FakeCdpEvent) => void> = []; let nodes = initialNodes as Array<{ backendDOMNodeId?: number }>; let cursorBackendIds: number[] = []; + let loaderId: string | undefined = "L0"; const sessionTrees = new Map>(); const frameTrees = new Map>(); const iframeFrameIds = new Map(); - const autoAttachFrames: Array<{ targetId: string; sessionId: string }> = []; - const failMethods = new Set(); + const boxModels = new Map(); + const frameLoaderIds = new Map(); + const oopifFrameKeys = new Set(); + const oopifSessionFrames = new Map(); + const autoAttachFrames: Array<{ targetId: string; sessionId: string; delayMs?: number }> = []; + // A same-process child frame's document loaderId defaults to a stable value so + // it verifies unchanged across processes; setFrameLoaderId overrides it. + const loaderFor = (frameKey: string) => frameLoaderIds.get(frameKey) ?? (frameKey === "TARGET-1" ? loaderId : "L0"); + const sameProcessChildFrames = () => { + const ids = new Set(); + for (const frameId of iframeFrameIds.values()) if (!oopifFrameKeys.has(frameId)) ids.add(frameId); + return [...ids]; + }; + const methodFailures = new Map(); + let failureProvider: ((method: string, params: Record, sessionId?: string) => Error | undefined) | undefined; + let axRead = 0; + let targetRead = 0; + let onAxRead: ((read: number, params: Record, sessionId?: string) => void) | undefined; + let targetProvider: ((read: number) => Array<{ targetId: string; type: string; title: string; url: string }>) | undefined; const emit = (event: FakeCdpEvent) => { for (const listener of listeners) listener(event); }; @@ -129,17 +148,26 @@ function createFakeCdp(initialNodes: unknown[] = []) { }, send: async (method: string, params: Record = {}, sessionId?: string) => { sent.push({ method, params, sessionId }); - if (failMethods.has(method)) throw new Error(`${method} rejected`); + const failure = methodFailures.get(method) ?? failureProvider?.(method, params, sessionId); + if (failure) throw failure; switch (method) { - case "Accessibility.getFullAXTree": - return { nodes: treeFor(sessionId, params.frameId) }; + case "Accessibility.getFullAXTree": { + const tree = treeFor(sessionId, params.frameId); + onAxRead?.(++axRead, params, sessionId); + return { nodes: tree }; + } case "Target.setAutoAttach": for (const frame of autoAttachFrames.splice(0)) { - emit({ - method: "Target.attachedToTarget", - params: { sessionId: frame.sessionId, targetInfo: { targetId: frame.targetId, type: "iframe" } }, - sessionId, - }); + const attach = () => + emit({ + method: "Target.attachedToTarget", + params: { sessionId: frame.sessionId, targetInfo: { targetId: frame.targetId, type: "iframe" } }, + sessionId, + }); + // A delayed attach models an OOPIF whose session surfaces only after + // setAutoAttach returns, exercising the reconcile's bounded wait. + if (frame.delayMs) setTimeout(attach, frame.delayMs); + else attach(); } return {}; case "DOM.scrollIntoViewIfNeeded": @@ -147,7 +175,32 @@ function createFakeCdp(initialNodes: unknown[] = []) { return {}; case "DOM.getBoxModel": requireBackendId(params.backendNodeId, sessionId); - return { model: { content: [0, 0, 10, 0, 10, 10, 0, 10] } }; + return { model: { content: boxModels.get(params.backendNodeId as number) ?? [0, 0, 10, 0, 10, 10, 0, 10] } }; + case "DOM.getFrameOwner": { + for (const [backendNodeId, frameId] of iframeFrameIds) { + if (frameId === params.frameId) return { backendNodeId }; + } + throw new Error("Frame with the given id was not found."); + } + case "Page.getFrameTree": { + // An OOPIF's own session reports itself as the root frame with its own loaderId. + const oopifFrame = sessionId ? oopifSessionFrames.get(sessionId) : undefined; + if (oopifFrame) { + const oopifLoader = loaderFor(oopifFrame); + return { frameTree: { frame: { id: oopifFrame, ...(oopifLoader !== undefined ? { loaderId: oopifLoader } : {}) } } }; + } + // The page session reports the main frame plus its same-process children. + const childFrames = sameProcessChildFrames().map((id) => { + const childLoader = loaderFor(id); + return { frame: { id, ...(childLoader !== undefined ? { loaderId: childLoader } : {}) } }; + }); + return { + frameTree: { + frame: { id: "TARGET-1", ...(loaderId !== undefined ? { loaderId } : {}) }, + ...(childFrames.length ? { childFrames } : {}), + }, + }; + } case "DOM.resolveNode": requireBackendId(params.backendNodeId, sessionId); return { object: { objectId: "node-obj" } }; @@ -170,7 +223,8 @@ function createFakeCdp(initialNodes: unknown[] = []) { return {}; } }, - pageTargets: async () => [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], + pageTargets: async () => + targetProvider?.(++targetRead) ?? [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], attachToTarget: async () => "session-1", createTarget: async () => "TARGET-2", close: () => {}, @@ -190,11 +244,33 @@ function createFakeCdp(initialNodes: unknown[] = []) { const setIframeFrame = (backendNodeId: number, frameId: string) => { iframeFrameIds.set(backendNodeId, frameId); }; - const addAutoAttachFrame = (frame: { targetId: string; sessionId: string }) => { + const setBoxModel = (backendNodeId: number, content: number[]) => { + boxModels.set(backendNodeId, content); + }; + const setLoaderId = (id: string | undefined) => { + loaderId = id; + }; + const setFrameLoaderId = (frameKey: string, id: string) => { + frameLoaderIds.set(frameKey, id); + }; + const addAutoAttachFrame = (frame: { targetId: string; sessionId: string; delayMs?: number }) => { + // Registering an OOPIF: its session is authoritative for its own document, + // and it is excluded from the page session's same-process child frames. + oopifFrameKeys.add(frame.targetId); + oopifSessionFrames.set(frame.sessionId, frame.targetId); autoAttachFrames.push(frame); }; - const failOn = (method: string) => { - failMethods.add(method); + const failOn = (method: string, error = new Error(`${method} rejected`)) => { + methodFailures.set(method, error); + }; + const setFailureProvider = (provider: typeof failureProvider) => { + failureProvider = provider; + }; + const setAxReadHook = (hook: typeof onAxRead) => { + onAxRead = hook; + }; + const setTargetProvider = (provider: typeof targetProvider) => { + targetProvider = provider; }; return { sent, @@ -204,8 +280,14 @@ function createFakeCdp(initialNodes: unknown[] = []) { setSessionTree, setFrameTree, setIframeFrame, + setBoxModel, + setLoaderId, + setFrameLoaderId, addAutoAttachFrame, failOn, + setFailureProvider, + setAxReadHook, + setTargetProvider, cdp: fake as unknown as CdpConnection, }; } @@ -245,6 +327,12 @@ async function snapshotText(executor: BrowserExecutor, action: Record { expect(sent.some((cmd) => cmd.method === "DOM.describeNode")).toBe(true); expect(sent.some((cmd) => cmd.method === "Runtime.releaseObjectGroup")).toBe(true); }); + + it("does not scan cursor metadata for find", async () => { + const { cdp, failOn, sent } = createFakeCdp(BUTTON_TREE); + failOn("Runtime.evaluate"); + const executor = new BrowserExecutor(cdp); + const results = await executor.execute({ type: "browser_find", query: "save button" } as CuaBrowserAction); + expect((results[0] as { text: string }).text).toContain('button "Save" [e1]'); + expect(sent.some((command) => command.method === "Runtime.evaluate")).toBe(false); + }); }); describe("BrowserExecutor dialog guard", () => { @@ -687,6 +784,27 @@ describe("BrowserExecutor iframe stitching", () => { return fake; }; + const importOopifRefWithoutOwner = async () => { + const source = setupOopif(); + const mint = new BrowserExecutor(source.cdp); + await snapshotText(mint); + const state = mint.exportRefState(); + + const fake = createFakeCdp(OOPIF_PAGE); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.setSessionTree("session-oop", OOPIF_CHILD); + const executor = new BrowserExecutor(fake.cdp); + executor.importRefState(state); + // Simulate an OOPIF session attaching without a parent session id. The + // imported ref state must still identify its owning page for invalidation. + fake.emit({ + method: "Target.attachedToTarget", + params: { sessionId: "session-oop", targetInfo: { targetId: "FRAME-OOP", type: "iframe" } }, + }); + expect([...refsOf(executor).keys()].sort()).toEqual(["e1", "e2", "e3"]); + return { executor, fake }; + }; + it("resolves an OOPIF ref's node through the child session but dispatches input on the page session", async () => { const { cdp, sent } = setupOopif(); const executor = new BrowserExecutor(cdp); @@ -701,6 +819,72 @@ describe("BrowserExecutor iframe stitching", () => { expect(pressed?.sessionId).toBe("session-1"); }); + it("shifts an OOPIF ref click by the frame owner's offset so it lands on the intended element", async () => { + const fake = setupOopif(); + // The iframe (owner backend node 50) sits at (100, 200) in the page; the + // child's own renderer reports the Pay button centered at frame-local (5, 5). + fake.setBoxModel(50, [100, 200, 110, 200, 110, 210, 100, 210]); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain(' button "Pay" [e3]'); + + await executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); + const owner = fake.sent.find((cmd) => cmd.method === "DOM.getFrameOwner" && cmd.params.frameId === "FRAME-OOP"); + expect(owner?.sessionId).toBe("session-1"); + const pressed = fake.sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed"); + expect(pressed?.sessionId).toBe("session-1"); + expect([pressed?.params.x, pressed?.params.y]).toEqual([105, 205]); + }); + + it("shifts an OOPIF ref hover by the frame owner's offset", async () => { + const fake = setupOopif(); + fake.setBoxModel(50, [100, 200, 110, 200, 110, 210, 100, 210]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + await executor.execute({ type: "browser_hover", ref: "e3" } as CuaBrowserAction); + const moved = fake.sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mouseMoved"); + expect(moved?.sessionId).toBe("session-1"); + expect([moved?.params.x, moved?.params.y]).toEqual([105, 205]); + }); + + it("does not offset a main-frame ref click even when an OOPIF owner has an offset", async () => { + const fake = setupOopif(); + fake.setBoxModel(50, [100, 200, 110, 200, 110, 210, 100, 210]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + // e1 is the top-level "Top" button (backend node 40): read through the page + // session, so its quads are already top-level and must not be shifted. + await executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(fake.sent.some((cmd) => cmd.method === "DOM.getFrameOwner")).toBe(false); + const pressed = fake.sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed"); + expect([pressed?.params.x, pressed?.params.y]).toEqual([5, 5]); + }); + + it("does not offset a same-process iframe ref click", async () => { + const tree = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(tree); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Embed", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "button", name: "Inside", backendDOMNodeId: 60, parentId: "f1" }), + ]); + // Even if the iframe element has an offset, a same-process frame is read + // through the page session and its quads are already top-level. + fake.setBoxModel(50, [100, 200, 110, 200, 110, 210, 100, 210]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + await executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); + expect(fake.sent.some((cmd) => cmd.method === "DOM.getFrameOwner")).toBe(false); + const pressed = fake.sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed"); + expect([pressed?.params.x, pressed?.params.y]).toEqual([5, 5]); + }); + it("invalidates a frame target's refs when a subframe inside it navigates", async () => { const { cdp, emit } = setupOopif(); const executor = new BrowserExecutor(cdp); @@ -710,6 +894,25 @@ describe("BrowserExecutor iframe stitching", () => { await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); + it.each([ + { + label: "Page.frameNavigated", + event: { method: "Page.frameNavigated", params: { frame: { id: "FRAME-OOP" } }, sessionId: "session-oop" }, + }, + { + label: "Page.frameDetached", + event: { method: "Page.frameDetached", params: { frameId: "FRAME-OOP", reason: "swap" }, sessionId: "session-oop" }, + }, + { + label: "Target.detachedFromTarget", + event: { method: "Target.detachedFromTarget", params: { sessionId: "session-oop" } }, + }, + ] as const)("drops imported OOPIF refs when $label fires before owner mapping is known", async ({ event }) => { + const { fake, executor } = await importOopifRefWithoutOwner(); + fake.emit(event); + expect([...refsOf(executor).keys()].sort()).toEqual(["e1", "e2"]); + }); + it("finds elements inside stitched iframes", async () => { const { cdp, sent } = setupOopif(); const executor = new BrowserExecutor(cdp); @@ -736,6 +939,236 @@ describe("BrowserExecutor iframe stitching", () => { const text = await snapshotText(executor); expect(text).toContain('button "Pay" [e'); }); + + it("invalidates and releases a same-process frame when it detaches and rotates", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "old", role: "button", name: "Old", backendDOMNodeId: 60 })]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + fake.emit({ method: "Page.frameDetached", params: { frameId: "FRAME-SP", reason: "swap" }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction)).rejects.toThrow(/stale/); + + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "new", role: "button", name: "New", backendDOMNodeId: 61 })]); + expect(await snapshotText(executor)).toContain('button "New" [e'); + fake.emit({ method: "Page.frameDetached", params: { frameId: "FRAME-SP", reason: "remove" }, sessionId: "session-1" }); + expect(executor.exportRefState().generations.map(([key]) => key)).toEqual(["TARGET-1"]); + }); + + it("does not retain generation state for rotating unreferenced frames", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + const executor = new BrowserExecutor(fake.cdp); + for (let index = 0; index < 25; index += 1) { + const frameId = `FRAME-${index}`; + fake.setIframeFrame(50, frameId); + fake.setFrameTree(frameId, [ax({ nodeId: `root-${index}`, role: "RootWebArea", name: "Embed" })]); + await snapshotText(executor); + } + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("does not retain a frame generation when its AX fetch is transiently inaccessible", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFailureProvider((method, params) => + method === "Accessibility.getFullAXTree" && params.frameId === "FRAME-SP" + ? new CdpProtocolError(method, -32000, "Frame with the given id was not found.") + : undefined, + ); + const executor = new BrowserExecutor(fake.cdp); + expect(await snapshotText(executor)).toContain("Iframe [e1]"); + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("propagates unexpected frame protocol failures with frame diagnostics and no registration leak", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFailureProvider((method, params) => + method === "Accessibility.getFullAXTree" && params.frameId === "FRAME-SP" ? new Error("decoder exploded") : undefined, + ); + const executor = new BrowserExecutor(fake.cdp); + await expect(snapshotText(executor)).rejects.toThrow( + /Failed to collect iframe FRAME-SP at backend node 50 during Accessibility\.getFullAXTree: decoder exploded/, + ); + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("propagates malformed frame trees as indexed collection failures", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", null as unknown as unknown[]); + const executor = new BrowserExecutor(fake.cdp); + await expect(snapshotText(executor)).rejects.toThrow(/FRAME-SP.*building the accessibility index/); + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("propagates unexpected iframe description failures instead of treating them as inaccessible", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.failOn("DOM.describeNode", new CdpProtocolError("DOM.describeNode", -32603, "Internal error")); + const executor = new BrowserExecutor(fake.cdp); + await expect(snapshotText(executor)).rejects.toThrow(/backend node 50 during DOM\.describeNode.*Internal error/); + expect(fake.sent.filter((command) => command.method === "DOM.describeNode")).toHaveLength(1); + }); + + it("retries a transiently omitted owning frame instead of staling its scoped ref", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Pay", backendDOMNodeId: 40, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + let omitted = false; + fake.setFailureProvider((method) => { + if (method !== "DOM.describeNode" || omitted) return undefined; + omitted = true; + return new CdpProtocolError(method, -32000, "Could not find node with given id"); + }); + await expect(snapshotText(executor, { ref: "e3" })).resolves.toContain('button "Pay"'); + expect(fake.sent.filter((command) => command.method === "DOM.describeNode")).toHaveLength(3); + }); + + it("reports a persistently omitted owning frame as unverifiable rather than stale", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + fake.failOn("DOM.describeNode", new CdpProtocolError("DOM.describeNode", -32000, "Could not find node with given id")); + await expect(snapshotText(executor, { ref: "e2" })).rejects.toThrow(/could not verify ref e2/i); + expect(fake.sent.filter((command) => command.method === "DOM.describeNode")).toHaveLength(4); + }); +}); + +describe("BrowserExecutor observation fencing", () => { + const STALE_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Old", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Stale", backendDOMNodeId: 41, parentId: "1" }), + ]; + const STABLE_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "New", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Stable", backendDOMNodeId: 42, parentId: "1" }), + ]; + + it.each(["browser_snapshot", "browser_find"] as const)("discards a navigated AX collection for %s before minting refs", async (type) => { + const fake = createFakeCdp(STALE_TREE); + fake.setAxReadHook((read, params) => { + if (read !== 1 || params.frameId) return; + fake.setNodes(STABLE_TREE); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + const results = await executor.execute( + (type === "browser_snapshot" ? { type } : { type, query: "stable button" }) as CuaBrowserAction, + ); + const text = (results[0] as { text: string }).text; + expect(text).toContain('button "Stable" [e1]'); + expect(text).not.toContain("Stale"); + expect([...refsOf(executor).keys()]).toEqual(["e1"]); + }); + + it.each([ + ["url", { title: "Page", url: "https://b.test/" }], + ["title", { title: "Changed", url: "https://a.test/" }], + ] as const)("retries when target %s changes across collection", async (_field, changed) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setTargetProvider((read) => [ + { targetId: "TARGET-1", type: "page", ...(read < 3 ? { title: "Page", url: "https://a.test/" } : changed) }, + ]); + const executor = new BrowserExecutor(fake.cdp); + expect(await snapshotText(executor)).toContain('button "Save" [e1]'); + expect(fake.sent.filter((command) => command.method === "Accessibility.getFullAXTree")).toHaveLength(2); + }); + + it("retries when a stitched frame changes during its AX read", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const oldChild = [ax({ nodeId: "f1", role: "button", name: "Old child", backendDOMNodeId: 60 })]; + const newChild = [ax({ nodeId: "f1", role: "button", name: "New child", backendDOMNodeId: 61 })]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", oldChild); + fake.setAxReadHook((_read, params) => { + if (params.frameId !== "FRAME-SP") return; + fake.setAxReadHook(undefined); + fake.setFrameTree("FRAME-SP", newChild); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-SP", parentId: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "New child"'); + expect(text).not.toContain("Old child"); + }); + + it("retries when a generation-zero OOPIF detaches after its AX read", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setSessionTree("session-oop", [ax({ nodeId: "f1", role: "button", name: "Detached", backendDOMNodeId: 60 })]); + fake.setFrameTree("FRAME-OOP", [ax({ nodeId: "f1", role: "button", name: "Current", backendDOMNodeId: 61 })]); + fake.setAxReadHook((_read, _params, sessionId) => { + if (sessionId !== "session-oop") return; + fake.setAxReadHook(undefined); + fake.emit({ method: "Target.detachedFromTarget", params: { sessionId: "session-oop" } }); + }); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "Current"'); + expect(text).not.toContain("Detached"); + }); + + it.each(["browser_snapshot", "browser_find"] as const)("fails %s after three changed collections without minting refs", async (type) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setAxReadHook((_read, params) => { + if (!params.frameId) fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + await expect( + executor.execute((type === "browser_snapshot" ? { type } : { type, query: "save" }) as CuaBrowserAction), + ).rejects.toThrow(/observation changed/i); + expect(fake.sent.filter((command) => command.method === "Accessibility.getFullAXTree")).toHaveLength(3); + expect(refsOf(executor).size).toBe(0); + }); }); describe("navigation tool grounding frame", () => { @@ -814,6 +1247,27 @@ describe("BrowserExecutor ref state export/import", () => { expect(pressed).toBeDefined(); }); + it("restores a same-process frame ref with its owning generation", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const firstFake = createFakeCdp(root); + firstFake.setIframeFrame(50, "FRAME-SP"); + firstFake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const secondFake = createFakeCdp(root); + secondFake.setIframeFrame(50, "FRAME-SP"); + secondFake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await second.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); + expect(secondFake.sent.some((command) => command.method === "DOM.getBoxModel" && command.params.backendNodeId === 70)).toBe(true); + }); + it("keeps minting unique refs after import and invalidates imported refs on navigation", async () => { const first = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); await snapshotText(first); @@ -828,3 +1282,375 @@ describe("BrowserExecutor ref state export/import", () => { await expect(second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); }); + +describe("BrowserExecutor cross-process document identity", () => { + it("records the main-frame document identity in exported ref state", async () => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setLoaderId("L0"); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + const state = executor.exportRefState(); + expect(state.generations).toEqual([["TARGET-1", 0]]); + expect(state.documents).toEqual([["TARGET-1", "L0"]]); + }); + + it("stales an imported ref when the document changed across processes", async () => { + const firstFake = createFakeCdp(BUTTON_TREE); + firstFake.setLoaderId("L0"); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + expect(state.documents).toEqual([["TARGET-1", "L0"]]); + + // A new process attaches to a browser whose document has since changed + // (e.g. a click-induced navigation that raced the first process's exit). + const secondFake = createFakeCdp(BUTTON_TREE); + secondFake.setLoaderId("L1"); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await expect(second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + expect(refsOf(second).size).toBe(0); + }); + + it("keeps an imported ref usable when the document is unchanged across processes", async () => { + const firstFake = createFakeCdp(BUTTON_TREE); + firstFake.setLoaderId("L0"); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const secondFake = createFakeCdp(BUTTON_TREE); + secondFake.setLoaderId("L0"); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + const pressed = secondFake.sent.find((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed"); + expect(pressed).toBeDefined(); + }); + + // browser_fill / browser_scroll_to resolve their ref without first attaching, so the + // imported-document reconcile (which runs on attach) must be forced ahead of the resolve; + // otherwise a changed document across the process boundary silently mis-targets a reused + // backend node id. These cover the two ref-consuming CLI actions that click/hover do not. + it("stales an imported ref on browser_fill when the document changed across processes", async () => { + const firstFake = createFakeCdp(BUTTON_TREE); + firstFake.setLoaderId("L0"); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const secondFake = createFakeCdp(BUTTON_TREE); + secondFake.setLoaderId("L1"); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await expect(second.execute({ type: "browser_fill", ref: "e1", value: "x" } as CuaBrowserAction)).rejects.toThrow(/stale/); + expect(refsOf(second).size).toBe(0); + expect(secondFake.sent.some((cmd) => cmd.method === "Runtime.callFunctionOn")).toBe(false); + }); + + it("keeps browser_fill usable when the document is unchanged across processes", async () => { + const firstFake = createFakeCdp(BUTTON_TREE); + firstFake.setLoaderId("L0"); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const secondFake = createFakeCdp(BUTTON_TREE); + secondFake.setLoaderId("L0"); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await second.execute({ type: "browser_fill", ref: "e1", value: "x" } as CuaBrowserAction); + expect(secondFake.sent.some((cmd) => cmd.method === "Runtime.callFunctionOn")).toBe(true); + }); + + it("stales an imported ref on browser_scroll_to when the document changed across processes", async () => { + const firstFake = createFakeCdp(BUTTON_TREE); + firstFake.setLoaderId("L0"); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const secondFake = createFakeCdp(BUTTON_TREE); + secondFake.setLoaderId("L1"); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await expect(second.execute({ type: "browser_scroll_to", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + expect(refsOf(second).size).toBe(0); + expect(secondFake.sent.some((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded")).toBe(false); + }); + + it("carries the document identity through an invocation that never re-attaches the target", async () => { + const firstFake = createFakeCdp(BUTTON_TREE); + firstFake.setLoaderId("L0"); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + + // Middle invocation imports and re-exports without ever attaching (e.g. `cua url`). + const middleFake = createFakeCdp(BUTTON_TREE); + middleFake.setLoaderId("L0"); + const middle = new BrowserExecutor(middleFake.cdp); + middle.importRefState(state); + const relayed = middle.exportRefState(); + expect(relayed.documents).toEqual([["TARGET-1", "L0"]]); + + // The next process still catches the changed document. + const lastFake = createFakeCdp(BUTTON_TREE); + lastFake.setLoaderId("L1"); + const last = new BrowserExecutor(lastFake.cdp); + last.importRefState(relayed); + await expect(last.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); + + // Legacy state predates document identity; generation alone is process-local + // and cannot prove the document is unchanged, so such a ref must fail safe + // (stale) rather than resolve against a possibly-reused backend node id. This + // asserts staling even when the live document is unchanged (L0 == L0), because + // the exporter recorded no identity to verify against. + it("stales an imported legacy ref that carries no document identity", async () => { + const firstFake = createFakeCdp(BUTTON_TREE); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + // Strip the identity to model state serialized before this field existed. + delete (state as { documents?: unknown }).documents; + + const secondFake = createFakeCdp(BUTTON_TREE); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await expect(second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + expect(refsOf(second).size).toBe(0); + expect(secondFake.sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent")).toBe(false); + + // A fresh snapshot re-mints refs and records identity, so the next export + // upgrades naturally — legacy state auto-heals forward through normal use. + expect(await snapshotText(second)).toContain('button "Save" [e2]'); + expect(second.exportRefState().documents).toEqual([["TARGET-1", "L0"]]); + }); +}); + +// Per-frame document identity: a ref carries the loaderId of its *owning* frame +// (the page target for main-frame refs, the child frame id for same-process +// iframes and OOPIFs). A later process reconciles each frame against the live +// browser before any ref resolves, so a child-frame or OOPIF navigation stales +// only that frame's refs — even when the main-frame loaderId is unchanged. +describe("BrowserExecutor cross-process frame document identity", () => { + const OOPIF_PAGE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Top", backendDOMNodeId: 40, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const OOPIF_CHILD = [ + ax({ nodeId: "f1", role: "RootWebArea", name: "Widget", childIds: ["f2"] }), + ax({ nodeId: "f2", role: "button", name: "Pay", backendDOMNodeId: 70, parentId: "f1" }), + ]; + // A page embedding one cross-origin iframe. `main`/`oopif` pin each frame's + // live document loaderId. `attach: false` models an OOPIF that is gone: no + // session surfaces and it is absent from the page frame tree. + const oopifProcess = (opts: { main: string; oopif?: string; attach?: boolean; attachDelayMs?: number }) => { + const fake = createFakeCdp(OOPIF_PAGE); + fake.setLoaderId(opts.main); + fake.setSessionTree("session-oop", OOPIF_CHILD); + if (opts.attach !== false) { + fake.setIframeFrame(50, "FRAME-OOP"); + if (opts.oopif !== undefined) fake.setFrameLoaderId("FRAME-OOP", opts.oopif); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop", ...(opts.attachDelayMs ? { delayMs: opts.attachDelayMs } : {}) }); + } + return fake; + }; + const mintOopifState = () => { + const fake = oopifProcess({ main: "M0", oopif: "O0" }); + const executor = new BrowserExecutor(fake.cdp); + return snapshotText(executor).then((text) => { + expect(text).toContain('button "Top" [e1]'); + expect(text).toContain('Iframe [e2]'); + expect(text).toContain('button "Pay" [e3]'); + return executor.exportRefState(); + }); + }; + + it("stales only OOPIF refs on an OOPIF-only navigation with the main loader unchanged", async () => { + const state = await mintOopifState(); + expect(state.documents).toEqual( + expect.arrayContaining([ + ["TARGET-1", "M0"], + ["FRAME-OOP", "O0"], + ]), + ); + + const importFake = oopifProcess({ main: "M0", oopif: "O1" }); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + await expect(second.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); + // Only the OOPIF's refs were dropped; the main-frame "Top" and parent + // Iframe refs survive. + expect([...refsOf(second).keys()].sort()).toEqual(["e1", "e2"]); + await second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(importFake.sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed")).toBe(true); + }); + + it("keeps every ref when no frame's document changed across processes", async () => { + const state = await mintOopifState(); + const importFake = oopifProcess({ main: "M0", oopif: "O0" }); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + await second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + await second.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); + const oopifPress = importFake.sent.find( + (cmd) => cmd.method === "DOM.getBoxModel" && cmd.params.backendNodeId === 70 && cmd.sessionId === "session-oop", + ); + expect(oopifPress).toBeDefined(); + }); + + it("reconciles an OOPIF whose session auto-attaches only after setAutoAttach returns", async () => { + const state = await mintOopifState(); + // The OOPIF's attachedToTarget is delayed past attach()'s setAutoAttach, so + // the reconcile must bounded-wait for the session before reading its loader. + const importFake = oopifProcess({ main: "M0", oopif: "O0", attachDelayMs: 40 }); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + await second.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); + const resolved = importFake.sent.find( + (cmd) => cmd.method === "DOM.getBoxModel" && cmd.params.backendNodeId === 70 && cmd.sessionId === "session-oop", + ); + expect(resolved).toBeDefined(); + }); + + it("stales an OOPIF ref whose frame is gone (no session, absent from the page tree) but keeps the main frame", async () => { + const state = await mintOopifState(); + const importFake = oopifProcess({ main: "M0", attach: false }); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + await expect(second.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(importFake.sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed")).toBe(true); + }); + + it.each(["browser_fill", "browser_scroll_to"] as const)( + "reconciles the OOPIF document before %s resolves its ref", + async (type) => { + const state = await mintOopifState(); + const importFake = oopifProcess({ main: "M0", oopif: "O1" }); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + const action = (type === "browser_fill" ? { type, ref: "e3", value: "x" } : { type, ref: "e3" }) as CuaBrowserAction; + await expect(second.execute(action)).rejects.toThrow(/stale/); + // The changed OOPIF document stales the ref before any mutation touches it. + expect(importFake.sent.some((cmd) => cmd.method === "Runtime.callFunctionOn")).toBe(false); + expect(importFake.sent.some((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded")).toBe(false); + }, + ); + + it("carries per-frame identities through an intermediate process that never re-attaches", async () => { + const state = await mintOopifState(); + // Middle invocation imports and re-exports without ever attaching. + const middleFake = oopifProcess({ main: "M0", oopif: "O0" }); + const middle = new BrowserExecutor(middleFake.cdp); + middle.importRefState(state); + const relayed = middle.exportRefState(); + expect(relayed.documents).toEqual( + expect.arrayContaining([ + ["TARGET-1", "M0"], + ["FRAME-OOP", "O0"], + ]), + ); + + // The final process still catches the OOPIF-only change through the relay. + const lastFake = oopifProcess({ main: "M0", oopif: "O1" }); + const last = new BrowserExecutor(lastFake.cdp); + last.importRefState(relayed); + await expect(last.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); + await last.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(lastFake.sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed")).toBe(true); + }); + + it("reads the frame tree at most once for the page and once per pending OOPIF", async () => { + const state = await mintOopifState(); + const importFake = oopifProcess({ main: "M0", oopif: "O0" }); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + await second.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); + const pageTrees = importFake.sent.filter((cmd) => cmd.method === "Page.getFrameTree" && cmd.sessionId === "session-1"); + const oopifTrees = importFake.sent.filter((cmd) => cmd.method === "Page.getFrameTree" && cmd.sessionId === "session-oop"); + expect(pageTrees).toHaveLength(1); + expect(oopifTrees).toHaveLength(1); + }); + + const TWO_FRAME_PAGE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 51, parentId: "1" }), + ]; + const FRAME_A_TREE = [ + ax({ nodeId: "a1", role: "RootWebArea", name: "A", childIds: ["a2"] }), + ax({ nodeId: "a2", role: "button", name: "PayA", backendDOMNodeId: 60, parentId: "a1" }), + ]; + const FRAME_B_TREE = [ + ax({ nodeId: "b1", role: "RootWebArea", name: "B", childIds: ["b2"] }), + ax({ nodeId: "b2", role: "button", name: "PayB", backendDOMNodeId: 61, parentId: "b1" }), + ]; + const twoFrameProcess = (a: string, b: string) => { + const fake = createFakeCdp(TWO_FRAME_PAGE); + fake.setIframeFrame(50, "FRAME-A"); + fake.setIframeFrame(51, "FRAME-B"); + fake.setFrameTree("FRAME-A", FRAME_A_TREE); + fake.setFrameTree("FRAME-B", FRAME_B_TREE); + fake.setFrameLoaderId("FRAME-A", a); + fake.setFrameLoaderId("FRAME-B", b); + return fake; + }; + + it("stales only the changed same-process child frame, keeping its sibling and the main frame", async () => { + const mintFake = twoFrameProcess("A0", "B0"); + const mint = new BrowserExecutor(mintFake.cdp); + const minted = await snapshotText(mint); + const payA = refFor(minted, "PayA"); + const payB = refFor(minted, "PayB"); + const state = mint.exportRefState(); + expect(state.documents).toEqual( + expect.arrayContaining([ + ["FRAME-A", "A0"], + ["FRAME-B", "B0"], + ]), + ); + + // Only FRAME-A's document changed across the boundary. + const importFake = twoFrameProcess("A1", "B0"); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + await expect(second.execute({ type: "browser_click", ref: payA } as CuaBrowserAction)).rejects.toThrow(/stale/); + // The sibling frame's ref still resolves against its unchanged document. + await second.execute({ type: "browser_click", ref: payB } as CuaBrowserAction); + expect(importFake.sent.some((cmd) => cmd.method === "DOM.getBoxModel" && cmd.params.backendNodeId === 61)).toBe(true); + }); + + it("fails safe for a partial state missing one frame's identity, even when nothing changed", async () => { + const state = await mintOopifState(); + // Drop just the OOPIF frame's identity, as an older/partial writer might. + state.documents = state.documents!.filter(([frameKey]) => frameKey !== "FRAME-OOP"); + + const importFake = oopifProcess({ main: "M0", oopif: "O0" }); + const second = new BrowserExecutor(importFake.cdp); + second.importRefState(state); + + // Unverifiable ⇒ stale, even though the live OOPIF document is unchanged. + await expect(second.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); + // The frame that *does* carry identity is preserved. + await second.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction); + expect(importFake.sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent" && cmd.params.type === "mousePressed")).toBe(true); + }); + + it("rejects ref state serialized by a newer, unknown identity scheme", () => { + const fake = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(fake.cdp); + const future: BrowserRefState = { version: 2, refCounter: 0, generations: [], refs: [] }; + expect(() => executor.importRefState(future)).toThrow(/version 2/); + }); +});