diff --git a/packages/agent/src/translator/browser-observation.ts b/packages/agent/src/translator/browser-observation.ts new file mode 100644 index 0000000..a27be76 --- /dev/null +++ b/packages/agent/src/translator/browser-observation.ts @@ -0,0 +1,111 @@ +/** Minimal CDP accessibility node used by browser observations. */ +export interface AXNode { + nodeId: string; + ignored?: boolean; + role?: { value?: string }; + name?: { value?: string }; + value?: { value?: unknown }; + properties?: Array<{ name: string; value?: { value?: unknown } }>; + backendDOMNodeId?: number; + parentId?: string; + childIds?: string[]; +} + +/** Role/name cohort positions used when minting and healing refs. */ +export interface NthIndex { + index: Map; + cohorts: Map; +} + +/** Frame and generation metadata required to render an observed node. */ +export interface RenderContext { + targetId: string; + frameKey: string; + sessionId: string; + generation: number; + interactiveOnly: boolean; + nthIndex: NthIndex; + cursorIds?: ReadonlySet; +} + +/** One normalized accessibility line before its ref is minted. */ +export interface ObservationLine { + text: string; + refNode?: AXNode; + ctx: RenderContext; +} + +/** Accessibility tree collected for one stitched child frame. */ +export interface FrameStitch { + byId: Map; + roots: string[]; + ctx: RenderContext; +} + +/** Accessibility node paired with its frame context. */ +export interface ObservedNode { + node: AXNode; + ctx: RenderContext; +} + +/** Stable structured browser state collected before presentation filtering. */ +export interface BrowserObservation { + targetId: string; + tree: FrameStitch; + stitches: Map; + nodes: ObservedNode[]; + url: string; + title: string; + generations: Map; +} + +/** Render-ready projection of one structured browser observation. */ +export interface BrowserPresentation { + observation: BrowserObservation; + cacheKey: string; + lines: ObservationLine[]; + shape: string; +} + +/** Signals that browser state changed while an observation was collected. */ +export class ObservationChangedError extends Error { + constructor(message = "Browser observation changed during collection") { + super(message); + this.name = "ObservationChangedError"; + } +} + +/** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ +export function buildNthIndex(nodes: AXNode[]): NthIndex { + const cohorts = new Map(); + const index = new Map(); + for (const node of nodes) { + if (node.ignored || node.backendDOMNodeId === undefined) continue; + const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); + const nth = cohorts.get(key) ?? 0; + cohorts.set(key, nth + 1); + index.set(node.nodeId, nth); + } + return { index, cohorts }; +} + +/** Build the stable key for a role/name ref-healing cohort. */ +export function cohortKey(role: string, name: string): string { + return `${role}\u0000${name}`; +} + +/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ +export function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { + let end = start; + const parts: string[] = []; + while (end < childIds.length) { + const node = tree.get(childIds[end]!); + if (!node || node.ignored || node.role?.value !== "StaticText") break; + const text = node.name?.value ?? ""; + if (text) parts.push(text); + end += 1; + } + if (end - start < 2) return undefined; + const first = tree.get(childIds[start]!)!; + return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; +} diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 9cb21a1..a04231a 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -13,6 +13,18 @@ import { type CuaBrowserAction, } from "@onkernel/cua-ai"; import { CdpConnection, type CdpEventMessage } from "./cdp"; +import { + ObservationChangedError, + buildNthIndex, + cohortKey, + staticTextRun, + type AXNode, + type BrowserObservation, + type BrowserPresentation, + type FrameStitch, + type ObservationLine, + type RenderContext, +} from "./browser-observation"; import type { BatchReadResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; @@ -25,18 +37,6 @@ const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references. const REF_PLACEHOLDER = "\u0000"; const UNCHANGED_SNAPSHOT = "Page unchanged since the last snapshot; previous element refs are still valid."; -interface AXNode { - nodeId: string; - ignored?: boolean; - role?: { value?: string }; - name?: { value?: string }; - value?: { value?: unknown }; - properties?: Array<{ name: string; value?: { value?: unknown } }>; - backendDOMNodeId?: number; - parentId?: string; - childIds?: string[]; -} - interface RefEntry { backendNodeId: number; targetId: string; @@ -52,33 +52,6 @@ interface RefEntry { cohort: number; } -interface NthIndex { - index: Map; - cohorts: Map; -} - -interface RenderContext { - targetId: string; - frameKey: string; - sessionId: string; - generation: number; - interactiveOnly: boolean; - nthIndex: NthIndex; - cursorIds?: ReadonlySet; -} - -interface RenderedLine { - text: string; - refNode?: AXNode; - ctx: RenderContext; -} - -interface FrameStitch { - byId: Map; - roots: string[]; - ctx: RenderContext; -} - export interface BrowserFindCandidate { ref: string; role: string; @@ -306,72 +279,126 @@ export class BrowserExecutor { } private async snapshot(action: CuaActionBrowserSnapshot): Promise { - const targetId = await this.resolveTarget(action.tab_id); - const pageSession = await this.attach(targetId); - const refEntry = action.ref ? this.resolveRef(action.ref, targetId) : undefined; - const frameKey = refEntry?.frameId ?? targetId; - const { nodes, sessionId } = await this.frameAxTree(frameKey, targetId, pageSession); - const byId = new Map(nodes.map((node) => [node.nodeId, node])); - let rootIds = nodes.filter((node) => !node.parentId).map((node) => node.nodeId); - if (action.ref && refEntry) { - const rootNode = - nodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, nodes); - rootIds = [rootNode.nodeId]; + return this.renderObservation(this.presentObservation(await this.observe(action.tab_id), action)); + } + + private async observe(tabId?: string, includeCursor = true): Promise { + for (let attempt = 0; ; attempt += 1) { + try { + return await this.collectObservation(tabId, includeCursor); + } catch (error) { + if (!(error instanceof ObservationChangedError) || attempt === 2) throw error; + } } + } - const interactiveOnly = action.filter === "interactive"; - const ctx: RenderContext = { + private async collectObservation(tabId: string | undefined, includeCursor: boolean): Promise { + const targetId = await this.resolveTarget(tabId); + const pageSession = await this.attach(targetId); + const before = (await this.cdp.pageTargets()).find((target) => target.targetId === targetId); + if (!before) throw new ObservationChangedError("Browser target disappeared during observation"); + + const generations = new Map(); + const rootGeneration = this.trackGeneration(targetId); + generations.set(targetId, rootGeneration); + const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); + const rootCtx: RenderContext = { targetId, - frameKey, + frameKey: targetId, sessionId, - generation: this.generation(frameKey), - interactiveOnly, + generation: rootGeneration, + interactiveOnly: false, nthIndex: buildNthIndex(nodes), - cursorIds: frameKey === targetId ? await this.cursorPointerIds(pageSession) : undefined, + ...(includeCursor ? { cursorIds: await this.cursorPointerIds(pageSession) } : {}), + }; + const tree = this.frameStitch(nodes, rootCtx); + const stitches = await this.stitchFrames(nodes, targetId, pageSession, generations); + const after = (await this.cdp.pageTargets()).find((target) => target.targetId === targetId); + if (!after || before.url !== after.url || before.title !== after.title) { + throw new ObservationChangedError("Browser target metadata changed during observation"); + } + for (const [frameKey, generation] of generations) { + if (this.generations.get(frameKey) !== generation) throw new ObservationChangedError(); + } + return { + targetId, + tree, + stitches, + nodes: [ + ...nodes.map((node) => ({ node, ctx: rootCtx })), + ...[...stitches.values()].flatMap((stitch) => [...stitch.byId.values()].map((node) => ({ node, ctx: stitch.ctx }))), + ], + url: before.url, + title: before.title, + generations, }; - const stitches = frameKey === targetId ? await this.stitchFrames(nodes, targetId, pageSession, interactiveOnly) : new Map(); - const lines: RenderedLine[] = []; + } + + private presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation { + const refEntry = action.ref ? this.resolveRef(action.ref, observation.targetId) : undefined; + let tree = observation.tree; + let rootIds = tree.roots; + if (action.ref && refEntry) { + if (refEntry.frameId === observation.targetId) { + tree = observation.tree; + } else { + const frameTree = [...observation.stitches.values()].find((stitch) => stitch.ctx.frameKey === refEntry.frameId); + if (!frameTree) throw staleRefError(action.ref); + tree = frameTree; + } + const treeNodes = [...tree.byId.values()]; + const rootNode = + treeNodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, treeNodes); + rootIds = [rootNode.nodeId]; + } + const interactiveOnly = action.filter === "interactive"; + const lines: ObservationLine[] = []; const maxDepth = action.depth ?? DEFAULT_SNAPSHOT_DEPTH; - const walk = (tree: Map, treeCtx: RenderContext, nodeId: string, depth: number, parentName: string): void => { - const node = tree.get(nodeId); + const walk = (current: FrameStitch, nodeId: string, depth: number, parentName: string): void => { + const node = current.byId.get(nodeId); if (!node) return; + const ctx = current.ctx.interactiveOnly === interactiveOnly ? current.ctx : { ...current.ctx, interactiveOnly }; let childDepth = depth; if (!node.ignored) { - const rendered = this.renderNode(node, depth, parentName, treeCtx); + const rendered = this.renderNode(node, depth, parentName, ctx); if (rendered) { - lines.push({ ...rendered, ctx: treeCtx }); + lines.push({ ...rendered, ctx }); childDepth = depth + 1; } } if (childDepth > maxDepth) return; - const stitch = treeCtx === ctx && node.backendDOMNodeId !== undefined ? stitches.get(node.backendDOMNodeId) : undefined; + const stitch = current === observation.tree && node.backendDOMNodeId !== undefined ? observation.stitches.get(node.backendDOMNodeId) : undefined; if (stitch) { - for (const frameRootId of stitch.roots) walk(stitch.byId, stitch.ctx, frameRootId, childDepth, ""); + for (const frameRootId of stitch.roots) walk(stitch, frameRootId, childDepth, ""); return; } - const name = node.name?.value ?? ""; - const childName = name || parentName; + const childName = node.name?.value || parentName; const childIds = node.childIds ?? []; for (let i = 0; i < childIds.length; i += 1) { - const run = staticTextRun(tree, childIds, i); + const run = staticTextRun(current.byId, childIds, i); if (run) { - const rendered = this.renderNode(run.node, childDepth, childName, treeCtx); - if (rendered) lines.push({ ...rendered, ctx: treeCtx }); + const rendered = this.renderNode(run.node, childDepth, childName, ctx); + if (rendered) lines.push({ ...rendered, ctx }); i = run.end; - continue; - } - walk(tree, treeCtx, childIds[i]!, childDepth, childName); + } else walk(current, childIds[i]!, childDepth, childName); } }; - for (const rootId of rootIds) walk(byId, ctx, rootId, 0, ""); - + for (const rootId of rootIds) walk(tree, rootId, 0, ""); const shape = lines.map((line) => line.text).join("\n"); - const frameGenerations = [...stitches.values()].map((stitch) => `${stitch.ctx.frameKey}:${stitch.ctx.generation}`); - const key = [action.ref ?? "", action.depth ?? "", action.filter ?? "", `${frameKey}:${ctx.generation}`, ...frameGenerations].join("|"); - const cached = this.lastSnapshots.get(targetId); - this.lastSnapshots.set(targetId, { key, shape }); - if (cached && cached.key === key && cached.shape === shape) return UNCHANGED_SNAPSHOT; + const generationKey = [...observation.generations].map(([frameKey, generation]) => `${frameKey}:${generation}`).join("|"); + return { + observation, + cacheKey: [action.ref ?? "", action.depth ?? "", action.filter ?? "", generationKey].join("|"), + lines, + shape, + }; + } + private renderObservation(presentation: BrowserPresentation): string { + const { observation, cacheKey, lines, shape } = presentation; + const cached = this.lastSnapshots.get(observation.targetId); + this.lastSnapshots.set(observation.targetId, { key: cacheKey, shape }); + if (cached?.key === cacheKey && cached.shape === shape) return UNCHANGED_SNAPSHOT; let text = ""; for (const line of lines) { if (text.length > SNAPSHOT_CHAR_LIMIT) break; @@ -381,7 +408,7 @@ export class BrowserExecutor { if (text.length > SNAPSHOT_CHAR_LIMIT) { text = `${text.slice(0, SNAPSHOT_CHAR_LIMIT)}\n… truncated at ${SNAPSHOT_CHAR_LIMIT} characters. Re-request with a smaller depth, filter: "interactive", or a ref to narrow the subtree.`; } - this.pruneRefs(targetId); + this.pruneRefs(observation.targetId); return text || "(empty accessibility tree)"; } @@ -418,12 +445,20 @@ export class BrowserExecutor { return { nodes, sessionId: pageSession }; } + private frameStitch(nodes: AXNode[], ctx: RenderContext): FrameStitch { + return { + byId: new Map(nodes.map((node) => [node.nodeId, node])), + roots: nodes.filter((node) => !node.parentId).map((node) => node.nodeId), + ctx, + }; + } + /** Resolve each iframe node's child frame and fetch its AX tree for stitching. One nesting level only. */ private async stitchFrames( nodes: AXNode[], targetId: string, pageSession: string, - interactiveOnly: boolean, + generations: Map, ): Promise> { const stitches = new Map(); for (const node of nodes) { @@ -436,19 +471,20 @@ export class BrowserExecutor { ); const frameId = dom.contentDocument?.frameId ?? dom.frameId; if (!frameId || frameId === targetId) continue; + const generation = this.trackGeneration(frameId); + generations.set(frameId, generation); const { nodes: frameNodes, sessionId } = await this.frameAxTree(frameId, targetId, pageSession); - stitches.set(node.backendDOMNodeId, { - byId: new Map(frameNodes.map((frameNode) => [frameNode.nodeId, frameNode])), - roots: frameNodes.filter((frameNode) => !frameNode.parentId).map((frameNode) => frameNode.nodeId), - ctx: { + stitches.set( + node.backendDOMNodeId, + this.frameStitch(frameNodes, { targetId, frameKey: frameId, sessionId, - generation: this.generation(frameId), - interactiveOnly, + generation, + interactiveOnly: false, nthIndex: buildNthIndex(frameNodes), - }, - }); + }), + ); } catch { // Cross-origin or already-detached frames can refuse the fetch; the iframe renders without children. } @@ -499,35 +535,13 @@ export class BrowserExecutor { * matches, best first. Structured counterpart of the `browser_find` action. */ async findCandidates(query: string, tabId?: string, roles?: ReadonlySet): Promise { - const targetId = await this.resolveTarget(tabId); - const session = await this.attach(targetId); - const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); - const pools: Array<{ nodes: AXNode[]; ctx: RenderContext }> = [ - { - nodes, - ctx: { - targetId, - frameKey: targetId, - sessionId: session, - generation: this.generation(targetId), - interactiveOnly: false, - nthIndex: buildNthIndex(nodes), - }, - }, - ]; - // Search stitched frames too so find sees everything snapshot renders. - for (const stitch of (await this.stitchFrames(nodes, targetId, session, false)).values()) { - pools.push({ nodes: [...stitch.byId.values()], ctx: stitch.ctx }); - } + const observation = await this.observe(tabId, false); const queryTokens = tokenize(query); - const scored = pools - .flatMap(({ nodes: poolNodes, ctx }) => - poolNodes - .filter( - (node) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), - ) - .map((node) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })), + const scored = observation.nodes + .filter( + ({ node }) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), ) + .map(({ node, ctx }) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })) .filter((entry) => entry.score > 0 && (!roles || roles.has(entry.node.role?.value ?? ""))) .sort((a, b) => b.score - a.score) .slice(0, FIND_MATCH_LIMIT); @@ -537,7 +551,7 @@ export class BrowserExecutor { name: node.name?.value ?? "", score, })); - this.pruneRefs(targetId); + this.pruneRefs(observation.targetId); return candidates; } @@ -847,6 +861,11 @@ export class BrowserExecutor { return this.generations.get(targetId) ?? 0; } + private trackGeneration(frameKey: string): number { + if (!this.generations.has(frameKey)) this.generations.set(frameKey, 0); + return this.generation(frameKey); + } + private invalidateRefs(targetId: string): void { this.generations.set(targetId, this.generation(targetId) + 1); for (const [ref, entry] of this.refs) { @@ -936,40 +955,6 @@ function staleRefError(ref: string, cause?: unknown): Error { return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); } -/** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ -function buildNthIndex(nodes: AXNode[]): NthIndex { - const cohorts = new Map(); - const index = new Map(); - for (const node of nodes) { - if (node.ignored || node.backendDOMNodeId === undefined) continue; - const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); - const nth = cohorts.get(key) ?? 0; - cohorts.set(key, nth + 1); - index.set(node.nodeId, nth); - } - return { index, cohorts }; -} - -function cohortKey(role: string, name: string): string { - return `${role}\u0000${name}`; -} - -/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ -function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { - let end = start; - const parts: string[] = []; - while (end < childIds.length) { - const node = tree.get(childIds[end]!); - if (!node || node.ignored || node.role?.value !== "StaticText") break; - const text = node.name?.value ?? ""; - if (text) parts.push(text); - end += 1; - } - if (end - start < 2) return undefined; - const first = tree.get(childIds[start]!)!; - return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; -} - function collectStates(node: AXNode): string[] { const states: string[] = []; for (const property of node.properties ?? []) { diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index b345b62..2ec57aa 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -111,6 +111,10 @@ function createFakeCdp(initialNodes: unknown[] = []) { const iframeFrameIds = new Map(); const autoAttachFrames: Array<{ targetId: string; sessionId: string }> = []; const failMethods = new Set(); + let axRead = 0; + let targetRead = 0; + let onAxRead: ((read: number, params: Record, sessionId?: string) => void) | undefined; + let targetProvider: ((read: number) => Array<{ targetId: string; type: string; title: string; url: string }>) | undefined; const emit = (event: FakeCdpEvent) => { for (const listener of listeners) listener(event); }; @@ -131,8 +135,11 @@ function createFakeCdp(initialNodes: unknown[] = []) { sent.push({ method, params, sessionId }); if (failMethods.has(method)) throw new Error(`${method} rejected`); switch (method) { - case "Accessibility.getFullAXTree": - return { nodes: treeFor(sessionId, params.frameId) }; + case "Accessibility.getFullAXTree": { + const tree = treeFor(sessionId, params.frameId); + onAxRead?.(++axRead, params, sessionId); + return { nodes: tree }; + } case "Target.setAutoAttach": for (const frame of autoAttachFrames.splice(0)) { emit({ @@ -170,7 +177,8 @@ function createFakeCdp(initialNodes: unknown[] = []) { return {}; } }, - pageTargets: async () => [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], + pageTargets: async () => + targetProvider?.(++targetRead) ?? [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], attachToTarget: async () => "session-1", createTarget: async () => "TARGET-2", close: () => {}, @@ -196,6 +204,12 @@ function createFakeCdp(initialNodes: unknown[] = []) { const failOn = (method: string) => { failMethods.add(method); }; + const setAxReadHook = (hook: typeof onAxRead) => { + onAxRead = hook; + }; + const setTargetProvider = (provider: typeof targetProvider) => { + targetProvider = provider; + }; return { sent, emit, @@ -206,6 +220,8 @@ function createFakeCdp(initialNodes: unknown[] = []) { setIframeFrame, addAutoAttachFrame, failOn, + setAxReadHook, + setTargetProvider, cdp: fake as unknown as CdpConnection, }; } @@ -587,6 +603,15 @@ describe("BrowserExecutor cursor-pointer hints", () => { expect(sent.some((cmd) => cmd.method === "DOM.describeNode")).toBe(true); expect(sent.some((cmd) => cmd.method === "Runtime.releaseObjectGroup")).toBe(true); }); + + it("does not scan cursor metadata for find", async () => { + const { cdp, failOn, sent } = createFakeCdp(BUTTON_TREE); + failOn("Runtime.evaluate"); + const executor = new BrowserExecutor(cdp); + const results = await executor.execute({ type: "browser_find", query: "save button" } as CuaBrowserAction); + expect((results[0] as { text: string }).text).toContain('button "Save" [e1]'); + expect(sent.some((command) => command.method === "Runtime.evaluate")).toBe(false); + }); }); describe("BrowserExecutor dialog guard", () => { @@ -736,6 +761,119 @@ describe("BrowserExecutor iframe stitching", () => { const text = await snapshotText(executor); expect(text).toContain('button "Pay" [e'); }); + + it("rejects a scoped frame ref when its owning frame is missing from the observation", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Pay", backendDOMNodeId: 40, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + fake.failOn("DOM.describeNode"); + await expect(snapshotText(executor, { ref: "e3" })).rejects.toThrow(/stale/); + }); +}); + +describe("BrowserExecutor observation fencing", () => { + const STALE_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Old", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Stale", backendDOMNodeId: 41, parentId: "1" }), + ]; + const STABLE_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "New", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Stable", backendDOMNodeId: 42, parentId: "1" }), + ]; + + it.each(["browser_snapshot", "browser_find"] as const)("discards a navigated AX collection for %s before minting refs", async (type) => { + const fake = createFakeCdp(STALE_TREE); + fake.setAxReadHook((read, params) => { + if (read !== 1 || params.frameId) return; + fake.setNodes(STABLE_TREE); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + const results = await executor.execute( + (type === "browser_snapshot" ? { type } : { type, query: "stable button" }) as CuaBrowserAction, + ); + const text = (results[0] as { text: string }).text; + expect(text).toContain('button "Stable" [e1]'); + expect(text).not.toContain("Stale"); + expect([...refsOf(executor).keys()]).toEqual(["e1"]); + }); + + it.each([ + ["url", { title: "Page", url: "https://b.test/" }], + ["title", { title: "Changed", url: "https://a.test/" }], + ] as const)("retries when target %s changes across collection", async (_field, changed) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setTargetProvider((read) => [ + { targetId: "TARGET-1", type: "page", ...(read < 3 ? { title: "Page", url: "https://a.test/" } : changed) }, + ]); + const executor = new BrowserExecutor(fake.cdp); + expect(await snapshotText(executor)).toContain('button "Save" [e1]'); + expect(fake.sent.filter((command) => command.method === "Accessibility.getFullAXTree")).toHaveLength(2); + }); + + it("retries when a stitched frame changes during its AX read", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const oldChild = [ax({ nodeId: "f1", role: "button", name: "Old child", backendDOMNodeId: 60 })]; + const newChild = [ax({ nodeId: "f1", role: "button", name: "New child", backendDOMNodeId: 61 })]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", oldChild); + fake.setAxReadHook((_read, params) => { + if (params.frameId !== "FRAME-SP") return; + fake.setAxReadHook(undefined); + fake.setFrameTree("FRAME-SP", newChild); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-SP", parentId: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "New child"'); + expect(text).not.toContain("Old child"); + }); + + it("retries when a generation-zero OOPIF detaches after its AX read", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setSessionTree("session-oop", [ax({ nodeId: "f1", role: "button", name: "Detached", backendDOMNodeId: 60 })]); + fake.setFrameTree("FRAME-OOP", [ax({ nodeId: "f1", role: "button", name: "Current", backendDOMNodeId: 61 })]); + fake.setAxReadHook((_read, _params, sessionId) => { + if (sessionId !== "session-oop") return; + fake.setAxReadHook(undefined); + fake.emit({ method: "Target.detachedFromTarget", params: { sessionId: "session-oop" } }); + }); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "Current"'); + expect(text).not.toContain("Detached"); + }); + + it.each(["browser_snapshot", "browser_find"] as const)("fails %s after three changed collections without minting refs", async (type) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setAxReadHook((_read, params) => { + if (!params.frameId) fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + await expect( + executor.execute((type === "browser_snapshot" ? { type } : { type, query: "save" }) as CuaBrowserAction), + ).rejects.toThrow(/observation changed/i); + expect(fake.sent.filter((command) => command.method === "Accessibility.getFullAXTree")).toHaveLength(3); + expect(refsOf(executor).size).toBe(0); + }); }); describe("navigation tool grounding frame", () => {