From 7b76f0629fd0268c2ad5577e947c5f684604d4eb Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 14 Jul 2026 01:37:24 +0000 Subject: [PATCH] Add semantic browser waits --- packages/agent/src/index.ts | 7 +- packages/agent/src/tools.ts | 19 +- .../src/translator/browser-observation.ts | 11 +- packages/agent/src/translator/browser-wait.ts | 185 ++++++++++ packages/agent/src/translator/browser.ts | 329 +++++++++++++----- packages/agent/src/translator/types.ts | 30 +- packages/agent/test/browser-wait.test.ts | 176 ++++++++++ .../agent/test/translator-browser.test.ts | 124 ++++++- packages/ai/src/actions/browser.ts | 46 +++ packages/ai/src/modes.ts | 3 + packages/ai/test/modes.test.ts | 33 ++ 11 files changed, 878 insertions(+), 85 deletions(-) create mode 100644 packages/agent/src/translator/browser-wait.ts create mode 100644 packages/agent/test/browser-wait.test.ts diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 220fff4b..f8d559ed 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -6,7 +6,12 @@ export { InternalComputerTranslator } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; export { BrowserExecutor } from "./translator/browser"; export type { BrowserFindCandidate, BrowserRefState } from "./translator/browser"; -export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; +export type { + BatchExecutionResult, + BatchReadResult, + BrowserExpectationEvidence, + BrowserWaitForResult, +} from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { BatchDetails, diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index 9a227434..e31e8941 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -16,6 +16,7 @@ import { } from "@onkernel/cua-ai"; import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core"; +import type { BrowserWaitForResult } from "./translator/types"; export interface ComputerToolOptions { browser: KernelBrowser; @@ -37,6 +38,7 @@ export interface BatchDetails { | { type: "screenshot"; bytes: number } | { type: "cursor_position"; x: number; y: number } | { type: "browser_text"; label: string; bytes: number } + | { type: "browser_wait_for"; result: BrowserWaitForResult } >; } @@ -170,6 +172,9 @@ async function executeBatchTool( } else if (read.type === "browser_text") { readResults.push({ type: "browser_text", label: read.label, bytes: read.text.length }); content.push({ type: "text", text: read.text }); + } else if (read.type === "browser_wait_for") { + readResults.push(read); + content.push({ type: "text", text: formatBrowserWaitResult(read.result) }); } else { readResults.push({ type: "screenshot", bytes: read.data.length }); content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType }); @@ -185,7 +190,14 @@ async function executeBatchTool( } catch (err) { throw new Error(`Actions failed: ${errorMessage(err)}`, { cause: err }); } - return { content, details: { statusText: "Actions executed successfully.", readResults } }; + const waits = readResults.flatMap((read) => read.type === "browser_wait_for" ? [read.result] : []); + const failedWait = ["interrupted", "timed_out", "unverifiable"].find((status) => waits.some((wait) => wait.status === status)); + const statusText = waits.length === 0 + ? "Actions executed successfully." + : failedWait + ? `Browser condition ${failedWait}.` + : "Browser condition satisfied."; + return { content, details: { statusText, readResults } }; } async function executeNavigationTool( @@ -257,6 +269,11 @@ async function executePlaywrightTool(translator: InternalComputerTranslator, par } } +function formatBrowserWaitResult(result: BrowserWaitForResult): string { + const reason = result.reason ? ` (${result.reason})` : ""; + return [`wait_for: ${result.status}/${result.evidence}${reason} after ${result.elapsed_ms}ms`, ...result.details].join("\n"); +} + function formatPlaywrightResult(result: unknown): string { return typeof result === "string" ? result : JSON.stringify(result); } diff --git a/packages/agent/src/translator/browser-observation.ts b/packages/agent/src/translator/browser-observation.ts index a27be764..ccbd2cbd 100644 --- a/packages/agent/src/translator/browser-observation.ts +++ b/packages/agent/src/translator/browser-observation.ts @@ -21,6 +21,8 @@ export interface NthIndex { export interface RenderContext { targetId: string; frameKey: string; + /** Target whose CDP session owns this frame (page target or OOPIF target). */ + sessionTargetId: string; sessionId: string; generation: number; interactiveOnly: boolean; @@ -51,12 +53,14 @@ export interface ObservedNode { /** Stable structured browser state collected before presentation filtering. */ export interface BrowserObservation { targetId: string; + navigationEpoch: number; tree: FrameStitch; - stitches: Map; + stitches: Map; nodes: ObservedNode[]; url: string; title: string; generations: Map; + complete: boolean; } /** Render-ready projection of one structured browser observation. */ @@ -67,6 +71,11 @@ export interface BrowserPresentation { shape: string; } +/** Build the lookup key for an iframe node's stitched child tree. */ +export function frameStitchKey(parentFrameKey: string, backendNodeId: number): string { + return `${parentFrameKey}\u0000${backendNodeId}`; +} + /** Signals that browser state changed while an observation was collected. */ export class ObservationChangedError extends Error { constructor(message = "Browser observation changed during collection") { diff --git a/packages/agent/src/translator/browser-wait.ts b/packages/agent/src/translator/browser-wait.ts new file mode 100644 index 00000000..9942d9a3 --- /dev/null +++ b/packages/agent/src/translator/browser-wait.ts @@ -0,0 +1,185 @@ +import type { CuaBrowserExpectation } from "@onkernel/cua-ai"; +import { staticTextRun, type AXNode, type BrowserObservation } from "./browser-observation"; +import type { BrowserExpectationEvidence, BrowserWaitForResult, BrowserWaitReason } from "./types"; + +/** Internal expectation truth and optional lifecycle reason. */ +export interface BrowserExpectationEvaluation extends BrowserExpectationEvidence { + reason?: BrowserWaitReason; +} + +type RefExpectation = Extract; +/** Resolve a ref expectation against one structured observation. */ +export type BrowserRefResolver = (expectation: RefExpectation, observation: BrowserObservation) => BrowserExpectationEvaluation; + +/** Browser capabilities required by the semantic wait engine. */ +export interface BrowserWaitRuntime { + selectTarget(tabId?: string): Promise; + observeTarget(targetId: string): Promise; + dialogCount(): number; + targetExists(targetId: string): Promise; + liveGeneration(frameId: string): number; + resolveRef: BrowserRefResolver; + now?(): number; + delay?(ms: number): Promise; +} + +/** Timeout, polling, target, and condition options for a semantic wait. */ +export interface BrowserWaitOptions { + expect: CuaBrowserExpectation; + timeoutMs?: number; + pollMs?: number; + tabId?: string; +} + +function expectationNodes(observation: BrowserObservation): AXNode[] { + const nodes = observation.nodes.map(({ node }) => node); + for (const tree of [observation.tree, ...observation.stitches.values()]) { + for (const node of tree.byId.values()) { + const children = node.childIds ?? []; + for (let index = 0; index < children.length; index += 1) { + const run = staticTextRun(tree.byId, children, index); + if (run) { nodes.push(run.node); index = run.end; } + } + } + } + return nodes; +} + +/** Evaluate a semantic condition without minting refs. Unknown means the observation cannot prove the claim. */ +export function evaluateBrowserExpectation( + expectation: CuaBrowserExpectation, + observation: BrowserObservation, + baseline: BrowserObservation, + resolveRef: BrowserRefResolver, +): BrowserExpectationEvaluation { + if ("all" in expectation || "any" in expectation) { + const all = "all" in expectation; + const children = (all ? expectation.all : expectation.any).map((child) => evaluateBrowserExpectation(child, observation, baseline, resolveRef)); + const truth = all + ? children.some((child) => child.truth === false) ? false : children.some((child) => child.truth === undefined) ? undefined : true + : children.some((child) => child.truth === true) ? true : children.some((child) => child.truth === undefined) ? undefined : false; + return { + truth, + details: children.flatMap((child) => child.details), + ...(truth === undefined ? { reason: children.find((child) => child.truth === undefined)?.reason } : {}), + }; + } + if (expectation.type === "text" || expectation.type === "role_name") { + const found = expectationNodes(observation).some((node) => !node.ignored && (expectation.type === "text" + ? (node.name?.value ?? "").replace(/\s+/g, " ").toLowerCase().includes(expectation.text.replace(/\s+/g, " ").toLowerCase()) + : (expectation.role === undefined || node.role?.value === expectation.role) && (expectation.name === undefined || node.name?.value === expectation.name))); + const expected = expectation.exists ?? true; + const truth = !found && !observation.complete ? undefined : found === expected; + return { truth, details: [`${expectation.type} ${found ? "present" : "absent"}${observation.complete ? "" : "; observation incomplete"}`], ...(!observation.complete && !found ? { reason: "incomplete_observation" as const } : {}) }; + } + if (expectation.type === "url" || expectation.type === "title") { + const value = observation[expectation.type]; + const initial = baseline[expectation.type]; + const truth = (expectation.equals === undefined || value === expectation.equals) && + (expectation.contains === undefined || value.includes(expectation.contains)) && + (expectation.changed === undefined || (value !== initial) === expectation.changed); + return { truth, details: [`${expectation.type}=${JSON.stringify(value)}`] }; + } + if (expectation.type === "ref") return resolveRef(expectation, observation); + return { truth: undefined, details: ["unsupported expectation"] }; +} + +/** Poll a semantic browser condition; in-flight browser reads settle before timeout is reported. */ +export async function waitForBrowserExpectation(runtime: BrowserWaitRuntime, options: BrowserWaitOptions): Promise { + const now = runtime.now ?? Date.now; + const sleep = runtime.delay ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const started = now(); + const timeout = options.timeoutMs ?? 2_000; + const poll = options.pollMs ?? 50; + const remaining = () => Math.max(0, timeout - (now() - started)); + const expired = () => now() - started >= timeout; + let targetId: string; + let baseline: BrowserObservation; + try { + targetId = await beforeDeadline(() => runtime.selectTarget(options.tabId), remaining(), now); + if (expired()) return timedOut(started, now()); + baseline = await runtime.observeTarget(targetId); + if (expired()) return timedOut(started, now()); + } catch (error) { + if (error instanceof WaitDeadlineError) return timedOut(started, now()); + return failedObservation(started, now(), error); + } + const initial = evaluateBrowserExpectation(options.expect, baseline, baseline, runtime.resolveRef); + if (!baseline.complete) return terminal("unverifiable", "unverifiable", initial, initial, started, now(), "incomplete_observation"); + if (expired()) return timedOut(started, now()); + if (initial.reason === "stale_ref") return terminal("interrupted", "unverifiable", initial, initial, started, now(), initial.reason); + if (initial.truth === true) return terminal("satisfied", "preexisting", initial, initial, started, now()); + const dialogs = runtime.dialogCount(); + let final = initial; + while (!expired()) { + await sleep(Math.min(poll, remaining())); + if (expired()) break; + let exists: boolean; + try { exists = await beforeDeadline(() => runtime.targetExists(targetId), remaining(), now); } + catch (error) { if (error instanceof WaitDeadlineError) break; return failedObservation(started, now(), error, initial, final); } + if (expired()) break; + if (!exists) return terminal("interrupted", "unverifiable", initial, final, started, now(), "target_detached"); + let observation: BrowserObservation; + try { observation = await runtime.observeTarget(targetId); } + catch (error) { return failedObservation(started, now(), error, initial, final); } + if (observation.targetId !== targetId) return terminal("interrupted", "unverifiable", initial, final, started, now(), "target_changed"); + if (runtime.dialogCount() > dialogs) return terminal("interrupted", "unverifiable", initial, final, started, now(), "dialog"); + const navigated = observation.navigationEpoch !== baseline.navigationEpoch; + if (navigated) { + if (isLocationExpectation(options.expect)) { + final = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef); + if (final.truth === true && !expired()) return terminal("satisfied", "newly_verified", initial, final, started, now()); + } + return terminal("interrupted", "unverifiable", initial, final, started, now(), "navigation"); + } + if (!observation.complete) { + const partial = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef); + final = { ...partial, truth: undefined, reason: "incomplete_observation" }; + continue; + } + const removedFrame = [...baseline.generations.keys()].some((key) => !observation.generations.has(key)); + if (removedFrame) return terminal("unverifiable", "unverifiable", initial, final, started, now(), "incomplete_observation"); + if (expired()) break; + final = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef); + if (expired()) break; + if (final.reason === "stale_ref") return terminal("interrupted", "unverifiable", initial, final, started, now(), final.reason); + if (final.truth === true) return terminal("satisfied", "newly_verified", initial, final, started, now()); + } + if (final.truth === undefined) return terminal("unverifiable", "unverifiable", initial, final, started, now(), final.reason ?? "incomplete_observation"); + return terminal("timed_out", "failed", initial, final, started, now()); +} + +class WaitDeadlineError extends Error {} + +async function beforeDeadline(operation: () => Promise, remaining: number, now: () => number): Promise { + if (remaining <= 0) throw new WaitDeadlineError(); + const started = now(); + try { + const result = await operation(); + if (Number.isFinite(remaining) && now() - started >= remaining) throw new WaitDeadlineError(); + return result; + } catch (error) { + if (error instanceof WaitDeadlineError || Number.isFinite(remaining) && now() - started >= remaining) throw new WaitDeadlineError(); + throw error; + } +} + +function isLocationExpectation(expectation: CuaBrowserExpectation): boolean { + if ("all" in expectation) return expectation.all.every(isLocationExpectation); + if ("any" in expectation) return expectation.any.every(isLocationExpectation); + return expectation.type === "url" || expectation.type === "title"; +} + +function timedOut(started: number, ended: number): BrowserWaitForResult { + const evidence = { truth: undefined, details: [] }; + return terminal("timed_out", "failed", evidence, evidence, started, ended); +} + +function terminal(status: BrowserWaitForResult["status"], evidence: BrowserWaitForResult["evidence"], initial: BrowserExpectationEvidence, final: BrowserExpectationEvidence, started: number, ended: number, reason?: BrowserWaitReason): BrowserWaitForResult { + return { status, evidence, initial, final, elapsed_ms: Math.max(0, ended - started), ...(reason ? { reason } : {}), details: [...initial.details.map((detail) => `initial: ${detail}`), ...final.details.map((detail) => `final: ${detail}`)] }; +} + +function failedObservation(started: number, ended: number, error: unknown, initial: BrowserExpectationEvidence = { truth: undefined, details: [] }, final = initial): BrowserWaitForResult { + const message = error instanceof Error ? error.message : String(error); + return { ...terminal("unverifiable", "unverifiable", initial, final, started, ended, "observation_failed"), details: [...initial.details, message] }; +} diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index a04231a5..457db3d2 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -10,13 +10,16 @@ import { type CuaActionBrowserScroll, type CuaActionBrowserScrollTo, type CuaActionBrowserSnapshot, + type CuaActionBrowserWaitFor, type CuaBrowserAction, + type CuaBrowserExpectation, } from "@onkernel/cua-ai"; import { CdpConnection, type CdpEventMessage } from "./cdp"; import { ObservationChangedError, buildNthIndex, cohortKey, + frameStitchKey, staticTextRun, type AXNode, type BrowserObservation, @@ -25,12 +28,14 @@ import { type ObservationLine, type RenderContext, } from "./browser-observation"; -import type { BatchReadResult } from "./types"; +import { waitForBrowserExpectation, type BrowserExpectationEvaluation } from "./browser-wait"; +import type { BatchReadResult, BrowserWaitForResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; const DEFAULT_SNAPSHOT_DEPTH = 15; const FIND_MATCH_LIMIT = 20; const REF_LIMIT_PER_TARGET = 1000; +const FRAME_STATE_LIMIT = 1000; const SCROLL_NOTCH_PX = 120; const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; @@ -42,6 +47,8 @@ interface RefEntry { targetId: string; /** Generation key: the owning page target id for main-frame refs, the frame id for iframe refs. */ frameId: string; + /** Page or OOPIF target whose session owns the frame. */ + sessionTargetId?: string; /** Session to route DOM/Input calls through: the frame's own session for OOPIFs, the page session otherwise. */ sessionId: string; generation: number; @@ -72,6 +79,10 @@ export interface BrowserRefState { activeTargetId?: string; generations: Array<[string, number]>; refs: Array<[string, Omit]>; + frameParents?: Array<[string, string]>; + frameOwners?: Array<[string, string]>; + framePages?: Array<[string, string]>; + navigationEpochs?: Array<[string, number]>; } /** @@ -108,6 +119,10 @@ export class BrowserExecutor { private readonly targetsBySession = new Map(); private readonly frameSessions = new Map(); private readonly frameTargets = new Set(); + private readonly frameParents = new Map(); + private readonly frameOwners = new Map(); + private readonly framePages = new Map(); + private readonly navigationEpochs = new Map(); private readonly lastSnapshots = new Map(); private readonly selfNavigations = new Set(); private readonly dialogNotes: string[] = []; @@ -122,37 +137,54 @@ export class BrowserExecutor { private handleCdpEvent(event: CdpEventMessage): void { switch (event.method) { + case "Page.frameAttached": { + const { frameId, parentFrameId } = event.params as { frameId?: string; parentFrameId?: string }; + if (!frameId || !parentFrameId) return; + this.frameParents.set(frameId, parentFrameId); + const sessionTarget = event.sessionId ? this.targetsBySession.get(event.sessionId) : undefined; + const page = this.pageForFrame(parentFrameId, sessionTarget); + if (page) this.framePages.set(frameId, page); + return; + } case "Page.frameNavigated": { const frame = event.params.frame as { id?: string; parentId?: string } | undefined; - if (!event.sessionId || !frame) return; - const targetId = this.targetsBySession.get(event.sessionId); - if (!targetId) return; - if (this.frameTargets.has(targetId)) { - // Refs from a frame target's tree (its root and any same-process - // subframes inlined in it) are all minted against the target's key, - // so any navigation observed in its session stales them. - this.invalidateFrame(targetId); - return; - } + if (!event.sessionId || !frame?.id) return; + const sessionTarget = this.targetsBySession.get(event.sessionId); + if (!sessionTarget) return; + const knownFrame = this.generations.has(frame.id) || this.frameParents.has(frame.id); if (frame.parentId) { - if (frame.id) this.invalidateFrame(frame.id); - return; + this.frameParents.set(frame.id, frame.parentId); + const page = this.pageForFrame(frame.parentId, sessionTarget); + if (page) this.framePages.set(frame.id, page); + } + if (!frame.parentId && !this.frameTargets.has(sessionTarget) && this.selfNavigations.delete(sessionTarget)) return; + this.invalidateFrame(frame.parentId && (knownFrame || !this.frameTargets.has(sessionTarget)) ? frame.id : sessionTarget); + return; + } + case "Page.frameDetached": { + const { frameId } = event.params as { frameId?: string }; + if (frameId) { + const page = this.pageForFrame(frameId); + if (page) this.navigationEpochs.set(page, this.navigationEpoch(page) + 1); + this.removeFrames(this.descendants(frameId)); } - if (!this.selfNavigations.delete(targetId)) this.invalidateRefs(targetId); return; } case "Page.navigatedWithinDocument": { - // A navigate() that turns out same-document never fires frameNavigated; - // consume the pending flag here so it can't swallow the next real navigation. if (!event.sessionId) return; const targetId = this.targetsBySession.get(event.sessionId); const { frameId } = event.params as { frameId?: string }; - if (targetId && frameId === targetId) this.selfNavigations.delete(targetId); + if (!targetId || !frameId) return; + if (frameId === targetId) this.selfNavigations.delete(targetId); + this.invalidateFrame(frameId); return; } case "Target.attachedToTarget": { const { sessionId, targetInfo } = event.params as { sessionId?: string; targetInfo?: { targetId?: string; type?: string } }; if (!sessionId || !targetInfo?.targetId || targetInfo.type !== "iframe") return; + const parentTarget = event.sessionId ? this.targetsBySession.get(event.sessionId) : undefined; + const page = parentTarget ? this.pageForFrame(parentTarget, parentTarget) : undefined; + if (page) this.framePages.set(targetInfo.targetId, page); this.frameSessions.set(targetInfo.targetId, sessionId); this.frameTargets.add(targetInfo.targetId); this.targetsBySession.set(sessionId, targetInfo.targetId); @@ -196,6 +228,10 @@ export class BrowserExecutor { ...(this.activeTargetId ? { activeTargetId: this.activeTargetId } : {}), generations: [...this.generations], refs: [...this.refs].map(([ref, { sessionId: _sessionId, ...entry }]) => [ref, entry]), + frameParents: [...this.frameParents], + frameOwners: [...this.frameOwners], + framePages: [...this.framePages], + navigationEpochs: [...this.navigationEpochs], }; } @@ -205,6 +241,10 @@ export class BrowserExecutor { this.activeTargetId = state.activeTargetId ?? this.activeTargetId; for (const [frameId, generation] of state.generations) this.generations.set(frameId, generation); for (const [ref, entry] of state.refs) this.refs.set(ref, { ...entry, sessionId: "" }); + for (const [frame, parent] of state.frameParents ?? []) this.frameParents.set(frame, parent); + for (const [frame, owner] of state.frameOwners ?? []) this.frameOwners.set(frame, owner); + for (const [frame, page] of state.framePages ?? []) this.framePages.set(frame, page); + for (const [target, epoch] of state.navigationEpochs ?? []) this.navigationEpochs.set(target, epoch); } async execute(action: CuaBrowserAction): Promise { @@ -218,6 +258,8 @@ export class BrowserExecutor { switch (action.type) { case "browser_snapshot": return [{ type: "browser_text", label: "snapshot", text: await this.snapshot(action) }]; + case "browser_wait_for": + return [{ type: "browser_wait_for", result: await this.waitFor(action) }]; case "browser_text": return [{ type: "browser_text", label: "text", text: await this.pageText(tabOf(action)) }]; case "browser_find": @@ -279,7 +321,52 @@ export class BrowserExecutor { } private async snapshot(action: CuaActionBrowserSnapshot): Promise { - return this.renderObservation(this.presentObservation(await this.observe(action.tab_id), action)); + const observation = await this.observe(action.tab_id); + try { + return this.renderObservation(this.presentObservation(observation, action)); + } catch (error) { + if (!action.ref || !/stale/.test(error instanceof Error ? error.message : "")) throw error; + return this.renderObservation(this.presentObservation(await this.observeReferencedFrame(action.ref, observation.targetId), action)); + } + } + + private waitFor(action: CuaActionBrowserWaitFor): Promise { + return waitForBrowserExpectation({ + selectTarget: (tabId) => this.resolveTarget(tabId), + observeTarget: (targetId) => this.observe(targetId, false), + dialogCount: () => this.dialogNotes.length, + targetExists: async (targetId) => (await this.cdp.pageTargets()).some((target) => target.targetId === targetId), + liveGeneration: (frameId) => this.generation(frameId), + resolveRef: (expectation, observation) => this.evaluateRefExpectation(expectation, observation), + }, { expect: action.expect, timeoutMs: action.timeout_ms, pollMs: action.poll_ms, tabId: action.tab_id }); + } + + private evaluateRefExpectation(expectation: Extract, observation: BrowserObservation): BrowserExpectationEvaluation { + const entry = this.refs.get(expectation.ref); + if (!entry || entry.targetId !== observation.targetId || entry.generation !== this.generation(entry.frameId)) { + return { truth: undefined, details: [`ref ${expectation.ref} is stale`], reason: "stale_ref" }; + } + const nodes = observation.nodes.filter(({ ctx }) => ctx.frameKey === entry.frameId).map(({ node }) => node); + let node = nodes.find((candidate) => candidate.backendDOMNodeId === entry.backendNodeId); + if (!node) { + try { node = this.healEntry(expectation.ref, entry, nodes); } + catch { return { truth: undefined, details: [`ref ${expectation.ref} is not observable`], reason: observation.complete ? "stale_ref" : "incomplete_observation" }; } + } + const checks: boolean[] = []; + let missing = false; + if (expectation.value !== undefined) { + if (node.value?.value === undefined) missing = true; + else checks.push(String(node.value.value) === expectation.value); + } + for (const state of ["checked", "selected", "expanded"] as const) { + if (expectation[state] === undefined) continue; + const property = node.properties?.find((candidate) => candidate.name === state)?.value?.value; + if (property === undefined) missing = true; + else checks.push(normalizeState(property) === expectation[state]); + } + if (missing) return { truth: undefined, details: [`ref ${expectation.ref} lacks requested value/state metadata`], reason: "incomplete_observation" }; + const truth = checks.every(Boolean); + return { truth, details: [`ref ${expectation.ref} value/state ${truth ? "matched" : "did not match"}`] }; } private async observe(tabId?: string, includeCursor = true): Promise { @@ -300,11 +387,13 @@ export class BrowserExecutor { const generations = new Map(); const rootGeneration = this.trackGeneration(targetId); + const navigationEpoch = this.navigationEpoch(targetId); generations.set(targetId, rootGeneration); const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); const rootCtx: RenderContext = { targetId, frameKey: targetId, + sessionTargetId: targetId, sessionId, generation: rootGeneration, interactiveOnly: false, @@ -312,16 +401,19 @@ export class BrowserExecutor { ...(includeCursor ? { cursorIds: await this.cursorPointerIds(pageSession) } : {}), }; const tree = this.frameStitch(nodes, rootCtx); - const stitches = await this.stitchFrames(nodes, targetId, pageSession, generations); + const { stitches, complete } = await this.stitchFrames(nodes, targetId, pageSession, generations); const after = (await this.cdp.pageTargets()).find((target) => target.targetId === targetId); - if (!after || before.url !== after.url || before.title !== after.title) { - throw new ObservationChangedError("Browser target metadata changed during observation"); + if (!after || before.url !== after.url || before.title !== after.title || navigationEpoch !== this.navigationEpoch(targetId)) { + throw new ObservationChangedError("Browser observation changed: target metadata changed during collection"); } for (const [frameKey, generation] of generations) { if (this.generations.get(frameKey) !== generation) throw new ObservationChangedError(); } + if (complete) this.pruneFrameState(targetId, new Set(generations.keys())); + this.boundFrameState(new Set(generations.keys())); return { targetId, + navigationEpoch, tree, stitches, nodes: [ @@ -331,9 +423,35 @@ export class BrowserExecutor { url: before.url, title: before.title, generations, + complete, }; } + private async observeReferencedFrame(ref: string, targetId: string): Promise { + for (let attempt = 0; ; attempt += 1) { + try { return await this.collectReferencedFrame(ref, targetId); } + catch (error) { if (!(error instanceof ObservationChangedError) || attempt === 2) throw error; } + } + } + + private async collectReferencedFrame(ref: string, targetId: string): Promise { + const entry = this.resolveRef(ref, targetId); + const pageSession = await this.attach(targetId); + const before = (await this.cdp.pageTargets()).find((candidate) => candidate.targetId === targetId); + if (!before) throw new ObservationChangedError(); + const frameGeneration = this.trackGeneration(entry.frameId); + const navigationEpoch = this.navigationEpoch(targetId); + const { nodes, sessionId } = await this.frameAxTree(entry.frameId, targetId, pageSession); + const owner = entry.sessionTargetId ?? this.frameOwners.get(entry.frameId) ?? targetId; + const generations = new Map([[targetId, this.trackGeneration(targetId)], [entry.frameId, frameGeneration]]); + const ctx: RenderContext = { targetId, frameKey: entry.frameId, sessionTargetId: owner, sessionId, generation: frameGeneration, interactiveOnly: false, nthIndex: buildNthIndex(nodes) }; + const { stitches, complete } = await this.stitchFrames(nodes, targetId, sessionId, generations, entry.frameId, owner); + const after = (await this.cdp.pageTargets()).find((candidate) => candidate.targetId === targetId); + if (!after || before.url !== after.url || before.title !== after.title || navigationEpoch !== this.navigationEpoch(targetId) || [...generations].some(([key, generation]) => this.generation(key) !== generation)) throw new ObservationChangedError(); + this.boundFrameState(new Set(generations.keys())); + return { targetId, navigationEpoch, tree: this.frameStitch(nodes, ctx), stitches, nodes: [...nodes.map((node) => ({ node, ctx })), ...[...stitches.values()].flatMap((stitch) => [...stitch.byId.values()].map((node) => ({ node, ctx: stitch.ctx })))], url: before.url, title: before.title, generations, complete }; + } + private presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation { const refEntry = action.ref ? this.resolveRef(action.ref, observation.targetId) : undefined; let tree = observation.tree; @@ -367,7 +485,7 @@ export class BrowserExecutor { } } if (childDepth > maxDepth) return; - const stitch = current === observation.tree && node.backendDOMNodeId !== undefined ? observation.stitches.get(node.backendDOMNodeId) : undefined; + const stitch = node.backendDOMNodeId !== undefined ? observation.stitches.get(frameStitchKey(current.ctx.frameKey, node.backendDOMNodeId)) : undefined; if (stitch) { for (const frameRootId of stitch.roots) walk(stitch, frameRootId, childDepth, ""); return; @@ -453,43 +571,44 @@ export class BrowserExecutor { }; } - /** Resolve each iframe node's child frame and fetch its AX tree for stitching. One nesting level only. */ + /** Recursively stitch same-process frames and OOPIFs. A failed child fetch makes absence unverifiable. */ private async stitchFrames( - nodes: AXNode[], - targetId: string, - pageSession: string, - generations: Map, - ): Promise> { - const stitches = new Map(); - for (const node of nodes) { - if (node.ignored || !FRAME_ROLES.has(node.role?.value ?? "") || node.backendDOMNodeId === undefined) continue; - try { - const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>( - "DOM.describeNode", - { backendNodeId: node.backendDOMNodeId, depth: 1 }, - pageSession, - ); - const frameId = dom.contentDocument?.frameId ?? dom.frameId; - if (!frameId || frameId === targetId) continue; - const generation = this.trackGeneration(frameId); - generations.set(frameId, generation); - const { nodes: frameNodes, sessionId } = await this.frameAxTree(frameId, targetId, pageSession); - stitches.set( - node.backendDOMNodeId, - this.frameStitch(frameNodes, { - targetId, - frameKey: frameId, - sessionId, - generation, - interactiveOnly: false, - nthIndex: buildNthIndex(frameNodes), - }), - ); - } catch { - // Cross-origin or already-detached frames can refuse the fetch; the iframe renders without children. + nodes: AXNode[], targetId: string, pageSession: string, generations: Map, + rootFrame = targetId, rootOwner = targetId, + ): Promise<{ stitches: Map; complete: boolean }> { + const stitches = new Map(); + const visited = new Set([rootFrame]); + let complete = true; + const visit = async (parentNodes: AXNode[], parentFrame: string, session: string, owner: string): Promise => { + for (const node of parentNodes) { + if (node.ignored || !FRAME_ROLES.has(node.role?.value ?? "") || node.backendDOMNodeId === undefined) continue; + try { + const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>("DOM.describeNode", { backendNodeId: node.backendDOMNodeId, depth: 1 }, session); + const frameId = dom.contentDocument?.frameId ?? dom.frameId; + if (frameId === targetId) continue; + if (!frameId || visited.has(frameId)) { complete = false; continue; } + visited.add(frameId); + this.frameParents.set(frameId, parentFrame); + const frameOwner = this.frameSessions.has(frameId) ? frameId : owner; + this.frameOwners.set(frameId, frameOwner); + this.framePages.set(frameId, targetId); + const generation = this.trackGeneration(frameId); + const { nodes: children, sessionId } = await this.frameAxTree(frameId, targetId, session); + if (generation !== this.generation(frameId)) throw new ObservationChangedError(); + generations.set(frameId, generation); + stitches.set(frameStitchKey(parentFrame, node.backendDOMNodeId), this.frameStitch(children, { + targetId, frameKey: frameId, sessionTargetId: frameOwner, sessionId, generation, + interactiveOnly: false, nthIndex: buildNthIndex(children), + })); + await visit(children, frameId, sessionId, frameOwner); + } catch (error) { + if (error instanceof ObservationChangedError) throw error; + complete = false; + } } - } - return stitches; + }; + await visit(nodes, rootFrame, pageSession, rootOwner); + return { stitches, complete }; } /** Resolve backend node ids for elements whose own computed cursor is "pointer", without touching the DOM. */ @@ -824,6 +943,7 @@ export class BrowserExecutor { backendNodeId: node.backendDOMNodeId!, targetId: ctx.targetId, frameId: ctx.frameKey, + sessionTargetId: ctx.sessionTargetId, sessionId: ctx.sessionId, generation: ctx.generation, role, @@ -842,7 +962,10 @@ export class BrowserExecutor { */ private async refSession(entry: RefEntry): Promise { if (!entry.sessionId) { - entry.sessionId = this.frameSessions.get(entry.frameId) ?? (await this.attach(entry.targetId)); + const pageSession = await this.attach(entry.targetId); + const owner = entry.sessionTargetId ?? this.frameOwners.get(entry.frameId) ?? entry.targetId; + entry.sessionId = owner === entry.targetId ? pageSession : (this.frameSessions.get(owner) ?? ""); + if (!entry.sessionId) throw staleRefError("owning frame session"); } return entry.sessionId; } @@ -866,38 +989,79 @@ export class BrowserExecutor { return this.generation(frameKey); } - private invalidateRefs(targetId: string): void { - this.generations.set(targetId, this.generation(targetId) + 1); - for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId) this.refs.delete(ref); + private navigationEpoch(targetId: string): number { return this.navigationEpochs.get(targetId) ?? 0; } + + private pageForFrame(frameId: string, sessionTarget?: string): string | undefined { + let current: string | undefined = frameId; + const seen = new Set(); + while (current && !seen.has(current)) { + seen.add(current); + const page = this.framePages.get(current); + if (page) return page; + if (!this.frameTargets.has(current) && [...this.targetsBySession.values()].includes(current)) return current; + current = this.frameParents.get(current); } + if (!sessionTarget) return undefined; + return this.framePages.get(sessionTarget) ?? (!this.frameTargets.has(sessionTarget) ? sessionTarget : undefined); } + /** Cycle-safe descendant closure, deepest descendants first. */ + private descendants(frameKey: string): string[] { + const found = new Set([frameKey]); + for (let changed = true; changed;) { + changed = false; + for (const [child, parent] of this.frameParents) if (found.has(parent) && !found.has(child)) { found.add(child); changed = true; } + } + const depth = (frame: string) => { let value = 0, current = frame; const seen = new Set(); while (!seen.has(current) && this.frameParents.has(current)) { seen.add(current); current = this.frameParents.get(current)!; value += 1; } return value; }; + return [...found].sort((a, b) => depth(b) - depth(a)); + } + + private removeFrames(frames: readonly string[]): void { + const removed = new Set(frames); + for (const [ref, entry] of this.refs) if (removed.has(entry.frameId) || (entry.sessionTargetId && removed.has(entry.sessionTargetId))) this.refs.delete(ref); + for (const frame of frames) { this.generations.delete(frame); this.frameParents.delete(frame); this.frameOwners.delete(frame); this.framePages.delete(frame); this.frameSessions.delete(frame); this.frameTargets.delete(frame); } + } + + private invalidateRefs(targetId: string): void { this.invalidateFrame(targetId); } + private invalidateFrame(frameKey: string): void { - let tracked = this.generations.has(frameKey) || this.frameSessions.has(frameKey); - for (const [ref, entry] of this.refs) { - if (entry.frameId === frameKey) { - this.refs.delete(ref); - tracked = true; - } + const frames = this.descendants(frameKey); + const page = this.pageForFrame(frameKey) ?? this.frameOwners.get(frameKey) ?? frameKey; + this.navigationEpochs.set(page, this.navigationEpoch(page) + 1); + for (const [ref, entry] of this.refs) if (frames.includes(entry.frameId) || (entry.sessionTargetId && frames.includes(entry.sessionTargetId))) this.refs.delete(ref); + this.generations.set(frameKey, this.generation(frameKey) + 1); + this.removeFrames(frames.filter((frame) => frame !== frameKey)); + } + + private pruneFrameState(targetId: string, observed: ReadonlySet): void { + const stale = this.descendants(targetId).filter((frame) => frame !== targetId && !observed.has(frame) && !this.frameSessions.has(frame)); + this.removeFrames(stale); + } + + private boundFrameState(observed: ReadonlySet): void { + const protectedFrames = new Set(observed); + for (const frame of this.frameSessions.keys()) protectedFrames.add(frame); + for (const entry of this.refs.values()) protectedFrames.add(entry.frameId); + for (const frame of [...protectedFrames]) { + let parent = this.frameParents.get(frame); + const seen = new Set(); + while (parent && !seen.has(parent)) { seen.add(parent); protectedFrames.add(parent); parent = this.frameParents.get(parent); } + } + for (const frame of this.frameParents.keys()) { + if (this.frameParents.size <= FRAME_STATE_LIMIT) break; + if (!protectedFrames.has(frame)) this.removeFrames(this.descendants(frame)); } - // Frames we never referenced don't get a generation entry, or pages with - // rotating ad iframes would grow the map without bound. - if (tracked) this.generations.set(frameKey, this.generation(frameKey) + 1); } private dropTarget(targetId: string): void { - this.generations.delete(targetId); + const page = this.framePages.get(targetId); + if (page) this.navigationEpochs.set(page, this.navigationEpoch(page) + 1); + const owned = [...this.framePages].filter(([, page]) => page === targetId).flatMap(([frame]) => this.descendants(frame)); + this.removeFrames([...new Set([...this.descendants(targetId), ...owned])]); this.selfNavigations.delete(targetId); this.lastSnapshots.delete(targetId); - this.frameSessions.delete(targetId); - this.frameTargets.delete(targetId); - for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId || entry.frameId === targetId) { - if (entry.frameId !== targetId) this.generations.delete(entry.frameId); - this.refs.delete(ref); - } - } + this.navigationEpochs.delete(targetId); + for (const [ref, entry] of this.refs) if (entry.targetId === targetId) this.refs.delete(ref); } /** SPAs can mint refs indefinitely without ever navigating; bound per-target growth by evicting the oldest. */ @@ -955,6 +1119,11 @@ function staleRefError(ref: string, cause?: unknown): Error { return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); } +function normalizeState(value: unknown): boolean | "mixed" { + if (value === "mixed") return "mixed"; + return value === true || value === "true"; +} + function collectStates(node: AXNode): string[] { const states: string[] = []; for (const property of node.properties ?? []) { diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index 55f793d7..26d3fdcb 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -1,8 +1,36 @@ +/** Three-valued semantic expectation result with model-facing evidence. */ +export interface BrowserExpectationEvidence { + truth?: boolean; + details: string[]; +} + +/** Browser lifecycle or observation reason that terminated a semantic wait. */ +export type BrowserWaitReason = + | "navigation" + | "dialog" + | "target_changed" + | "target_detached" + | "stale_ref" + | "observation_failed" + | "incomplete_observation"; + +/** Structured result returned by the standalone `browser_wait_for` action. */ +export interface BrowserWaitForResult { + status: "satisfied" | "timed_out" | "unverifiable" | "interrupted"; + evidence: "preexisting" | "newly_verified" | "failed" | "unverifiable"; + initial: BrowserExpectationEvidence; + final: BrowserExpectationEvidence; + elapsed_ms: number; + reason?: BrowserWaitReason; + details: string[]; +} + export type BatchReadResult = | { type: "screenshot"; data: Buffer; mimeType: string } | { type: "url"; url: string } | { type: "cursor_position"; x: number; y: number } - | { type: "browser_text"; label: string; text: string }; + | { type: "browser_text"; label: string; text: string } + | { type: "browser_wait_for"; result: BrowserWaitForResult }; export interface BatchExecutionResult { readResults: BatchReadResult[]; diff --git a/packages/agent/test/browser-wait.test.ts b/packages/agent/test/browser-wait.test.ts new file mode 100644 index 00000000..0422c7da --- /dev/null +++ b/packages/agent/test/browser-wait.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it } from "vitest"; +import type { CuaBrowserExpectation } from "@onkernel/cua-ai"; +import { buildNthIndex, type AXNode, type BrowserObservation, type RenderContext } from "../src/translator/browser-observation"; +import { evaluateBrowserExpectation, waitForBrowserExpectation, type BrowserRefResolver } from "../src/translator/browser-wait"; + +const nodes = (names: string[]): AXNode[] => [ + { nodeId: "root", role: { value: "RootWebArea" }, childIds: names.map((_, index) => `n${index}`) }, + ...names.map((name, index) => ({ nodeId: `n${index}`, parentId: "root", role: { value: index === 0 ? "button" : "StaticText" }, name: { value: name }, backendDOMNodeId: index + 1 })), +]; + +function observation(names: string[] = [], complete = true, overrides: Partial = {}): BrowserObservation { + const list = nodes(names); + const ctx: RenderContext = { targetId: "page", frameKey: "page", sessionTargetId: "page", sessionId: "s", generation: 0, interactiveOnly: false, nthIndex: buildNthIndex(list) }; + const tree = { byId: new Map(list.map((node) => [node.nodeId, node])), roots: ["root"], ctx }; + return { targetId: "page", navigationEpoch: 0, tree, stitches: new Map(), nodes: list.map((node) => ({ node, ctx })), url: "https://a.test", title: "A", generations: new Map([["page", 0]]), complete, ...overrides }; +} + +const missingRef: BrowserRefResolver = (expectation) => ({ truth: undefined, details: [`${expectation.ref} stale`], reason: "stale_ref" }); + +describe("evaluateBrowserExpectation", () => { + it.each([ + [{ type: "text", text: "SAVE" }, true], + [{ type: "role_name", role: "button", name: "Save" }, true], + [{ type: "role_name", role: "button", name: "save" }, false], + [{ type: "url", contains: "a.test" }, true], + [{ type: "title", equals: "A" }, true], + [{ type: "url", changed: false }, true], + ] as Array<[CuaBrowserExpectation, boolean]>) ("evaluates %#", (condition, truth) => { + const current = observation(["Save"]); + expect(evaluateBrowserExpectation(condition, current, current, missingRef).truth).toBe(truth); + }); + + it("merges static text runs and ignores ignored nodes", () => { + const current = observation(["button", "Hello", "world"], false); + current.nodes[2]!.node.ignored = true; + expect(evaluateBrowserExpectation({ type: "text", text: "Hello" }, current, current, missingRef).truth).toBeUndefined(); + current.nodes[2]!.node.ignored = false; + expect(evaluateBrowserExpectation({ type: "text", text: "Hello world" }, current, current, missingRef).truth).toBe(true); + }); + + it.each([ + [true, undefined, { all: [{ type: "text", text: "Save" }, { type: "text", text: "Missing" }] }, undefined], + [false, undefined, { all: [{ type: "text", text: "Save", exists: false }, { type: "text", text: "Missing" }] }, false], + [true, undefined, { any: [{ type: "text", text: "Save" }, { type: "text", text: "Missing" }] }, true], + [false, undefined, { any: [{ type: "text", text: "No" }, { type: "text", text: "Missing" }] }, undefined], + ] as Array<[boolean, undefined, CuaBrowserExpectation, boolean | undefined]>) ("combines three-valued condition %#", (_a, _b, condition, truth) => { + const current = observation(["Save"], false); + expect(evaluateBrowserExpectation(condition, current, current, missingRef).truth).toBe(truth); + }); + + it("distinguishes complete absence from incomplete absence", () => { + expect(evaluateBrowserExpectation({ type: "text", text: "Gone", exists: false }, observation(), observation(), missingRef).truth).toBe(true); + const incomplete = observation([], false); + expect(evaluateBrowserExpectation({ type: "text", text: "Gone", exists: false }, incomplete, incomplete, missingRef).truth).toBeUndefined(); + }); +}); + +describe("waitForBrowserExpectation", () => { + it.each([ + [[["Ready"]], "satisfied", "preexisting"], + [[[], ["Ready"]], "satisfied", "newly_verified"], + [[[]], "timed_out", "failed"], + ] as Array<[string[][], string, string]>) ("returns %s as %s", async (states, status, evidence) => { + let time = 0, read = 0; + const result = await waitForBrowserExpectation({ + selectTarget: async () => "page", observeTarget: async () => observation(states[Math.min(read++, states.length - 1)]!), + dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, + now: () => time, delay: async (ms) => { time += ms; }, + }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); + expect(result).toMatchObject({ status, evidence }); + }); + + it("reports initial observation failures and stale refs honestly", async () => { + const failed = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => { throw new Error("boom"); }, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef }, { expect: { type: "text", text: "Ready" }, timeoutMs: 10 }); + expect(failed).toMatchObject({ status: "unverifiable", reason: "observation_failed" }); + const stale = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation(), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef }, { expect: { type: "ref", ref: "e1", checked: true }, timeoutMs: 10 }); + expect(stale).toMatchObject({ status: "interrupted", reason: "stale_ref" }); + }); + + it.each([ + ["target_detached", { exists: false }], + ["target_changed", { targetId: "other" }], + ["dialog", { dialog: true }], + ["navigation", { navigationEpoch: 1 }], + ["observation_failed", { observationError: true }], + ] as const)("returns the public %s interruption reason", async (reason, scenario) => { + let time = 0, reads = 0, dialogReads = 0; + const result = await waitForBrowserExpectation({ + selectTarget: async () => "page", + observeTarget: async () => { + if (reads++ > 0 && "observationError" in scenario) throw new Error("boom"); + return observation([], true, { ...(reads > 1 && scenario.targetId ? { targetId: scenario.targetId } : {}), ...(reads > 1 && scenario.navigationEpoch ? { navigationEpoch: scenario.navigationEpoch } : {}) }); + }, + dialogCount: () => "dialog" in scenario && scenario.dialog && dialogReads++ > 0 ? 1 : 0, + targetExists: async () => !("exists" in scenario) || scenario.exists, + liveGeneration: () => 0, resolveRef: missingRef, + now: () => time, delay: async (ms) => { time += ms; }, + }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); + expect(result).toMatchObject({ status: reason === "observation_failed" ? "unverifiable" : "interrupted", reason }); + }); + + it("treats a dropped baseline frame as incomplete rather than navigation or absence", async () => { + let time = 0, reads = 0; + const baseline = observation(["Gone"]); + baseline.generations.set("frame", 1); + const result = await waitForBrowserExpectation({ + selectTarget: async () => "page", observeTarget: async () => reads++ === 0 ? baseline : observation(), + dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, + resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, + }, { expect: { type: "text", text: "Gone", exists: false }, timeoutMs: 20, pollMs: 10 }); + expect(result).toMatchObject({ status: "unverifiable", reason: "incomplete_observation" }); + }); + + it("interrupts when a ref becomes stale after the baseline", async () => { + let time = 0, resolves = 0; + const resolveRef: BrowserRefResolver = () => resolves++ === 0 ? { truth: false, details: ["not checked"] } : { truth: undefined, details: ["stale"], reason: "stale_ref" }; + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation(), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "ref", ref: "e1", checked: true }, timeoutMs: 20, pollMs: 10 }); + expect(result).toMatchObject({ status: "interrupted", reason: "stale_ref" }); + }); + + it("retries transient incomplete observations", async () => { + let time = 0, reads = 0; + const states = [observation(), observation([], false), observation(["Ready"])]; + const result = await waitForBrowserExpectation({ + selectTarget: async () => "page", observeTarget: async () => states[Math.min(reads++, states.length - 1)]!, + dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, + resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, + }, { expect: { type: "text", text: "Ready" }, timeoutMs: 30, pollMs: 10 }); + expect(result).toMatchObject({ status: "satisfied", evidence: "newly_verified" }); + }); + + it("returns incomplete observations as unverifiable", async () => { + let time = 0; + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation([], false), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "text", text: "Gone", exists: false }, timeoutMs: 10, pollMs: 10 }); + expect(result).toMatchObject({ status: "unverifiable", reason: "incomplete_observation" }); + }); + + it("allows location expectations to be satisfied by navigation", async () => { + let time = 0, reads = 0; + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => reads++ === 0 ? observation() : observation([], true, { url: "https://a.test/done", navigationEpoch: 1 }), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "url", contains: "/done" }, timeoutMs: 20, pollMs: 10 }); + expect(result).toMatchObject({ status: "satisfied", evidence: "newly_verified" }); + }); + + it("reports navigation instead of timeout when a location change lands at the deadline", async () => { + let time = 0, reads = 0; + const result = await waitForBrowserExpectation({ + selectTarget: async () => "page", + observeTarget: async () => { + if (reads++ === 0) return observation(); + time = 100; + return observation([], true, { url: "https://a.test/other", navigationEpoch: 1 }); + }, + dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, + resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, + }, { expect: { type: "url", contains: "/done" }, timeoutMs: 100, pollMs: 10 }); + expect(result).toMatchObject({ status: "interrupted", reason: "navigation" }); + }); + + it("does not accept a condition completed after the deadline", async () => { + let time = 0, reads = 0; + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => { if (reads++ > 0) time += 20; return observation(reads > 1 ? ["Ready"] : []); }, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); + expect(result.status).toBe("timed_out"); + }); + + it.each(["select", "observe", "evaluate"] as const)("includes initial %s in the hard deadline", async (phase) => { + let time = 0; + const result = await waitForBrowserExpectation({ + selectTarget: async () => { if (phase === "select") time = 10; return "page"; }, + observeTarget: async () => { if (phase === "observe") time = 10; return observation(); }, + dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, + resolveRef: () => { if (phase === "evaluate") time = 10; return { truth: true, details: [] }; }, + now: () => time, + }, { expect: phase === "evaluate" ? { type: "ref", ref: "e1", checked: true } : { type: "text", text: "Ready" }, timeoutMs: 10 }); + expect(result.status).toBe("timed_out"); + }); +}); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 2ec57aab..4f64fcc7 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -110,6 +110,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { const frameTrees = new Map>(); const iframeFrameIds = new Map(); const autoAttachFrames: Array<{ targetId: string; sessionId: string }> = []; + const targetSessions = new Map(); const failMethods = new Set(); let axRead = 0; let targetRead = 0; @@ -179,7 +180,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { }, pageTargets: async () => targetProvider?.(++targetRead) ?? [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], - attachToTarget: async () => "session-1", + attachToTarget: async (targetId: string) => targetSessions.get(targetId) ?? "session-1", createTarget: async () => "TARGET-2", close: () => {}, }; @@ -201,6 +202,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { const addAutoAttachFrame = (frame: { targetId: string; sessionId: string }) => { autoAttachFrames.push(frame); }; + const setTargetSession = (targetId: string, sessionId: string) => targetSessions.set(targetId, sessionId); const failOn = (method: string) => { failMethods.add(method); }; @@ -219,6 +221,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { setFrameTree, setIframeFrame, addAutoAttachFrame, + setTargetSession, failOn, setAxReadHook, setTargetProvider, @@ -305,6 +308,28 @@ describe("BrowserExecutor ref lifecycle", () => { expect(refsOf(executor).size).toBe(0); }); + it("attributes unseen subframe navigation and detach to the owning page", async () => { + const { cdp, emit } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + const epochs = (executor as unknown as { navigationEpochs: Map }).navigationEpochs; + + emit({ method: "Page.frameNavigated", params: { frame: { id: "NEW-FRAME", parentId: "TARGET-1" } }, sessionId: "session-1" }); + expect(epochs.get("TARGET-1")).toBe(1); + + emit({ method: "Page.frameAttached", params: { frameId: "DETACHED-FRAME", parentFrameId: "NEW-FRAME" }, sessionId: "session-1" }); + emit({ method: "Page.frameDetached", params: { frameId: "DETACHED-FRAME" }, sessionId: "session-1" }); + expect(epochs.get("TARGET-1")).toBe(2); + }); + + it("invalidates refs on same-document navigation", async () => { + const { cdp, emit } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + emit({ method: "Page.navigatedWithinDocument", params: { frameId: "TARGET-1", url: "https://a.test/#section" }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e1" } as CuaBrowserAction)).rejects.toThrow(/stale/); + }); + it("does not let a same-document navigation suppress the next real navigation's invalidation", async () => { const { cdp, emit } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); @@ -674,6 +699,40 @@ describe("BrowserExecutor snapshot diffing", () => { }); }); +describe("BrowserExecutor semantic waits", () => { + it.each([ + ["selected target", "TARGET-1", "session-1", "interrupted", "navigation"], + ["unrelated target", "TARGET-2", "session-2", "timed_out", undefined], + ] as const)("handles same-document navigation on the %s", async (_label, frameId, sessionId, status, reason) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setTargetSession("TARGET-2", "session-2"); + fake.setTargetProvider(() => [ + { targetId: "TARGET-1", type: "page", title: "One", url: "https://a.test/" }, + { targetId: "TARGET-2", type: "page", title: "Two", url: "https://b.test/" }, + ]); + const executor = new BrowserExecutor(fake.cdp); + await executor.execute({ type: "browser_text", tab_id: "TARGET-2" } as CuaBrowserAction); + fake.setAxReadHook((read) => { + if (read === 2) fake.emit({ method: "Page.navigatedWithinDocument", params: { frameId }, sessionId }); + }); + const [read] = await executor.execute({ type: "browser_wait_for", tab_id: "TARGET-1", expect: { type: "text", text: "Ready" }, timeout_ms: 20, poll_ms: 1 } as CuaBrowserAction); + expect(read).toMatchObject({ type: "browser_wait_for", result: { status, ...(reason ? { reason } : {}) } }); + }); + + it.each([ + ["missing value", {}, { value: "" }, "unverifiable"], + ["empty value", { value: "" }, { value: "" }, "satisfied"], + ["missing false state", {}, { checked: false }, "unverifiable"], + ["actual false state", { properties: [{ name: "checked", value: false }] }, { checked: false }, "satisfied"], + ] as const)("treats %s distinctly", async (_label, metadata, expected, status) => { + const tree = [ax({ nodeId: "1", role: "RootWebArea", childIds: ["2"] }), ax({ nodeId: "2", role: "checkbox", name: "Option", backendDOMNodeId: 42, parentId: "1", ...metadata })]; + const executor = new BrowserExecutor(createFakeCdp(tree).cdp); + await snapshotText(executor); + const [read] = await executor.execute({ type: "browser_wait_for", expect: { type: "ref", ref: "e1", ...expected }, timeout_ms: 2, poll_ms: 1 } as CuaBrowserAction); + expect(read).toMatchObject({ type: "browser_wait_for", result: { status } }); + }); +}); + describe("BrowserExecutor iframe stitching", () => { it("stitches a same-process iframe subtree indented under its iframe node", async () => { const tree = [ @@ -726,6 +785,44 @@ describe("BrowserExecutor iframe stitching", () => { expect(pressed?.sessionId).toBe("session-1"); }); + it("rebinds an imported OOPIF ref after auto-attach", async () => { + const first = new BrowserExecutor(setupOopif().cdp); + await snapshotText(first); + const state = first.exportRefState(); + const secondCdp = setupOopif(); + const second = new BrowserExecutor(secondCdp.cdp); + second.importRefState(state); + await second.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction); + expect(secondCdp.sent.find((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 70)?.sessionId).toBe("session-oop"); + }); + + it("uses the OOPIF session for a nested same-process frame, including imported refs", async () => { + const nestedSetup = () => { + const fake = createFakeCdp(OOPIF_PAGE); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.setIframeFrame(80, "FRAME-INNER"); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setSessionTree("session-oop", [ + ax({ nodeId: "o1", role: "RootWebArea", childIds: ["o2"] }), + ax({ nodeId: "o2", role: "Iframe", backendDOMNodeId: 80, parentId: "o1" }), + ax({ nodeId: "hidden", role: "button", name: "Nested", backendDOMNodeId: 90, parentId: "unrendered" }), + ]); + fake.setFrameTree("FRAME-INNER", [ax({ nodeId: "i1", role: "RootWebArea", childIds: ["i2"] }), ax({ nodeId: "i2", role: "button", name: "Nested", backendDOMNodeId: 90, parentId: "i1" })]); + return fake; + }; + const firstCdp = nestedSetup(); + const first = new BrowserExecutor(firstCdp.cdp); + expect(await snapshotText(first)).toContain('button "Nested" [e4]'); + await first.execute({ type: "browser_click", ref: "e4" } as CuaBrowserAction); + expect(firstCdp.sent.find((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 90)?.sessionId).toBe("session-oop"); + + const secondCdp = nestedSetup(); + const second = new BrowserExecutor(secondCdp.cdp); + second.importRefState(first.exportRefState()); + await second.execute({ type: "browser_click", ref: "e4" } as CuaBrowserAction); + expect(secondCdp.sent.find((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 90)?.sessionId).toBe("session-oop"); + }); + it("invalidates a frame target's refs when a subframe inside it navigates", async () => { const { cdp, emit } = setupOopif(); const executor = new BrowserExecutor(cdp); @@ -735,6 +832,31 @@ describe("BrowserExecutor iframe stitching", () => { await expect(executor.execute({ type: "browser_click", ref: "e3" } as CuaBrowserAction)).rejects.toThrow(/stale/); }); + it("keeps attached OOPIF sessions when a complete observation temporarily omits the frame", async () => { + const fake = setupOopif(); + const executor = new BrowserExecutor(fake.cdp); + expect(await snapshotText(executor)).toContain('button "Pay"'); + fake.setNodes(BUTTON_TREE); + await snapshotText(executor); + fake.setNodes(OOPIF_PAGE); + expect(await snapshotText(executor)).toContain('button "Pay"'); + }); + + it("treats a page-target iframe stub as complete", async () => { + const fake = createFakeCdp(OOPIF_PAGE); + fake.setIframeFrame(50, "TARGET-1"); + const executor = new BrowserExecutor(fake.cdp); + const [read] = await executor.execute({ type: "browser_wait_for", expect: { type: "text", text: "Missing", exists: false }, timeout_ms: 20 } as CuaBrowserAction); + expect(read).toMatchObject({ type: "browser_wait_for", result: { status: "satisfied", evidence: "preexisting" } }); + }); + + it("waits on semantic content inside stitched iframes", async () => { + const { cdp } = setupOopif(); + const executor = new BrowserExecutor(cdp); + const [read] = await executor.execute({ type: "browser_wait_for", expect: { type: "text", text: "Pay" }, timeout_ms: 20 } as CuaBrowserAction); + expect(read).toMatchObject({ type: "browser_wait_for", result: { status: "satisfied", evidence: "preexisting" } }); + }); + it("finds elements inside stitched iframes", async () => { const { cdp, sent } = setupOopif(); const executor = new BrowserExecutor(cdp); diff --git a/packages/ai/src/actions/browser.ts b/packages/ai/src/actions/browser.ts index 55589ac6..8076b6fb 100644 --- a/packages/ai/src/actions/browser.ts +++ b/packages/ai/src/actions/browser.ts @@ -17,6 +17,7 @@ import { Type, type TSchema } from "@earendil-works/pi-ai"; */ export const CUA_BROWSER_ACTION_TYPES = [ "browser_snapshot", + "browser_wait_for", "browser_text", "browser_find", "browser_click", @@ -44,6 +45,39 @@ export interface CuaActionBrowserSnapshot { tab_id?: string; } +type RoleNameExpectation = + | { type: "role_name"; role: string; name?: string; exists?: boolean } + | { type: "role_name"; role?: string; name: string; exists?: boolean }; +type RefExpectationState = { value?: string; checked?: boolean | "mixed"; selected?: boolean; expanded?: boolean }; +type RefExpectation = { type: "ref"; ref: string } & ( + | (RefExpectationState & { value: string }) + | (RefExpectationState & { checked: boolean | "mixed" }) + | (RefExpectationState & { selected: boolean }) + | (RefExpectationState & { expanded: boolean }) +); +type LocationExpectation = { type: "url" | "title" } & ( + | { equals: string; contains?: string; changed?: boolean } + | { equals?: string; contains: string; changed?: boolean } + | { equals?: string; contains?: string; changed: boolean } +); +type CuaBrowserExpectationLeaf = + | { type: "text"; text: string; exists?: boolean } + | RoleNameExpectation + | RefExpectation + | LocationExpectation; +type NonEmptyArray = [T, ...T[]]; +/** Semantic condition over accessible content, ref state, location, or all/any leaf groups. */ +export type CuaBrowserExpectation = CuaBrowserExpectationLeaf | { all: NonEmptyArray } | { any: NonEmptyArray }; + +export interface CuaActionBrowserWaitFor { + type: "browser_wait_for"; + expect: CuaBrowserExpectation; + /** Semantic polling timeout; an in-flight browser read settles before timeout is reported. */ + timeout_ms?: number; + poll_ms?: number; + tab_id?: string; +} + export interface CuaActionBrowserText { type: "browser_text"; tab_id?: string; @@ -146,6 +180,7 @@ export interface CuaActionBrowserEvaluate { export type CuaBrowserAction = | CuaActionBrowserSnapshot + | CuaActionBrowserWaitFor | CuaActionBrowserText | CuaActionBrowserFind | CuaActionBrowserClick @@ -179,6 +214,13 @@ const TabId = () => Type.Optional(Type.String({ description: "Tab to act on. Def const RefProperty = () => Type.String({ description: "Element reference from browser_snapshot or browser_find, e.g. \"e12\"." }); export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOptions): Record { + const roleName = (required: "role" | "name") => Type.Object({ type: Type.Literal("role_name"), role: required === "role" ? Type.String() : Type.Optional(Type.String()), name: required === "name" ? Type.String() : Type.Optional(Type.String()), exists: Type.Optional(Type.Boolean()) }, { additionalProperties: false }); + const refState = (required: "value" | "checked" | "selected" | "expanded") => Type.Object({ type: Type.Literal("ref"), ref: RefProperty(), value: required === "value" ? Type.String() : Type.Optional(Type.String()), checked: required === "checked" ? Type.Union([Type.Boolean(), Type.Literal("mixed")]) : Type.Optional(Type.Union([Type.Boolean(), Type.Literal("mixed")])), selected: required === "selected" ? Type.Boolean() : Type.Optional(Type.Boolean()), expanded: required === "expanded" ? Type.Boolean() : Type.Optional(Type.Boolean()) }, { additionalProperties: false }); + const location = (required: "equals" | "contains" | "changed") => Type.Object({ type: Type.Union([Type.Literal("url"), Type.Literal("title")]), equals: required === "equals" ? Type.String() : Type.Optional(Type.String()), contains: required === "contains" ? Type.String() : Type.Optional(Type.String()), changed: required === "changed" ? Type.Boolean() : Type.Optional(Type.Boolean()) }, { additionalProperties: false }); + const leaves = [Type.Object({ type: Type.Literal("text"), text: Type.String(), exists: Type.Optional(Type.Boolean()) }, { additionalProperties: false }), roleName("role"), roleName("name"), refState("value"), refState("checked"), refState("selected"), refState("expanded"), location("equals"), location("contains"), location("changed")]; + const leaf = Type.Union(leaves); + const expectation = Type.Union([leaf, Type.Object({ all: Type.Array(leaf, { minItems: 1 }) }, { additionalProperties: false }), Type.Object({ any: Type.Array(leaf, { minItems: 1 }) }, { additionalProperties: false })]); + const clickTarget: Record = options.coordinates ? { ref: Type.Optional(RefProperty()), @@ -198,6 +240,10 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti }, { additionalProperties: false }, ), + browser_wait_for: Type.Object( + { type: Type.Literal("browser_wait_for"), expect: expectation, timeout_ms: Type.Optional(Type.Number({ minimum: 0, maximum: 30_000, description: "Semantic polling timeout; in-flight browser reads settle before timeout is reported." })), poll_ms: Type.Optional(Type.Number({ minimum: 10, maximum: 1_000 })), tab_id: TabId() }, + { additionalProperties: false }, + ), browser_text: Type.Object( { type: Type.Literal("browser_text"), diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index cbc83f7d..a7922ddb 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -56,6 +56,7 @@ export const CUA_HYBRID_COMPUTER_ACTION_TYPES: readonly CuaComputerActionType[] */ export const CUA_HYBRID_BROWSER_ACTION_TYPES: readonly CuaBrowserActionType[] = [ "browser_snapshot", + "browser_wait_for", "browser_text", "browser_find", "browser_click", @@ -109,6 +110,7 @@ export function cuaToolNameForAction(action: CuaActionType, mode: CuaMode): stri } const BROWSER_ACTION_DESCRIPTIONS: Record = { + browser_wait_for: "Wait for a semantic page condition without delivering input.", browser_snapshot: "Return an accessibility-tree snapshot of the page, including iframe content, with element references like [e12]. " + "Use the refs to target elements in other page tools. Refs are only valid until the page changes; re-snapshot when told a ref is stale. " + @@ -149,6 +151,7 @@ const HYBRID_BROWSER_DESCRIPTION_OVERRIDES: Partial { for (const action of CUA_BROWSER_ACTION_TYPES) { expect(names).toContain(action.slice("browser_".length)); } + expect(names).toContain("wait_for"); + }); + + it("registers semantic waits in hybrid mode with bounded polling", () => { + const wait = computerTools({ mode: "hybrid" }).find((tool) => tool.name === "browser_wait_for")!; + expect(wait.parameters.properties.timeout_ms).toMatchObject({ minimum: 0, maximum: 30_000 }); + expect(wait.parameters.properties.poll_ms).toMatchObject({ minimum: 10, maximum: 1_000 }); + }); + + it.each([ + { type: "text", text: "Ready" }, + { type: "role_name", role: "button" }, + { type: "ref", ref: "e1", checked: true }, + { type: "url", changed: true }, + { all: [{ type: "text", text: "Ready" }] }, + { any: [{ type: "title", contains: "Done" }] }, + ])("accepts semantic expectation %#", (condition) => { + const tool = computerTools({ mode: "hybrid" }).find((candidate) => candidate.name === "browser_wait_for")!; + expect(() => validateToolArguments(tool, { type: "toolCall", id: "1", name: tool.name, arguments: { expect: condition } })).not.toThrow(); + }); + + it.each([ + {}, + { type: "text" }, + { type: "role_name" }, + { type: "ref", ref: "e1" }, + { type: "url" }, + { all: [] }, + { all: [{ any: [{ type: "text", text: "nested" }] }] }, + ])("rejects malformed semantic expectation %#", (condition) => { + const tool = computerTools({ mode: "hybrid" }).find((candidate) => candidate.name === "browser_wait_for")!; + expect(() => validateToolArguments(tool, { type: "toolCall", id: "1", name: tool.name, arguments: { expect: condition } })).toThrow(); }); });