From 80564137707c2e53bcdc2e5f39366fb31922b17c Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 14 Jul 2026 01:52:24 +0000 Subject: [PATCH] Add verified browser action plans --- packages/agent/src/tools.ts | 47 ++- packages/agent/src/translator/browser-act.ts | 212 ++++++++++ .../src/translator/browser-observation.ts | 23 ++ packages/agent/src/translator/browser-wait.ts | 24 +- packages/agent/src/translator/browser.ts | 52 ++- packages/agent/src/translator/translator.ts | 5 +- packages/agent/src/translator/types.ts | 47 ++- packages/agent/test/browser-wait.test.ts | 34 +- .../agent/test/translator-browser.test.ts | 391 +++++++++++++++++- packages/ai/src/actions/browser.ts | 50 ++- packages/ai/src/modes.ts | 2 + packages/ai/test/modes.test.ts | 25 +- 12 files changed, 877 insertions(+), 35 deletions(-) create mode 100644 packages/agent/src/translator/browser-act.ts diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index e31e8941..4c15c039 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -16,7 +16,9 @@ import { } from "@onkernel/cua-ai"; import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core"; -import type { BrowserWaitForResult } from "./translator/types"; +import type { BrowserActResult, BrowserWaitForResult } from "./translator/types"; + +const BROWSER_ACT_DIFF_LINE_LIMIT = 200; export interface ComputerToolOptions { browser: KernelBrowser; @@ -39,6 +41,7 @@ export interface BatchDetails { | { type: "cursor_position"; x: number; y: number } | { type: "browser_text"; label: string; bytes: number } | { type: "browser_wait_for"; result: BrowserWaitForResult } + | { type: "browser_act"; result: BrowserActResult } >; } @@ -175,6 +178,9 @@ async function executeBatchTool( } else if (read.type === "browser_wait_for") { readResults.push(read); content.push({ type: "text", text: formatBrowserWaitResult(read.result) }); + } else if (read.type === "browser_act") { + readResults.push(read); + content.push({ type: "text", text: formatBrowserActResult(read.result) }); } else { readResults.push({ type: "screenshot", bytes: read.data.length }); content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType }); @@ -190,13 +196,18 @@ async function executeBatchTool( } catch (err) { throw new Error(`Actions failed: ${errorMessage(err)}`, { cause: err }); } + const acts = readResults.flatMap((read) => read.type === "browser_act" ? [read.result] : []); const waits = readResults.flatMap((read) => read.type === "browser_wait_for" ? [read.result] : []); const failedWait = ["interrupted", "timed_out", "unverifiable"].find((status) => waits.some((wait) => wait.status === status)); - const statusText = waits.length === 0 - ? "Actions executed successfully." + const statusText = acts.some((act) => act.outcome === "didnt") + ? "Browser action plan did not satisfy its expectations." : failedWait ? `Browser condition ${failedWait}.` - : "Browser condition satisfied."; + : acts.some((act) => act.outcome === "unknown") + ? "Browser action plan outcome is unknown." + : acts.length > 0 + ? "Browser action plan worked." + : waits.length > 0 ? "Browser condition satisfied." : "Actions executed successfully."; return { content, details: { statusText, readResults } }; } @@ -274,6 +285,34 @@ function formatBrowserWaitResult(result: BrowserWaitForResult): string { return [`wait_for: ${result.status}/${result.evidence}${reason} after ${result.elapsed_ms}ms`, ...result.details].join("\n"); } +/** Format a browser action result for the model while bounding displayed diff lines. */ +export function formatBrowserActResult(result: BrowserActResult): string { + const lines = [`browser_act: ${result.outcome}`]; + if (result.stopped_at !== undefined) lines.push(`stopped_at: ${result.stopped_at} (${result.stop_reason ?? "unknown"})`); + for (const step of result.steps) { + lines.push(`step ${step.index} ${step.type}: ${step.outcome} — ${step.evidence.join("; ")}`); + for (const detail of step.expectation?.details ?? []) lines.push(` ${detail}`); + } + if (result.final_expectation) { + lines.push(`final expectation: ${result.final_expectation.status}`); + for (const detail of result.final_expectation.details) lines.push(` ${detail}`); + } + if (result.successor.status === "unavailable") { + lines.push(`successor unavailable: ${result.successor.error}`); + return lines.join("\n"); + } + const { diff } = result.successor; + lines.push(`successor: ${result.successor.title} (${result.successor.url})`); + lines.push(`diff: ${diff.changed ? `+${diff.added.length} -${diff.removed.length}` : "unchanged"}`); + if (diff.url) lines.push(` url: ${diff.url.before} -> ${diff.url.after}`); + if (diff.title) lines.push(` title: ${diff.title.before} -> ${diff.title.after}`); + const changes = [...diff.added.map((line) => ` + ${line}`), ...diff.removed.map((line) => ` - ${line}`)]; + lines.push(...changes.slice(0, BROWSER_ACT_DIFF_LINE_LIMIT)); + if (changes.length > BROWSER_ACT_DIFF_LINE_LIMIT) lines.push(` … ${changes.length - BROWSER_ACT_DIFF_LINE_LIMIT} more diff lines omitted`); + lines.push(result.successor.text); + return lines.join("\n"); +} + function formatPlaywrightResult(result: unknown): string { return typeof result === "string" ? result : JSON.stringify(result); } diff --git a/packages/agent/src/translator/browser-act.ts b/packages/agent/src/translator/browser-act.ts new file mode 100644 index 00000000..eaca9983 --- /dev/null +++ b/packages/agent/src/translator/browser-act.ts @@ -0,0 +1,212 @@ +import type { CuaActionBrowserAct, CuaActionBrowserSnapshot, CuaBrowserActStep, CuaBrowserExpectation } from "@onkernel/cua-ai"; +import { diffObservations, type BrowserObservation, type BrowserPresentation } from "./browser-observation"; +import type { BrowserExpectationEvaluation } from "./browser-wait"; +import type { BrowserActExpectationEvidence, BrowserActResult, BrowserActStepResult, BrowserWaitForResult } from "./types"; + +/** Browser capabilities injected into the dependent action-plan orchestrator. */ +export interface BrowserActRuntime { + observe(tabId?: string): Promise; + targetIds(): Promise; + dialogCount(): number; + liveGeneration(frameId: string): number; + liveNavigationEpoch(targetId: string): number; + executeStep(step: CuaBrowserActStep, tabId?: string): Promise; + wait(expect: CuaBrowserExpectation, baseline: BrowserObservation, targetId: string, tabId?: string, timeoutMs?: number, pollMs?: number): Promise; + evaluate(expect: CuaBrowserExpectation, observation: BrowserObservation, baseline: BrowserObservation): BrowserExpectationEvaluation; + present(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation; + render(presentation: BrowserPresentation): string; +} + +/** Execute a dependent action plan and derive outcomes, stop reasons, and successor state. */ +export async function runBrowserAct(action: CuaActionBrowserAct, runtime: BrowserActRuntime): Promise { + const finalStepIndex = action.steps.length - 1; + let baseline: BrowserObservation; + let current: BrowserObservation; + let targets: string[]; + try { + baseline = current = await runtime.observe(action.tab_id); + if (!baseline.complete) throw new Error("baseline observation incomplete"); + targets = await runtime.targetIds(); + } catch (error) { + return unavailable(error, 0); + } + let dialogs = runtime.dialogCount(); + const steps: BrowserActStepResult[] = []; + let stoppedAt: number | undefined; + let stopReason: BrowserActResult["stop_reason"]; + let timedOut = false; + + for (let index = 0; index < action.steps.length; index += 1) { + const step = action.steps[index]!; + let before: BrowserObservation; + let nextTargets: string[]; + try { + before = await runtime.observe(action.tab_id); + nextTargets = await runtime.targetIds(); + } catch (error) { + steps.push(stepResult(index, step, "unknown", [`pre-action observation failed: ${message(error)}`])); + stoppedAt = index; stopReason = "control_flow"; break; + } + const preBoundary = boundary(current, before, targets, nextTargets, dialogs, runtime); + current = before; targets = nextTargets; dialogs = runtime.dialogCount(); + if (!before.complete || preBoundary) { + const reason = before.complete ? `${preBoundary} detected before action` : "pre-action observation incomplete"; + steps.push(stepResult(index, step, "unknown", [reason])); + stoppedAt = index; stopReason = preBoundary ?? "control_flow"; break; + } + + const evidence: string[] = []; + let actionError: unknown; + try { + await runtime.executeStep(step, action.tab_id); + evidence.push("action dispatched"); + } catch (error) { + actionError = error; + evidence.push(message(error)); + } + + let expectation: BrowserActExpectationEvidence | undefined; + let waitResult: BrowserWaitForResult | undefined; + let after: BrowserObservation | undefined; + let afterTargets: string[] | undefined; + try { + if (step.expect) { + waitResult = await runtime.wait(step.expect!, before, before.targetId, action.tab_id, action.timeout_ms, action.poll_ms); + expectation = expectationEvidence(waitResult); + } + after = await runtime.observe(action.tab_id); + afterTargets = await runtime.targetIds(); + } catch (error) { + evidence.push(`post-action observation failed: ${message(error)}`); + } + + timedOut ||= waitResult?.status === "timed_out"; + const stale = isStale(actionError) || waitResult?.reason === "stale_ref" || expectation?.status === "unverifiable" && expectation.details.some((line) => /stale_ref| is stale/i.test(line)); + const outcome = expectation?.status === "newly_verified" && !actionError ? "worked" : expectation?.status === "failed" || stale ? "didnt" : "unknown"; + if (expectation) evidence.push(`expectation ${expectation.status}`); + steps.push(stepResult(index, step, outcome, evidence, expectation)); + + const postBoundary = after && afterTargets ? boundary(before, after, targets, afterTargets, dialogs, runtime) : undefined; + if (after && afterTargets) { current = after; targets = afterTargets; dialogs = runtime.dialogCount(); } + stopReason = stale + ? "stale_ref" + : expectation?.status === "failed" + ? "expectation_failed" + : actionError + ? "action_failed" + : waitStopReason(waitResult) + ?? postBoundary + ?? (!after?.complete || expectation?.status === "preexisting" || expectation?.status === "unverifiable" ? "control_flow" : undefined); + if (stopReason) { stoppedAt = index; break; } + } + + let finalExpectation: BrowserActExpectationEvidence | undefined; + const terminalNavigation = stopReason === "navigation" && stoppedAt === finalStepIndex && steps.at(-1)?.outcome === "worked"; + if (action.expect && (!stopReason || terminalNavigation)) { + try { + const result = await runtime.wait(action.expect!, baseline, baseline.targetId, action.tab_id, action.timeout_ms, action.poll_ms); + finalExpectation = expectationEvidence(result); + timedOut ||= result.status === "timed_out"; + if (finalExpectation.status === "failed") { stopReason = "expectation_failed"; stoppedAt = finalStepIndex; } + else if (finalExpectation.status === "unverifiable") { stopReason = waitStopReason(result) ?? "control_flow"; stoppedAt = finalStepIndex; } + } catch (error) { + finalExpectation = { status: "unverifiable", details: [message(error)] }; + stopReason = "control_flow"; stoppedAt = finalStepIndex; + } + } + + let successor: BrowserActResult["successor"] | undefined; + let successorError: unknown; + for (let attempt = 0; attempt < 3 && !successor; attempt += 1) { + try { + const observed = await runtime.observe(action.tab_id); + const successorTargets = await runtime.targetIds(); + const lateBoundary = boundary(current, observed, targets, successorTargets, dialogs, runtime); + current = observed; targets = successorTargets; dialogs = runtime.dialogCount(); + if (lateBoundary) { + stopReason ??= lateBoundary; + successorError = new Error(`${lateBoundary} changed successor observation`); + continue; + } + if (!observed.complete) throw new Error("successor observation incomplete"); + if (action.expect && finalExpectation && finalExpectation.status !== "failed") { + const evaluation = runtime.evaluate(action.expect, observed, baseline); + finalExpectation = { + status: evaluation.truth === true ? finalExpectation.before === true ? "preexisting" : "newly_verified" : evaluation.truth === false ? "failed" : "unverifiable", + before: finalExpectation.before, after: evaluation.truth, + details: [...finalExpectation.details, ...evaluation.details.map((detail) => `successor: ${detail}`)], + }; + if (evaluation.truth !== true) { + stopReason = evaluation.truth === false ? "expectation_failed" : "control_flow"; + stoppedAt = finalStepIndex; + } else if (stopReason === "control_flow" && stoppedAt === finalStepIndex) { + stopReason = undefined; + stoppedAt = undefined; + } + } + const complete: CuaActionBrowserSnapshot = { type: "browser_snapshot", tab_id: action.tab_id, depth: Number.MAX_SAFE_INTEGER }; + const presentation = runtime.present(observed, { type: "browser_snapshot", tab_id: action.tab_id, ...action.successor }); + successor = { + status: "observed", + text: runtime.render(presentation), + url: observed.url, + title: observed.title, + diff: diffObservations(runtime.present(baseline, complete), runtime.present(observed, complete)), + }; + } catch (error) { + successorError = error; + } + } + if (!successor) successor = { status: "unavailable", error: message(successorError ?? new Error("successor observation unavailable")) }; + + const failed = steps.some((step) => step.outcome === "didnt") || finalExpectation?.status === "failed"; + const verified = action.expect + ? finalExpectation?.status === "newly_verified" + : steps.length === action.steps.length && steps.every((step) => step.outcome === "worked"); + const verifiedNavigation = stopReason === "navigation" && steps.length === action.steps.length && steps.at(-1)?.outcome === "worked"; + return { + outcome: failed ? "didnt" : !timedOut && verified && (!stopReason || verifiedNavigation) ? "worked" : "unknown", + steps, + ...(stoppedAt === undefined ? {} : { stopped_at: stoppedAt }), + ...(stopReason ? { stop_reason: stopReason } : {}), + ...(finalExpectation ? { final_expectation: finalExpectation } : {}), + successor, + }; +} + +function expectationEvidence(result: BrowserWaitForResult): BrowserActExpectationEvidence { + return { status: result.evidence, before: result.initial.truth, after: result.final.truth, details: result.details }; +} + +function boundary(before: BrowserObservation, after: BrowserObservation, oldTargets: readonly string[], newTargets: readonly string[], dialogs: number, runtime: BrowserActRuntime): BrowserActResult["stop_reason"] { + if (runtime.dialogCount() > dialogs) return "dialog"; + if (oldTargets.length !== newTargets.length || oldTargets.some((id, index) => id !== newTargets[index])) return "control_flow"; + if (before.targetId !== after.targetId || before.navigationEpoch !== after.navigationEpoch || runtime.liveNavigationEpoch(after.targetId) !== after.navigationEpoch) return "navigation"; + if (before.generations.size !== after.generations.size) return "navigation"; + for (const [frame, generation] of after.generations) { + if (runtime.liveGeneration(frame) !== generation || before.generations.get(frame) !== generation) return "navigation"; + } + return undefined; +} + +function waitStopReason(result?: BrowserWaitForResult): BrowserActResult["stop_reason"] { + if (!result?.reason) return undefined; + if (result.reason === "navigation" || result.reason === "dialog" || result.reason === "stale_ref") return result.reason; + return "control_flow"; +} + +function stepResult(index: number, step: CuaBrowserActStep, outcome: BrowserActStepResult["outcome"], evidence: string[], expectation?: BrowserActExpectationEvidence): BrowserActStepResult { + return { index, type: step.type, outcome, evidence, ...(expectation ? { expectation } : {}) }; +} + +function unavailable(error: unknown, stoppedAt: number): BrowserActResult { + return { outcome: "unknown", steps: [], stopped_at: stoppedAt, stop_reason: "control_flow", successor: { status: "unavailable", error: message(error) } }; +} + +function isStale(error: unknown): boolean { + return !!error && /(?:stale.*ref|ref.*stale)/i.test(message(error)); +} + +function message(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/agent/src/translator/browser-observation.ts b/packages/agent/src/translator/browser-observation.ts index ccbd2cbd..557d3f55 100644 --- a/packages/agent/src/translator/browser-observation.ts +++ b/packages/agent/src/translator/browser-observation.ts @@ -1,3 +1,5 @@ +import type { BrowserObservationDiff } from "./types"; + /** Minimal CDP accessibility node used by browser observations. */ export interface AXNode { nodeId: string; @@ -71,6 +73,27 @@ export interface BrowserPresentation { shape: string; } +/** Compare complete presentations while normalizing away snapshot-scoped refs. */ +export function diffObservations(before: BrowserPresentation, after: BrowserPresentation): BrowserObservationDiff { + const normalize = (line: string) => line.replace(/\u0000/g, "ref"); + const counts = new Map(); + for (const { text } of before.lines) { + const line = normalize(text); + counts.set(line, (counts.get(line) ?? 0) + 1); + } + const added: string[] = []; + for (const entry of after.lines) { + const line = normalize(entry.text); + const count = counts.get(line) ?? 0; + if (count === 0) added.push(line); + else counts.set(line, count - 1); + } + const removed = [...counts].flatMap(([line, count]) => Array.from({ length: count }, () => line)); + const url = before.observation.url === after.observation.url ? undefined : { before: before.observation.url, after: after.observation.url }; + const title = before.observation.title === after.observation.title ? undefined : { before: before.observation.title, after: after.observation.title }; + return { changed: added.length > 0 || removed.length > 0 || !!url || !!title, added, removed, ...(url ? { url } : {}), ...(title ? { title } : {}) }; +} + /** Build the lookup key for an iframe node's stitched child tree. */ export function frameStitchKey(parentFrameKey: string, backendNodeId: number): string { return `${parentFrameKey}\u0000${backendNodeId}`; diff --git a/packages/agent/src/translator/browser-wait.ts b/packages/agent/src/translator/browser-wait.ts index 9942d9a3..c8fbbdc2 100644 --- a/packages/agent/src/translator/browser-wait.ts +++ b/packages/agent/src/translator/browser-wait.ts @@ -18,6 +18,7 @@ export interface BrowserWaitRuntime { dialogCount(): number; targetExists(targetId: string): Promise; liveGeneration(frameId: string): number; + liveNavigationEpoch(targetId: string): number; resolveRef: BrowserRefResolver; now?(): number; delay?(ms: number): Promise; @@ -29,6 +30,9 @@ export interface BrowserWaitOptions { timeoutMs?: number; pollMs?: number; tabId?: string; + /** Supply the pre-action state when verifying an action postcondition. */ + baseline?: BrowserObservation; + targetId?: string; } function expectationNodes(observation: BrowserObservation): AXNode[] { @@ -96,9 +100,9 @@ export async function waitForBrowserExpectation(runtime: BrowserWaitRuntime, opt let targetId: string; let baseline: BrowserObservation; try { - targetId = await beforeDeadline(() => runtime.selectTarget(options.tabId), remaining(), now); + targetId = options.targetId ?? await beforeDeadline(() => runtime.selectTarget(options.tabId), remaining(), now); if (expired()) return timedOut(started, now()); - baseline = await runtime.observeTarget(targetId); + baseline = options.baseline ?? await runtime.observeTarget(targetId); if (expired()) return timedOut(started, now()); } catch (error) { if (error instanceof WaitDeadlineError) return timedOut(started, now()); @@ -106,13 +110,15 @@ export async function waitForBrowserExpectation(runtime: BrowserWaitRuntime, opt } const initial = evaluateBrowserExpectation(options.expect, baseline, baseline, runtime.resolveRef); if (!baseline.complete) return terminal("unverifiable", "unverifiable", initial, initial, started, now(), "incomplete_observation"); + if (!options.baseline && (runtime.liveNavigationEpoch(targetId) !== baseline.navigationEpoch || [...baseline.generations].some(([key, generation]) => runtime.liveGeneration(key) !== generation))) return terminal("interrupted", "unverifiable", initial, initial, started, now(), "navigation"); if (expired()) return timedOut(started, now()); if (initial.reason === "stale_ref") return terminal("interrupted", "unverifiable", initial, initial, started, now(), initial.reason); - if (initial.truth === true) return terminal("satisfied", "preexisting", initial, initial, started, now()); + if (initial.truth === true && !options.baseline) return terminal("satisfied", "preexisting", initial, initial, started, now()); const dialogs = runtime.dialogCount(); let final = initial; while (!expired()) { - await sleep(Math.min(poll, remaining())); + try { await beforeDeadline(() => sleep(Math.min(poll, remaining())), remaining(), now); } + catch (error) { if (error instanceof WaitDeadlineError) break; throw error; } if (expired()) break; let exists: boolean; try { exists = await beforeDeadline(() => runtime.targetExists(targetId), remaining(), now); } @@ -128,7 +134,7 @@ export async function waitForBrowserExpectation(runtime: BrowserWaitRuntime, opt if (navigated) { if (isLocationExpectation(options.expect)) { final = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef); - if (final.truth === true && !expired()) return terminal("satisfied", "newly_verified", initial, final, started, now()); + if (final.truth === true && !expired()) return terminal("satisfied", satisfiedEvidence(initial), initial, final, started, now()); } return terminal("interrupted", "unverifiable", initial, final, started, now(), "navigation"); } @@ -139,11 +145,13 @@ export async function waitForBrowserExpectation(runtime: BrowserWaitRuntime, opt } const removedFrame = [...baseline.generations.keys()].some((key) => !observation.generations.has(key)); if (removedFrame) return terminal("unverifiable", "unverifiable", initial, final, started, now(), "incomplete_observation"); + const unstable = runtime.liveNavigationEpoch(targetId) !== observation.navigationEpoch || [...observation.generations].some(([key, generation]) => generation !== runtime.liveGeneration(key)); + if (unstable) return terminal("interrupted", "unverifiable", initial, final, started, now(), "navigation"); if (expired()) break; final = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef); if (expired()) break; if (final.reason === "stale_ref") return terminal("interrupted", "unverifiable", initial, final, started, now(), final.reason); - if (final.truth === true) return terminal("satisfied", "newly_verified", initial, final, started, now()); + if (final.truth === true) return terminal("satisfied", satisfiedEvidence(initial), initial, final, started, now()); } if (final.truth === undefined) return terminal("unverifiable", "unverifiable", initial, final, started, now(), final.reason ?? "incomplete_observation"); return terminal("timed_out", "failed", initial, final, started, now()); @@ -164,6 +172,10 @@ async function beforeDeadline(operation: () => Promise, remaining: number, } } +function satisfiedEvidence(initial: BrowserExpectationEvidence): BrowserWaitForResult["evidence"] { + return initial.truth === true ? "preexisting" : "newly_verified"; +} + function isLocationExpectation(expectation: CuaBrowserExpectation): boolean { if ("all" in expectation) return expectation.all.every(isLocationExpectation); if ("any" in expectation) return expectation.any.every(isLocationExpectation); diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 457db3d2..9f1f6bc6 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -1,5 +1,6 @@ import { normalizeGotoUrl, + type CuaActionBrowserAct, type CuaActionBrowserClick, type CuaActionBrowserDrag, type CuaActionBrowserFill, @@ -12,6 +13,7 @@ import { type CuaActionBrowserSnapshot, type CuaActionBrowserWaitFor, type CuaBrowserAction, + type CuaBrowserActStep, type CuaBrowserExpectation, } from "@onkernel/cua-ai"; import { CdpConnection, type CdpEventMessage } from "./cdp"; @@ -28,8 +30,9 @@ import { type ObservationLine, type RenderContext, } from "./browser-observation"; -import { waitForBrowserExpectation, type BrowserExpectationEvaluation } from "./browser-wait"; -import type { BatchReadResult, BrowserWaitForResult } from "./types"; +import { runBrowserAct } from "./browser-act"; +import { evaluateBrowserExpectation, waitForBrowserExpectation, type BrowserExpectationEvaluation } from "./browser-wait"; +import type { BatchReadResult, BrowserActResult, BrowserWaitForResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; const DEFAULT_SNAPSHOT_DEPTH = 15; @@ -258,6 +261,8 @@ export class BrowserExecutor { switch (action.type) { case "browser_snapshot": return [{ type: "browser_text", label: "snapshot", text: await this.snapshot(action) }]; + case "browser_act": + return [{ type: "browser_act", result: await this.act(action) }]; case "browser_wait_for": return [{ type: "browser_wait_for", result: await this.waitFor(action) }]; case "browser_text": @@ -331,14 +336,50 @@ export class BrowserExecutor { } private waitFor(action: CuaActionBrowserWaitFor): Promise { + return this.waitForExpectation(action.expect, { timeoutMs: action.timeout_ms, pollMs: action.poll_ms, tabId: action.tab_id }); + } + + private waitForExpectation(expect: CuaBrowserExpectation, options: { timeoutMs?: number; pollMs?: number; tabId?: string; baseline?: BrowserObservation; targetId?: string }): Promise { return waitForBrowserExpectation({ selectTarget: (tabId) => this.resolveTarget(tabId), observeTarget: (targetId) => this.observe(targetId, false), dialogCount: () => this.dialogNotes.length, targetExists: async (targetId) => (await this.cdp.pageTargets()).some((target) => target.targetId === targetId), liveGeneration: (frameId) => this.generation(frameId), + liveNavigationEpoch: (targetId) => this.navigationEpoch(targetId), resolveRef: (expectation, observation) => this.evaluateRefExpectation(expectation, observation), - }, { expect: action.expect, timeoutMs: action.timeout_ms, pollMs: action.poll_ms, tabId: action.tab_id }); + }, { expect, ...options }); + } + + private act(action: CuaActionBrowserAct): Promise { + return runBrowserAct(action, { + observe: (tabId) => this.observe(tabId, false), + targetIds: async () => (await this.cdp.pageTargets()).map((target) => target.targetId).sort(), + dialogCount: () => this.dialogNotes.length, + liveGeneration: (frameId) => this.generation(frameId), + liveNavigationEpoch: (targetId) => this.navigationEpoch(targetId), + executeStep: (step, tabId) => this.executeActStep(step, tabId), + wait: (expect, baseline, targetId, tabId, timeoutMs, pollMs) => this.waitForExpectation(expect, { baseline, targetId, tabId, timeoutMs, pollMs }), + evaluate: (expect, observation, baseline) => evaluateBrowserExpectation(expect, observation, baseline, (ref, state) => this.evaluateRefExpectation(ref, state)), + present: (observation, snapshot) => this.presentObservation(observation, snapshot), + render: (presentation) => this.renderObservation(presentation, false), + }); + } + + private async executeActStep(step: CuaBrowserActStep, tabId?: string): Promise { + switch (step.type) { + case "click": return this.click({ type: "browser_click", ref: step.ref, button: step.button, num_clicks: step.num_clicks, modifiers: step.modifiers, tab_id: tabId }); + case "hover": return this.hover({ type: "browser_hover", ref: step.ref, tab_id: tabId }); + case "fill": return this.fill({ type: "browser_fill", ref: step.ref, value: step.value, tab_id: tabId }); + case "scroll_to": return this.scrollTo({ type: "browser_scroll_to", ref: step.ref, tab_id: tabId }); + case "key": return this.key({ type: "browser_key", text: step.text, repeat: step.repeat, tab_id: tabId }); + case "type": { + const session = await this.session(tabId); + await this.cdp.send("Input.insertText", { text: step.text }, session); + return; + } + case "wait": await new Promise((resolve) => setTimeout(resolve, step.ms ?? 0)); return; + } } private evaluateRefExpectation(expectation: Extract, observation: BrowserObservation): BrowserExpectationEvaluation { @@ -512,11 +553,11 @@ export class BrowserExecutor { }; } - private renderObservation(presentation: BrowserPresentation): string { + private renderObservation(presentation: BrowserPresentation, comparePrevious = true): string { const { observation, cacheKey, lines, shape } = presentation; const cached = this.lastSnapshots.get(observation.targetId); this.lastSnapshots.set(observation.targetId, { key: cacheKey, shape }); - if (cached?.key === cacheKey && cached.shape === shape) return UNCHANGED_SNAPSHOT; + if (comparePrevious && cached?.key === cacheKey && cached.shape === shape) return UNCHANGED_SNAPSHOT; let text = ""; for (const line of lines) { if (text.length > SNAPSHOT_CHAR_LIMIT) break; @@ -681,6 +722,7 @@ export class BrowserExecutor { const modifiers = modifierBits(action.modifiers); const button = action.button ?? "left"; const clicks = action.num_clicks ?? 1; + if (!Number.isInteger(clicks) || clicks < 1 || clicks > 3) throw new Error("num_clicks must be an integer between 1 and 3"); await this.cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: point.x, y: point.y, modifiers }, point.session); // Native multi-clicks are separate press/release cycles with an // incrementing clickCount; a single pair with the final count is not how diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 5610d31b..80125307 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -134,7 +134,10 @@ export class InternalComputerTranslator { for (const action of actions) { if (isCuaBrowserAction(action)) { await flush(); - result.readResults.push(...(await this.browser().execute(action))); + const reads = await this.browser().execute(action); + result.readResults.push(...reads); + if (action.type === "browser_act" && reads.some((read) => read.type === "browser_act" && read.result.stop_reason)) break; + if (action.type === "browser_wait_for" && reads.some((read) => read.type === "browser_wait_for" && read.result.status !== "satisfied")) break; continue; } switch (action.type) { diff --git a/packages/agent/src/translator/types.ts b/packages/agent/src/translator/types.ts index 26d3fdcb..f27ae587 100644 --- a/packages/agent/src/translator/types.ts +++ b/packages/agent/src/translator/types.ts @@ -4,6 +4,50 @@ export interface BrowserExpectationEvidence { details: string[]; } +/** Semantic result of one action or a complete dependent action plan. */ +export type BrowserActOutcome = "worked" | "didnt" | "unknown"; + +/** Whether an action expectation was newly verified, preexisting, failed, or unverifiable. */ +export type BrowserActExpectationStatus = "newly_verified" | "preexisting" | "failed" | "unverifiable"; + +/** Before/after semantic evidence for an action expectation. */ +export interface BrowserActExpectationEvidence { + status: BrowserActExpectationStatus; + before?: boolean; + after?: boolean; + details: string[]; +} + +/** Outcome and evidence for one requested browser action-plan step. */ +export interface BrowserActStepResult { + index: number; + type: string; + outcome: BrowserActOutcome; + evidence: string[]; + expectation?: BrowserActExpectationEvidence; +} + +/** Complete normalized changes between baseline and successor observations. */ +export interface BrowserObservationDiff { + changed: boolean; + added: string[]; + removed: string[]; + url?: { before: string; after: string }; + title?: { before: string; after: string }; +} + +/** Structured result returned by a dependent `browser_act` plan. */ +export interface BrowserActResult { + outcome: BrowserActOutcome; + steps: BrowserActStepResult[]; + stopped_at?: number; + stop_reason?: "action_failed" | "expectation_failed" | "navigation" | "stale_ref" | "dialog" | "control_flow"; + final_expectation?: BrowserActExpectationEvidence; + successor: + | { status: "observed"; text: string; url: string; title: string; diff: BrowserObservationDiff } + | { status: "unavailable"; error: string }; +} + /** Browser lifecycle or observation reason that terminated a semantic wait. */ export type BrowserWaitReason = | "navigation" @@ -30,7 +74,8 @@ export type BatchReadResult = | { type: "url"; url: string } | { type: "cursor_position"; x: number; y: number } | { type: "browser_text"; label: string; text: string } - | { type: "browser_wait_for"; result: BrowserWaitForResult }; + | { type: "browser_wait_for"; result: BrowserWaitForResult } + | { type: "browser_act"; result: BrowserActResult }; export interface BatchExecutionResult { readResults: BatchReadResult[]; diff --git a/packages/agent/test/browser-wait.test.ts b/packages/agent/test/browser-wait.test.ts index 0422c7da..384a359c 100644 --- a/packages/agent/test/browser-wait.test.ts +++ b/packages/agent/test/browser-wait.test.ts @@ -64,16 +64,17 @@ describe("waitForBrowserExpectation", () => { let time = 0, read = 0; const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation(states[Math.min(read++, states.length - 1)]!), - dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, + dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); expect(result).toMatchObject({ status, evidence }); }); + it("reports initial observation failures and stale refs honestly", async () => { - const failed = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => { throw new Error("boom"); }, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef }, { expect: { type: "text", text: "Ready" }, timeoutMs: 10 }); + const failed = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => { throw new Error("boom"); }, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: missingRef }, { expect: { type: "text", text: "Ready" }, timeoutMs: 10 }); expect(failed).toMatchObject({ status: "unverifiable", reason: "observation_failed" }); - const stale = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation(), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef }, { expect: { type: "ref", ref: "e1", checked: true }, timeoutMs: 10 }); + const stale = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation(), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: missingRef }, { expect: { type: "ref", ref: "e1", checked: true }, timeoutMs: 10 }); expect(stale).toMatchObject({ status: "interrupted", reason: "stale_ref" }); }); @@ -93,7 +94,7 @@ describe("waitForBrowserExpectation", () => { }, dialogCount: () => "dialog" in scenario && scenario.dialog && dialogReads++ > 0 ? 1 : 0, targetExists: async () => !("exists" in scenario) || scenario.exists, - liveGeneration: () => 0, resolveRef: missingRef, + liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); expect(result).toMatchObject({ status: reason === "observation_failed" ? "unverifiable" : "interrupted", reason }); @@ -105,8 +106,9 @@ describe("waitForBrowserExpectation", () => { baseline.generations.set("frame", 1); const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => reads++ === 0 ? baseline : observation(), - dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, - resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, + dialogCount: () => 0, targetExists: async () => true, + liveGeneration: (key) => key === "frame" && reads <= 1 ? 1 : 0, + liveNavigationEpoch: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, }, { expect: { type: "text", text: "Gone", exists: false }, timeoutMs: 20, pollMs: 10 }); expect(result).toMatchObject({ status: "unverifiable", reason: "incomplete_observation" }); }); @@ -114,7 +116,7 @@ describe("waitForBrowserExpectation", () => { it("interrupts when a ref becomes stale after the baseline", async () => { let time = 0, resolves = 0; const resolveRef: BrowserRefResolver = () => resolves++ === 0 ? { truth: false, details: ["not checked"] } : { truth: undefined, details: ["stale"], reason: "stale_ref" }; - const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation(), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "ref", ref: "e1", checked: true }, timeoutMs: 20, pollMs: 10 }); + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation(), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "ref", ref: "e1", checked: true }, timeoutMs: 20, pollMs: 10 }); expect(result).toMatchObject({ status: "interrupted", reason: "stale_ref" }); }); @@ -124,20 +126,26 @@ describe("waitForBrowserExpectation", () => { const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => states[Math.min(reads++, states.length - 1)]!, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, - resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, + liveNavigationEpoch: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, }, { expect: { type: "text", text: "Ready" }, timeoutMs: 30, pollMs: 10 }); expect(result).toMatchObject({ status: "satisfied", evidence: "newly_verified" }); }); it("returns incomplete observations as unverifiable", async () => { let time = 0; - const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation([], false), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "text", text: "Gone", exists: false }, timeoutMs: 10, pollMs: 10 }); + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation([], false), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "text", text: "Gone", exists: false }, timeoutMs: 10, pollMs: 10 }); + expect(result).toMatchObject({ status: "unverifiable", reason: "incomplete_observation" }); + }); + + it("stops on an incomplete intermediate observation", async () => { + let time = 0, reads = 0; + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => observation([], reads++ === 0), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); expect(result).toMatchObject({ status: "unverifiable", reason: "incomplete_observation" }); }); it("allows location expectations to be satisfied by navigation", async () => { let time = 0, reads = 0; - const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => reads++ === 0 ? observation() : observation([], true, { url: "https://a.test/done", navigationEpoch: 1 }), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "url", contains: "/done" }, timeoutMs: 20, pollMs: 10 }); + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => reads++ === 0 ? observation() : observation([], true, { url: "https://a.test/done", navigationEpoch: 1 }), dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => reads > 1 ? 1 : 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "url", contains: "/done" }, timeoutMs: 20, pollMs: 10 }); expect(result).toMatchObject({ status: "satisfied", evidence: "newly_verified" }); }); @@ -151,14 +159,14 @@ describe("waitForBrowserExpectation", () => { return observation([], true, { url: "https://a.test/other", navigationEpoch: 1 }); }, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, - resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, + liveNavigationEpoch: () => 1, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; }, }, { expect: { type: "url", contains: "/done" }, timeoutMs: 100, pollMs: 10 }); expect(result).toMatchObject({ status: "interrupted", reason: "navigation" }); }); it("does not accept a condition completed after the deadline", async () => { let time = 0, reads = 0; - const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => { if (reads++ > 0) time += 20; return observation(reads > 1 ? ["Ready"] : []); }, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); + const result = await waitForBrowserExpectation({ selectTarget: async () => "page", observeTarget: async () => { if (reads++ > 0) time += 20; return observation(reads > 1 ? ["Ready"] : []); }, dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: missingRef, now: () => time, delay: async (ms) => { time += ms; } }, { expect: { type: "text", text: "Ready" }, timeoutMs: 20, pollMs: 10 }); expect(result.status).toBe("timed_out"); }); @@ -167,7 +175,7 @@ describe("waitForBrowserExpectation", () => { const result = await waitForBrowserExpectation({ selectTarget: async () => { if (phase === "select") time = 10; return "page"; }, observeTarget: async () => { if (phase === "observe") time = 10; return observation(); }, - dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, + dialogCount: () => 0, targetExists: async () => true, liveGeneration: () => 0, liveNavigationEpoch: () => 0, resolveRef: () => { if (phase === "evaluate") time = 10; return { truth: true, details: [] }; }, now: () => time, }, { expect: phase === "evaluate" ? { type: "ref", ref: "e1", checked: true } : { type: "text", text: "Ready" }, timeoutMs: 10 }); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index 4f64fcc7..72914b37 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -4,9 +4,12 @@ import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; import { BrowserExecutor } from "../src/translator/browser"; import type { CdpConnection } from "../src/translator/cdp"; -import { buildCuaComputerTools } from "../src/tools"; +import { buildCuaComputerTools, formatBrowserActResult } from "../src/tools"; +import { runBrowserAct, type BrowserActRuntime } from "../src/translator/browser-act"; +import { evaluateBrowserExpectation, waitForBrowserExpectation } from "../src/translator/browser-wait"; +import type { BrowserObservation, BrowserPresentation } from "../src/translator/browser-observation"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; -import type { BatchReadResult } from "../src/translator/types"; +import type { BatchReadResult, BrowserActResult, BrowserWaitForResult } from "../src/translator/types"; const browser = { session_id: "browser_123", cdp_ws_url: "wss://example.test/cdp" } as KernelBrowser; @@ -63,6 +66,254 @@ describe("InternalComputerTranslator browser plane", () => { }); }); +describe("browser_act orchestration", () => { + const observation = (name: string, epoch = 0): BrowserObservation => ({ + targetId: "target-1", navigationEpoch: epoch, url: `https://example.test/${name}`, title: name, + generations: new Map([["target-1", epoch]]), complete: true, nodes: [], stitches: new Map(), + tree: { byId: new Map(), roots: [], ctx: {} as never }, + }); + const waitResult = (evidence: BrowserWaitForResult["evidence"]): BrowserWaitForResult => ({ + status: evidence === "failed" ? "timed_out" : evidence === "unverifiable" ? "unverifiable" : "satisfied", + evidence, + initial: { truth: evidence === "preexisting" ? true : false, details: ["before"] }, + final: { truth: evidence === "newly_verified" || evidence === "preexisting", details: ["after"] }, + elapsed_ms: 1, details: ["initial: before", "final: after"], + }); + function runtime(states: BrowserObservation[], waits: BrowserWaitForResult[] = [], options: { dispatchError?: Error; targets?: string[][]; dialogAfterDispatch?: boolean; failObservationAt?: number | number[]; wait?: BrowserActRuntime["wait"] } = {}): BrowserActRuntime & { dispatched: string[] } { + let read = 0; + let targetRead = 0; + let dialogs = 0; + const dispatched: string[] = []; + return { + dispatched, + observe: async () => { + const failures = Array.isArray(options.failObservationAt) ? options.failObservationAt : [options.failObservationAt]; + if (failures.includes(read)) { read += 1; throw new Error("observation unavailable"); } + const state = states[Math.min(read++, states.length - 1)]; + if (!state) throw new Error("observation unavailable"); + return state; + }, + targetIds: async () => options.targets?.[Math.min(targetRead++, options.targets.length - 1)] ?? ["target-1"], + dialogCount: () => dialogs, + liveGeneration: (frame) => states[Math.max(0, Math.min(read - 1, states.length - 1))]?.generations.get(frame) ?? 0, + liveNavigationEpoch: () => states[Math.max(0, Math.min(read - 1, states.length - 1))]?.navigationEpoch ?? 0, + executeStep: async (step) => { + dispatched.push(step.type); + if (options.dialogAfterDispatch) dialogs += 1; + if (options.dispatchError) throw options.dispatchError; + }, + wait: options.wait ?? (async () => waits.shift() ?? waitResult("newly_verified")), + evaluate: (expect, state, baseline) => evaluateBrowserExpectation(expect, state, baseline, () => ({ truth: undefined, details: ["ref unavailable"] })), + present: (state): BrowserPresentation => ({ observation: state, cacheKey: "", shape: state.title, lines: [{ text: `RootWebArea ${state.title} [\u0000]`, ctx: {} as never }] }), + render: (presentation) => presentation.shape, + }; + } + + it.each([ + ["newly_verified", "worked", undefined], + ["preexisting", "unknown", "control_flow"], + ["failed", "didnt", "expectation_failed"], + ] as const)("maps %s evidence to an honest %s outcome", async (evidence, outcome, stopReason) => { + const states = [observation("before"), observation("before"), observation("after"), observation("after")]; + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }, runtime(states, [waitResult(evidence)])); + expect(result).toMatchObject({ outcome, steps: [{ outcome, expectation: { status: evidence } }], ...(stopReason ? { stop_reason: stopReason } : {}) }); + }); + + it("checks a preexisting condition after input before stopping dependent steps", async () => { + let tick = 0; + const after = observation("after"); + const wait: BrowserActRuntime["wait"] = (expect, baseline, targetId) => waitForBrowserExpectation({ + selectTarget: async () => targetId, + observeTarget: async () => after, + dialogCount: () => 0, + targetExists: async () => true, + liveGeneration: () => 0, + liveNavigationEpoch: () => 0, + resolveRef: () => ({ truth: undefined, details: [] }), + now: () => tick, + delay: async (ms) => { tick += ms; }, + }, { expect, baseline, targetId, timeoutMs: 2, pollMs: 1 }); + const rt = runtime([observation("before"), observation("before"), after, after], [], { wait }); + const result = await runBrowserAct({ type: "browser_act", steps: [ + { type: "click", ref: "e1", expect: { type: "url", contains: "before" } }, + { type: "type", text: "no" }, + ] }, rt); + expect(result).toMatchObject({ outcome: "didnt", stopped_at: 0, stop_reason: "expectation_failed", steps: [{ expectation: { status: "failed", before: true, after: false } }] }); + expect(rt.dispatched).toEqual(["click"]); + }); + + it("does not dispatch dependent steps after a preexisting expectation", async () => { + const rt = runtime([observation("before"), observation("before"), observation("after"), observation("after")], [waitResult("preexisting")]); + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }, { type: "type", text: "no" }] }, rt); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "control_flow" }); + expect(rt.dispatched).toEqual(["click"]); + }); + + it("does not dispatch later steps after a failed expectation", async () => { + const rt = runtime([observation("before"), observation("before"), observation("after"), observation("after")], [waitResult("failed")]); + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Missing" } }, { type: "type", text: "no" }] }, rt); + expect(result).toMatchObject({ outcome: "didnt", stopped_at: 0, stop_reason: "expectation_failed" }); + expect(rt.dispatched).toEqual(["click"]); + }); + + it("stops on a stale ref without dispatching later steps", async () => { + const rt = runtime([observation("before"), observation("before"), observation("successor")], [], { dispatchError: new Error("ref e1 is stale") }); + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1" }, { type: "type", text: "no" }] }, rt); + expect(result).toMatchObject({ outcome: "didnt", stopped_at: 0, stop_reason: "stale_ref" }); + expect(rt.dispatched).toEqual(["click"]); + }); + + it.each([ + ["navigation", [observation("before"), observation("before"), observation("after", 1), observation("after", 1)], {}, "navigation"], + ["dialog", [observation("before"), observation("before"), observation("after"), observation("after")], { dialogAfterDispatch: true }, "dialog"], + ["target", [observation("before"), observation("before"), observation("after"), observation("after")], { targets: [["target-1"], ["target-1"], ["target-1", "target-2"]] }, "control_flow"], + ] as const)("stops the dependent list at a %s boundary", async (_name, states, options, reason) => { + const rt = runtime([...states], [], options); + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1" }, { type: "type", text: "no" }] }, rt); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: reason }); + expect(rt.dispatched).toEqual(["click"]); + }); + + it("reports a verified terminal navigation as worked", async () => { + const before = observation("before"); + const after = observation("after", 1); + const result = await runBrowserAct({ + type: "browser_act", + steps: [{ type: "click", ref: "e1", expect: { type: "url", changed: true } }], + expect: { type: "url", contains: "after" }, + }, runtime([before, before, after, after, after], [waitResult("newly_verified"), waitResult("newly_verified")])); + expect(result).toMatchObject({ outcome: "worked", stopped_at: 0, stop_reason: "navigation", final_expectation: { status: "newly_verified" } }); + }); + + it("recollects a successor after a raced target boundary", async () => { + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }, runtime([ + observation("before"), observation("before"), observation("after"), observation("stale"), observation("stable"), + ], [waitResult("newly_verified")], { targets: [["target-1"], ["target-1"], ["target-1"], ["target-1", "target-2"], ["target-1", "target-2"]] })); + expect(result).toMatchObject({ stop_reason: "control_flow", successor: { status: "observed", title: "stable", text: "stable" } }); + }); + + it("preserves a timed-out final expectation when the successor satisfies it", async () => { + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "wait" }], expect: { type: "url", contains: "after" } }, runtime([ + observation("before"), observation("before"), observation("after"), observation("after"), + ], [waitResult("failed")])); + expect(result).toMatchObject({ outcome: "didnt", stop_reason: "expectation_failed", final_expectation: { status: "failed", after: false }, successor: { status: "observed", title: "after" } }); + }); + + it("does not report incomplete successor observations as authoritative", async () => { + const incomplete = { ...observation("after"), complete: false }; + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }, runtime([ + observation("before"), observation("before"), observation("after"), incomplete, incomplete, incomplete, + ], [waitResult("newly_verified")])); + expect(result.successor).toMatchObject({ status: "unavailable", error: "successor observation incomplete" }); + }); + + it("revalidates the final expectation against the returned successor", async () => { + const result = await runBrowserAct({ + type: "browser_act", + steps: [{ type: "click", ref: "e1", expect: { type: "url", contains: "after" } }], + expect: { type: "url", contains: "after" }, + }, runtime([ + observation("before"), observation("before"), observation("after"), observation("before"), + ], [waitResult("newly_verified"), waitResult("newly_verified")])); + expect(result).toMatchObject({ outcome: "didnt", stop_reason: "expectation_failed", final_expectation: { status: "failed", after: false }, successor: { status: "observed", title: "before" } }); + }); + + + it("rejects incomplete baselines without dispatch or authoritative diffs", async () => { + const rt = runtime([{ ...observation("before"), complete: false }]); + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1" }] }, rt); + expect(rt.dispatched).toEqual([]); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, successor: { status: "unavailable", error: "baseline observation incomplete" } }); + }); + + it("stops when an intermediate observation is incomplete", async () => { + const incomplete = { ...observation("after"), complete: false }; + const rt = runtime([observation("before"), observation("before"), incomplete, observation("stable")]); + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1" }, { type: "type", text: "no" }] }, rt); + expect(rt.dispatched).toEqual(["click"]); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "control_flow" }); + }); + + it("stops before dispatch when the observed navigation epoch is no longer live", async () => { + const rt = runtime([observation("before"), observation("before")]); + rt.liveNavigationEpoch = () => 1; + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1" }] }, rt); + expect(rt.dispatched).toEqual([]); + expect(result).toMatchObject({ stopped_at: 0, stop_reason: "navigation" }); + }); + + it("treats removed frames as a navigation boundary", async () => { + const framed = { ...observation("before"), generations: new Map([["target-1", 0], ["frame-1", 0]]) }; + const rt = runtime([framed, framed, observation("after"), observation("after")]); + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1" }, { type: "type", text: "no" }] }, rt); + expect(rt.dispatched).toEqual(["click"]); + expect(result).toMatchObject({ stopped_at: 0, stop_reason: "navigation" }); + }); + + it("returns a complete normalized successor diff", async () => { + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "wait", expect: { type: "text", text: "Done" } }] }, runtime([ + observation("before"), observation("before"), observation("after"), observation("after"), + ], [waitResult("newly_verified")])); + expect(result.successor).toMatchObject({ status: "observed", diff: { changed: true, added: ["RootWebArea after [ref]"], removed: ["RootWebArea before [ref]"] } }); + expect(JSON.stringify(result.successor)).not.toMatch(/\be\d+\b/); + }); + + it("preserves a definitive outcome when the successor is unavailable", async () => { + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Missing" } }] }, runtime([ + observation("before"), observation("before"), observation("after"), + ], [waitResult("failed")], { failObservationAt: [3, 4, 5] })); + expect(result).toMatchObject({ outcome: "didnt", stop_reason: "expectation_failed", successor: { status: "unavailable" } }); + }); + + it("preserves a verified outcome when successor collection fails", async () => { + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }, runtime([ + observation("before"), observation("before"), observation("after"), + ], [waitResult("newly_verified")], { failObservationAt: [3, 4, 5] })); + expect(result).toMatchObject({ outcome: "worked", successor: { status: "unavailable" } }); + expect(result).not.toHaveProperty("stop_reason"); + }); + + it("reports semantic evidence after uncertain action delivery", async () => { + const result = await runBrowserAct({ type: "browser_act", steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }, runtime([ + observation("before"), observation("before"), observation("after"), observation("after"), + ], [waitResult("newly_verified")], { dispatchError: new Error("input acknowledgement lost") })); + expect(result).toMatchObject({ outcome: "unknown", stop_reason: "action_failed", steps: [{ outcome: "unknown", expectation: { status: "newly_verified" } }], successor: { status: "observed" } }); + }); + + it("bounds formatted diff output while preserving the structured diff", () => { + const added = Array.from({ length: 250 }, (_, index) => `line ${index}`); + const result = { outcome: "unknown", steps: [], successor: { status: "observed", text: "successor", url: "u", title: "t", diff: { changed: true, added, removed: [] } } } satisfies BrowserActResult; + const formatted = formatBrowserActResult(result); + expect(formatted).toContain("50 more diff lines omitted"); + expect(formatted).not.toContain("line 249"); + expect(result.successor.diff.added).toHaveLength(250); + }); + + it("stops a mixed batch after a failed semantic wait", async () => { + const { client } = createClient(); + const executed: string[] = []; + const executor = { execute: async (action: CuaBrowserAction) => { + executed.push(action.type); + return action.type === "browser_wait_for" ? [{ type: "browser_wait_for", result: { status: "timed_out", evidence: "failed", initial: { truth: false, details: [] }, final: { truth: false, details: [] }, elapsed_ms: 20, details: [] } } as BatchReadResult] : []; + } } as unknown as BrowserExecutor; + const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => executor }); + await translator.executeBatch([{ type: "browser_wait_for", expect: { type: "text", text: "Ready" } }, { type: "browser_text" }]); + expect(executed).toEqual(["browser_wait_for"]); + }); + + it("stops a mixed batch after a plan stop reason", async () => { + const { client } = createClient(); + const executed: string[] = []; + const executor = { execute: async (action: CuaBrowserAction) => { + executed.push(action.type); + return action.type === "browser_act" ? [{ type: "browser_act", result: { outcome: "unknown", steps: [], stopped_at: 0, stop_reason: "navigation", successor: { status: "unavailable", error: "changed" } } } as BatchReadResult] : []; + } } as unknown as BrowserExecutor; + const translator = new InternalComputerTranslator({ browser, client, createBrowserExecutor: () => executor }); + await translator.executeBatch([{ type: "browser_act", steps: [{ type: "wait" }] }, { type: "browser_text" }]); + expect(executed).toEqual(["browser_act"]); + }); +}); + describe("InternalComputerTranslator computer additions", () => { it("crops the OS screenshot for zoom, staying in the screenshot frame", async () => { const { client } = createClient(); @@ -115,6 +366,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { let axRead = 0; let targetRead = 0; let onAxRead: ((read: number, params: Record, sessionId?: string) => void) | undefined; + let onSend: ((method: string, params: Record, sessionId?: string) => void) | undefined; let targetProvider: ((read: number) => Array<{ targetId: string; type: string; title: string; url: string }>) | undefined; const emit = (event: FakeCdpEvent) => { for (const listener of listeners) listener(event); @@ -134,6 +386,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { }, send: async (method: string, params: Record = {}, sessionId?: string) => { sent.push({ method, params, sessionId }); + onSend?.(method, params, sessionId); if (failMethods.has(method)) throw new Error(`${method} rejected`); switch (method) { case "Accessibility.getFullAXTree": { @@ -209,6 +462,9 @@ function createFakeCdp(initialNodes: unknown[] = []) { const setAxReadHook = (hook: typeof onAxRead) => { onAxRead = hook; }; + const setSendHook = (hook: typeof onSend) => { + onSend = hook; + }; const setTargetProvider = (provider: typeof targetProvider) => { targetProvider = provider; }; @@ -224,6 +480,7 @@ function createFakeCdp(initialNodes: unknown[] = []) { setTargetSession, failOn, setAxReadHook, + setSendHook, setTargetProvider, cdp: fake as unknown as CdpConnection, }; @@ -733,6 +990,98 @@ describe("BrowserExecutor semantic waits", () => { }); }); +describe("BrowserExecutor action plans", () => { + async function act(executor: BrowserExecutor, action: Record) { + const [read] = await executor.execute({ type: "browser_act", timeout_ms: 100, poll_ms: 10, ...action } as CuaBrowserAction); + if (read?.type !== "browser_act") throw new Error("expected browser_act result"); + return read.result; + } + + it("waits for a delayed postcondition and returns a consistent successor diff", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const done = [ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), ax({ nodeId: "2", role: "button", name: "Done", backendDOMNodeId: 43, parentId: "1" })]; + fake.setSendHook((method, params) => { + if (method === "Input.dispatchMouseEvent" && params.type === "mousePressed") setTimeout(() => fake.setNodes(done), 15); + }); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + const result = await act(executor, { steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }); + expect(result).toMatchObject({ outcome: "worked", steps: [{ expectation: { status: "newly_verified" } }], successor: { status: "observed", text: expect.stringContaining('button "Done"'), diff: { added: [expect.stringContaining('button "Done"')], removed: [expect.stringContaining('button "Save"')] } } }); + }); + + it.each([ + ["same-document", "Page.navigatedWithinDocument", { frameId: "TARGET-1", url: "https://a.test/#next" }], + ["cross-document", "Page.frameNavigated", { frame: { id: "TARGET-1" } }], + ] as const)("stops at a selected-target %s navigation", async (_label, method, params) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setSendHook((sent, input, sessionId) => { + if (sent === "Input.dispatchMouseEvent" && input.type === "mousePressed") fake.emit({ method, params, sessionId }); + }); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + const result = await act(executor, { steps: [{ type: "click", ref: "e1" }, { type: "type", text: "no" }] }); + expect(result).toMatchObject({ outcome: "unknown", stopped_at: 0, stop_reason: "navigation" }); + expect(fake.sent.some((command) => command.method === "Input.insertText")).toBe(false); + }); + + it("stops at dialog and popup boundaries", async () => { + const dialog = createFakeCdp(BUTTON_TREE); + dialog.setSendHook((method, params, sessionId) => { + if (method === "Input.dispatchMouseEvent" && params.type === "mousePressed") dialog.emit({ method: "Page.javascriptDialogOpening", params: { type: "alert", message: "done" }, sessionId }); + }); + const dialogExecutor = new BrowserExecutor(dialog.cdp); + await snapshotText(dialogExecutor); + expect(await act(dialogExecutor, { steps: [{ type: "click", ref: "e1" }, { type: "type", text: "no" }] })).toMatchObject({ stopped_at: 0, stop_reason: "dialog" }); + + const popup = createFakeCdp(BUTTON_TREE); + let opened = false; + popup.setTargetProvider(() => [ + { targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }, + ...(opened ? [{ targetId: "TARGET-2", type: "page", title: "Popup", url: "https://b.test/" }] : []), + ]); + popup.setSendHook((method, params) => { if (method === "Input.dispatchMouseEvent" && params.type === "mousePressed") opened = true; }); + const popupExecutor = new BrowserExecutor(popup.cdp); + await snapshotText(popupExecutor); + expect(await act(popupExecutor, { steps: [{ type: "click", ref: "e1" }, { type: "type", text: "no" }] })).toMatchObject({ stopped_at: 0, stop_reason: "control_flow" }); + }); + + it("isolates navigation epochs to the selected target", async () => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setTargetSession("TARGET-2", "session-2"); + fake.setTargetProvider(() => [ + { targetId: "TARGET-1", type: "page", title: "One", url: "https://a.test/" }, + { targetId: "TARGET-2", type: "page", title: "Two", url: "https://b.test/" }, + ]); + fake.setSendHook((method, params) => { + if (method === "Input.dispatchMouseEvent" && params.type === "mousePressed") fake.emit({ method: "Page.navigatedWithinDocument", params: { frameId: "TARGET-2", url: "https://b.test/#next" }, sessionId: "session-2" }); + }); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor, { tab_id: "TARGET-1" }); + const result = await act(executor, { tab_id: "TARGET-1", steps: [{ type: "click", ref: "e1" }, { type: "type", text: "yes" }] }); + expect(result).toMatchObject({ outcome: "unknown", steps: [{}, {}] }); + expect(result).not.toHaveProperty("stop_reason"); + expect(fake.sent.some((command) => command.method === "Input.insertText")).toBe(true); + }); + + it("does not verify absence when a nested frame is incomplete", async () => { + const tree = [ax({ nodeId: "1", role: "RootWebArea", childIds: ["2"] }), ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" })]; + const result = await act(new BrowserExecutor(createFakeCdp(tree).cdp), { timeout_ms: 10, steps: [{ type: "wait", expect: { type: "text", text: "Missing", exists: false } }] }); + expect(result).toMatchObject({ outcome: "unknown", stop_reason: "control_flow", steps: [], successor: { status: "unavailable", error: "baseline observation incomplete" } }); + }); + + it("reports verified state but unknown delivery when input acknowledgement is lost", async () => { + const fake = createFakeCdp(BUTTON_TREE); + const done = [ax({ nodeId: "1", role: "RootWebArea", childIds: ["2"] }), ax({ nodeId: "2", role: "button", name: "Done", backendDOMNodeId: 43, parentId: "1" })]; + fake.setSendHook((method, params) => { + if (method === "Input.dispatchMouseEvent" && params.type === "mouseMoved") { fake.setNodes(done); throw new Error("input acknowledgement lost"); } + }); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + const result = await act(executor, { steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }); + expect(result).toMatchObject({ outcome: "unknown", stop_reason: "action_failed", steps: [{ outcome: "unknown", expectation: { status: "newly_verified" } }] }); + }); +}); + describe("BrowserExecutor iframe stitching", () => { it("stitches a same-process iframe subtree indented under its iframe node", async () => { const tree = [ @@ -823,6 +1172,36 @@ describe("BrowserExecutor iframe stitching", () => { expect(secondCdp.sent.find((cmd) => cmd.method === "DOM.scrollIntoViewIfNeeded" && cmd.params.backendNodeId === 90)?.sessionId).toBe("session-oop"); }); + it("rebinds and invalidates nested OOPIF refs through action plans", async () => { + const nestedSetup = () => { + const fake = createFakeCdp(OOPIF_PAGE); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.setIframeFrame(80, "FRAME-INNER"); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setSessionTree("session-oop", [ax({ nodeId: "o1", role: "RootWebArea", childIds: ["o2"] }), ax({ nodeId: "o2", role: "Iframe", backendDOMNodeId: 80, parentId: "o1" }), ax({ nodeId: "hidden", role: "button", name: "Nested", backendDOMNodeId: 90, parentId: "unrendered" })]); + fake.setFrameTree("FRAME-INNER", [ax({ nodeId: "i1", role: "RootWebArea", childIds: ["i2"] }), ax({ nodeId: "i2", role: "button", name: "Nested", backendDOMNodeId: 90, parentId: "i1" })]); + return fake; + }; + const source = new BrowserExecutor(nestedSetup().cdp); + await snapshotText(source); + const state = source.exportRefState(); + + const reboundCdp = nestedSetup(); + const rebound = new BrowserExecutor(reboundCdp.cdp); + rebound.importRefState(state); + const [worked] = await rebound.execute({ type: "browser_act", steps: [{ type: "click", ref: "e4" }] } as CuaBrowserAction); + expect(worked).toMatchObject({ type: "browser_act", result: { steps: [{ evidence: ["action dispatched"] }] } }); + expect(reboundCdp.sent.find((command) => command.method === "DOM.scrollIntoViewIfNeeded" && command.params.backendNodeId === 90)?.sessionId).toBe("session-oop"); + + const staleCdp = nestedSetup(); + const stale = new BrowserExecutor(staleCdp.cdp); + stale.importRefState(state); + await snapshotText(stale); + staleCdp.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-INNER", parentId: "FRAME-OOP" } }, sessionId: "session-oop" }); + const [failed] = await stale.execute({ type: "browser_act", steps: [{ type: "click", ref: "e4" }] } as CuaBrowserAction); + expect(failed).toMatchObject({ type: "browser_act", result: { outcome: "didnt", stop_reason: "stale_ref" } }); + }); + it("invalidates a frame target's refs when a subframe inside it navigates", async () => { const { cdp, emit } = setupOopif(); const executor = new BrowserExecutor(cdp); @@ -1043,6 +1422,14 @@ describe("navigation tool grounding frame", () => { }); describe("BrowserExecutor multi-click", () => { + it("rejects invalid click counts before dispatch", async () => { + const { cdp, sent } = createFakeCdp(BUTTON_TREE); + const executor = new BrowserExecutor(cdp); + await snapshotText(executor); + for (const num_clicks of [0, 1.5, 4]) await expect(executor.execute({ type: "browser_click", ref: "e1", num_clicks } as CuaBrowserAction)).rejects.toThrow(/integer between 1 and 3/); + expect(sent.some((cmd) => cmd.method === "Input.dispatchMouseEvent")).toBe(false); + }); + it("dispatches one press/release cycle per click with incrementing clickCount", async () => { const { cdp, sent } = createFakeCdp(BUTTON_TREE); const executor = new BrowserExecutor(cdp); diff --git a/packages/ai/src/actions/browser.ts b/packages/ai/src/actions/browser.ts index 8076b6fb..576ee858 100644 --- a/packages/ai/src/actions/browser.ts +++ b/packages/ai/src/actions/browser.ts @@ -17,6 +17,7 @@ import { Type, type TSchema } from "@earendil-works/pi-ai"; */ export const CUA_BROWSER_ACTION_TYPES = [ "browser_snapshot", + "browser_act", "browser_wait_for", "browser_text", "browser_find", @@ -78,6 +79,28 @@ export interface CuaActionBrowserWaitFor { tab_id?: string; } +/** Ref- or focus-based operation with an optional per-step semantic expectation. */ +export type CuaBrowserActStep = + | { type: "click"; ref: string; button?: "left" | "right" | "middle"; num_clicks?: 1 | 2 | 3; modifiers?: string[]; expect?: CuaBrowserExpectation } + | { type: "hover"; ref: string; expect?: CuaBrowserExpectation } + | { type: "fill"; ref: string; value: string | number | boolean; expect?: CuaBrowserExpectation } + | { type: "type"; text: string; expect?: CuaBrowserExpectation } + | { type: "key"; text: string; repeat?: number; expect?: CuaBrowserExpectation } + | { type: "scroll_to"; ref: string; expect?: CuaBrowserExpectation } + | { type: "wait"; ms?: number; expect?: CuaBrowserExpectation }; + +/** Dependent action plan whose optional `expect` verifies the complete plan result. */ +export interface CuaActionBrowserAct { + type: "browser_act"; + steps: NonEmptyArray; + expect?: CuaBrowserExpectation; + /** Timeout applied independently to each step or plan expectation. */ + timeout_ms?: number; + poll_ms?: number; + successor?: { filter?: "all" | "interactive"; depth?: number }; + tab_id?: string; +} + export interface CuaActionBrowserText { type: "browser_text"; tab_id?: string; @@ -95,7 +118,7 @@ export interface CuaActionBrowserClick { x?: number; y?: number; button?: "left" | "right" | "middle"; - num_clicks?: number; + num_clicks?: 1 | 2 | 3; modifiers?: string[]; tab_id?: string; } @@ -180,6 +203,7 @@ export interface CuaActionBrowserEvaluate { export type CuaBrowserAction = | CuaActionBrowserSnapshot + | CuaActionBrowserAct | CuaActionBrowserWaitFor | CuaActionBrowserText | CuaActionBrowserFind @@ -220,6 +244,16 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti const leaves = [Type.Object({ type: Type.Literal("text"), text: Type.String(), exists: Type.Optional(Type.Boolean()) }, { additionalProperties: false }), roleName("role"), roleName("name"), refState("value"), refState("checked"), refState("selected"), refState("expanded"), location("equals"), location("contains"), location("changed")]; const leaf = Type.Union(leaves); const expectation = Type.Union([leaf, Type.Object({ all: Type.Array(leaf, { minItems: 1 }) }, { additionalProperties: false }), Type.Object({ any: Type.Array(leaf, { minItems: 1 }) }, { additionalProperties: false })]); + const stepExpectation = { expect: Type.Optional(expectation) }; + const actStep = Type.Union([ + Type.Object({ type: Type.Literal("click"), ref: RefProperty(), button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")])), num_clicks: Type.Optional(Type.Integer({ minimum: 1, maximum: 3 })), modifiers: Type.Optional(Type.Array(Type.String())), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("hover"), ref: RefProperty(), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("fill"), ref: RefProperty(), value: Type.Union([Type.String(), Type.Number(), Type.Boolean()]), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("type"), text: Type.String(), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("key"), text: Type.String(), repeat: Type.Optional(Type.Number()), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("scroll_to"), ref: RefProperty(), ...stepExpectation }, { additionalProperties: false }), + Type.Object({ type: Type.Literal("wait"), ms: Type.Optional(Type.Number({ minimum: 0, maximum: 30_000 })), ...stepExpectation }, { additionalProperties: false }), + ]); const clickTarget: Record = options.coordinates ? { @@ -244,6 +278,18 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti { type: Type.Literal("browser_wait_for"), expect: expectation, timeout_ms: Type.Optional(Type.Number({ minimum: 0, maximum: 30_000, description: "Semantic polling timeout; in-flight browser reads settle before timeout is reported." })), poll_ms: Type.Optional(Type.Number({ minimum: 10, maximum: 1_000 })), tab_id: TabId() }, { additionalProperties: false }, ), + browser_act: Type.Object( + { + type: Type.Literal("browser_act"), + steps: Type.Array(actStep, { minItems: 1, maxItems: 20 }), + expect: Type.Optional(expectation), + timeout_ms: Type.Optional(Type.Number({ minimum: 0, maximum: 30_000, description: "Timeout applied independently to each semantic expectation." })), + poll_ms: Type.Optional(Type.Number({ minimum: 10, maximum: 1_000 })), + successor: Type.Optional(Type.Object({ filter: Type.Optional(Type.Union([Type.Literal("all"), Type.Literal("interactive")])), depth: Type.Optional(Type.Number()) }, { additionalProperties: false })), + tab_id: TabId(), + }, + { additionalProperties: false }, + ), browser_text: Type.Object( { type: Type.Literal("browser_text"), @@ -264,7 +310,7 @@ export function createCuaBrowserActionSchemaByType(options: CuaBrowserSchemaOpti type: Type.Literal("browser_click"), ...clickTarget, button: Type.Optional(Type.Union([Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")])), - num_clicks: Type.Optional(Type.Number()), + num_clicks: Type.Optional(Type.Integer({ minimum: 1, maximum: 3 })), modifiers: Type.Optional(Type.Array(Type.String())), tab_id: TabId(), }, diff --git a/packages/ai/src/modes.ts b/packages/ai/src/modes.ts index a7922ddb..e14ea051 100644 --- a/packages/ai/src/modes.ts +++ b/packages/ai/src/modes.ts @@ -56,6 +56,7 @@ export const CUA_HYBRID_COMPUTER_ACTION_TYPES: readonly CuaComputerActionType[] */ export const CUA_HYBRID_BROWSER_ACTION_TYPES: readonly CuaBrowserActionType[] = [ "browser_snapshot", + "browser_act", "browser_wait_for", "browser_text", "browser_find", @@ -110,6 +111,7 @@ export function cuaToolNameForAction(action: CuaActionType, mode: CuaMode): stri } const BROWSER_ACTION_DESCRIPTIONS: Record = { + browser_act: "Run a dependent, ref/focus-only action list and return semantic outcomes with one successor diff.", browser_wait_for: "Wait for a semantic page condition without delivering input.", browser_snapshot: "Return an accessibility-tree snapshot of the page, including iframe content, with element references like [e12]. " + diff --git a/packages/ai/test/modes.test.ts b/packages/ai/test/modes.test.ts index 621c4c0b..65198293 100644 --- a/packages/ai/test/modes.test.ts +++ b/packages/ai/test/modes.test.ts @@ -65,11 +65,13 @@ describe("mode tool naming", () => { }); describe("mode tool schemas", () => { - it("browser mode click accepts refs or viewport coordinates", () => { + it("browser mode click accepts refs or viewport coordinates and bounds click count", () => { const tools = computerTools({ mode: "browser" }); const click = tools.find((tool) => tool.name === "click")!; expect(click.parameters.properties.ref).toBeDefined(); expect(click.parameters.properties.x).toBeDefined(); + expect(click.parameters.properties.num_clicks).toMatchObject({ type: "integer", minimum: 1, maximum: 3 }); + for (const num_clicks of [0, 4]) expect(() => validateToolArguments(click, { type: "toolCall", id: "1", name: "click", arguments: { ref: "e1", num_clicks } })).toThrow(); }); it("hybrid mode browser_click is ref-only, keeping one coordinate frame", () => { @@ -89,6 +91,27 @@ describe("mode tool schemas", () => { expect(names).toContain("wait_for"); }); + it("exposes dependent action plans with bounded steps", () => { + const browserAct = computerTools({ mode: "browser" }).find((tool) => tool.name === "act")!; + const hybridAct = computerTools({ mode: "hybrid" }).find((tool) => tool.name === "browser_act")!; + expect(browserAct.parameters.properties.steps).toMatchObject({ minItems: 1, maxItems: 20 }); + expect(browserAct.parameters.properties.timeout_ms).toMatchObject({ minimum: 0, maximum: 30_000 }); + expect(browserAct.parameters.properties.poll_ms).toMatchObject({ minimum: 10, maximum: 1_000 }); + const clickStep = (browserAct.parameters.properties.steps.items as { anyOf: Array<{ properties: Record }> }).anyOf[0]!; + expect(clickStep.properties.num_clicks).toMatchObject({ type: "integer", minimum: 1, maximum: 3 }); + expect(hybridAct).toBeDefined(); + for (const arguments_ of [ + { steps: [] }, + { steps: [{ type: "wait", ms: 30_001 }] }, + { steps: [{ type: "click", ref: "e1", num_clicks: 0 }] }, + { steps: [{ type: "click", ref: "e1", num_clicks: 4 }] }, + ]) expect(() => validateToolArguments(browserAct, { type: "toolCall", id: "1", name: "act", arguments: arguments_ })).toThrow(); + for (const arguments_ of [ + { steps: [{ type: "click", ref: "e1", expect: { type: "text", text: "Done" } }] }, + { steps: [{ type: "wait" }], expect: { all: [{ type: "url", changed: true }] } }, + ]) expect(() => validateToolArguments(browserAct, { type: "toolCall", id: "1", name: "act", arguments: arguments_ })).not.toThrow(); + }); + it("registers semantic waits in hybrid mode with bounded polling", () => { const wait = computerTools({ mode: "hybrid" }).find((tool) => tool.name === "browser_wait_for")!; expect(wait.parameters.properties.timeout_ms).toMatchObject({ minimum: 0, maximum: 30_000 });