Add verified browser action plans

packages/agent/src/tools.ts

@@ -16,7 +16,9 @@ import {
 } from "@onkernel/cua-ai";
 import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator";
 import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core";
-import type { BrowserWaitForResult } from "./translator/types";
+import type { BrowserActResult, BrowserWaitForResult } from "./translator/types";
+
+const BROWSER_ACT_DIFF_LINE_LIMIT = 200;
 
 export interface ComputerToolOptions {
 	browser: KernelBrowser;
@@ -39,6 +41,7 @@ export interface BatchDetails {
 		| { type: "cursor_position"; x: number; y: number }
 		| { type: "browser_text"; label: string; bytes: number }
 		| { type: "browser_wait_for"; result: BrowserWaitForResult }
+		| { type: "browser_act"; result: BrowserActResult }
 	>;
 }
 
@@ -175,6 +178,9 @@ async function executeBatchTool(
 			} else if (read.type === "browser_wait_for") {
 				readResults.push(read);
 				content.push({ type: "text", text: formatBrowserWaitResult(read.result) });
+			} else if (read.type === "browser_act") {
+				readResults.push(read);
+				content.push({ type: "text", text: formatBrowserActResult(read.result) });
 			} else {
 				readResults.push({ type: "screenshot", bytes: read.data.length });
 				content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType });
@@ -190,13 +196,18 @@ async function executeBatchTool(
 	} catch (err) {
 		throw new Error(`Actions failed: ${errorMessage(err)}`, { cause: err });
 	}
+	const acts = readResults.flatMap((read) => read.type === "browser_act" ? [read.result] : []);
 	const waits = readResults.flatMap((read) => read.type === "browser_wait_for" ? [read.result] : []);
 	const failedWait = ["interrupted", "timed_out", "unverifiable"].find((status) => waits.some((wait) => wait.status === status));
-	const statusText = waits.length === 0
-		? "Actions executed successfully."
+	const statusText = acts.some((act) => act.outcome === "didnt")
+		? "Browser action plan did not satisfy its expectations."
 		: failedWait
 			? `Browser condition ${failedWait}.`
-			: "Browser condition satisfied.";
+			: acts.some((act) => act.outcome === "unknown")
+				? "Browser action plan outcome is unknown."
+				: acts.length > 0
+					? "Browser action plan worked."
+					: waits.length > 0 ? "Browser condition satisfied." : "Actions executed successfully.";
 	return { content, details: { statusText, readResults } };
 }
 
@@ -274,6 +285,34 @@ function formatBrowserWaitResult(result: BrowserWaitForResult): string {
 	return [`wait_for: ${result.status}/${result.evidence}${reason} after ${result.elapsed_ms}ms`, ...result.details].join("\n");
 }
 
+/** Format a browser action result for the model while bounding displayed diff lines. */
+export function formatBrowserActResult(result: BrowserActResult): string {
+	const lines = [`browser_act: ${result.outcome}`];
+	if (result.stopped_at !== undefined) lines.push(`stopped_at: ${result.stopped_at} (${result.stop_reason ?? "unknown"})`);
+	for (const step of result.steps) {
+		lines.push(`step ${step.index} ${step.type}: ${step.outcome} — ${step.evidence.join("; ")}`);
+		for (const detail of step.expectation?.details ?? []) lines.push(`  ${detail}`);
+	}
+	if (result.final_expectation) {
+		lines.push(`final expectation: ${result.final_expectation.status}`);
+		for (const detail of result.final_expectation.details) lines.push(`  ${detail}`);
+	}
+	if (result.successor.status === "unavailable") {
+		lines.push(`successor unavailable: ${result.successor.error}`);
+		return lines.join("\n");
+	}
+	const { diff } = result.successor;
+	lines.push(`successor: ${result.successor.title} (${result.successor.url})`);
+	lines.push(`diff: ${diff.changed ? `+${diff.added.length} -${diff.removed.length}` : "unchanged"}`);
+	if (diff.url) lines.push(`  url: ${diff.url.before} -> ${diff.url.after}`);
+	if (diff.title) lines.push(`  title: ${diff.title.before} -> ${diff.title.after}`);
+	const changes = [...diff.added.map((line) => `  + ${line}`), ...diff.removed.map((line) => `  - ${line}`)];
+	lines.push(...changes.slice(0, BROWSER_ACT_DIFF_LINE_LIMIT));
+	if (changes.length > BROWSER_ACT_DIFF_LINE_LIMIT) lines.push(`  … ${changes.length - BROWSER_ACT_DIFF_LINE_LIMIT} more diff lines omitted`);
+	lines.push(result.successor.text);
+	return lines.join("\n");
+}
+
 function formatPlaywrightResult(result: unknown): string {
 	return typeof result === "string" ? result : JSON.stringify(result);
 }

packages/agent/src/translator/browser-act.ts

@@ -0,0 +1,212 @@
+import type { CuaActionBrowserAct, CuaActionBrowserSnapshot, CuaBrowserActStep, CuaBrowserExpectation } from "@onkernel/cua-ai";
+import { diffObservations, type BrowserObservation, type BrowserPresentation } from "./browser-observation";
+import type { BrowserExpectationEvaluation } from "./browser-wait";
+import type { BrowserActExpectationEvidence, BrowserActResult, BrowserActStepResult, BrowserWaitForResult } from "./types";
+
+/** Browser capabilities injected into the dependent action-plan orchestrator. */
+export interface BrowserActRuntime {
+	observe(tabId?: string): Promise<BrowserObservation>;
+	targetIds(): Promise<string[]>;
+	dialogCount(): number;
+	liveGeneration(frameId: string): number;
+	liveNavigationEpoch(targetId: string): number;
+	executeStep(step: CuaBrowserActStep, tabId?: string): Promise<void>;
+	wait(expect: CuaBrowserExpectation, baseline: BrowserObservation, targetId: string, tabId?: string, timeoutMs?: number, pollMs?: number): Promise<BrowserWaitForResult>;
+	evaluate(expect: CuaBrowserExpectation, observation: BrowserObservation, baseline: BrowserObservation): BrowserExpectationEvaluation;
+	present(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation;
+	render(presentation: BrowserPresentation): string;
+}
+
+/** Execute a dependent action plan and derive outcomes, stop reasons, and successor state. */
+export async function runBrowserAct(action: CuaActionBrowserAct, runtime: BrowserActRuntime): Promise<BrowserActResult> {
+	const finalStepIndex = action.steps.length - 1;
+	let baseline: BrowserObservation;
+	let current: BrowserObservation;
+	let targets: string[];
+	try {
+		baseline = current = await runtime.observe(action.tab_id);
+		if (!baseline.complete) throw new Error("baseline observation incomplete");
+		targets = await runtime.targetIds();
+	} catch (error) {
+		return unavailable(error, 0);
+	}
+	let dialogs = runtime.dialogCount();
+	const steps: BrowserActStepResult[] = [];
+	let stoppedAt: number | undefined;
+	let stopReason: BrowserActResult["stop_reason"];
+	let timedOut = false;
+
+	for (let index = 0; index < action.steps.length; index += 1) {
+		const step = action.steps[index]!;
+		let before: BrowserObservation;
+		let nextTargets: string[];
+		try {
+			before = await runtime.observe(action.tab_id);
+			nextTargets = await runtime.targetIds();
+		} catch (error) {
+			steps.push(stepResult(index, step, "unknown", [`pre-action observation failed: ${message(error)}`]));
+			stoppedAt = index; stopReason = "control_flow"; break;
+		}
+		const preBoundary = boundary(current, before, targets, nextTargets, dialogs, runtime);
+		current = before; targets = nextTargets; dialogs = runtime.dialogCount();
+		if (!before.complete || preBoundary) {
+			const reason = before.complete ? `${preBoundary} detected before action` : "pre-action observation incomplete";
+			steps.push(stepResult(index, step, "unknown", [reason]));
+			stoppedAt = index; stopReason = preBoundary ?? "control_flow"; break;
+		}
+
+		const evidence: string[] = [];
+		let actionError: unknown;
+		try {
+			await runtime.executeStep(step, action.tab_id);
+			evidence.push("action dispatched");
+		} catch (error) {
+			actionError = error;
+			evidence.push(message(error));
+		}
+
+		let expectation: BrowserActExpectationEvidence | undefined;
+		let waitResult: BrowserWaitForResult | undefined;
+		let after: BrowserObservation | undefined;
+		let afterTargets: string[] | undefined;
+		try {
+			if (step.expect) {
+				waitResult = await runtime.wait(step.expect!, before, before.targetId, action.tab_id, action.timeout_ms, action.poll_ms);
+				expectation = expectationEvidence(waitResult);
+			}
+			after = await runtime.observe(action.tab_id);
+			afterTargets = await runtime.targetIds();
+		} catch (error) {
+			evidence.push(`post-action observation failed: ${message(error)}`);
+		}
+
+		timedOut ||= waitResult?.status === "timed_out";
+		const stale = isStale(actionError) || waitResult?.reason === "stale_ref" || expectation?.status === "unverifiable" && expectation.details.some((line) => /stale_ref| is stale/i.test(line));
+		const outcome = expectation?.status === "newly_verified" && !actionError ? "worked" : expectation?.status === "failed" || stale ? "didnt" : "unknown";
+		if (expectation) evidence.push(`expectation ${expectation.status}`);
+		steps.push(stepResult(index, step, outcome, evidence, expectation));
+
+		const postBoundary = after && afterTargets ? boundary(before, after, targets, afterTargets, dialogs, runtime) : undefined;
+		if (after && afterTargets) { current = after; targets = afterTargets; dialogs = runtime.dialogCount(); }
+		stopReason = stale
+			? "stale_ref"
+			: expectation?.status === "failed"
+				? "expectation_failed"
+				: actionError
+					? "action_failed"
+					: waitStopReason(waitResult)
+						?? postBoundary
+						?? (!after?.complete || expectation?.status === "preexisting" || expectation?.status === "unverifiable" ? "control_flow" : undefined);
+		if (stopReason) { stoppedAt = index; break; }
+	}
+
+	let finalExpectation: BrowserActExpectationEvidence | undefined;
+	const terminalNavigation = stopReason === "navigation" && stoppedAt === finalStepIndex && steps.at(-1)?.outcome === "worked";
+	if (action.expect && (!stopReason || terminalNavigation)) {
+		try {
+			const result = await runtime.wait(action.expect!, baseline, baseline.targetId, action.tab_id, action.timeout_ms, action.poll_ms);
+			finalExpectation = expectationEvidence(result);
+			timedOut ||= result.status === "timed_out";
+			if (finalExpectation.status === "failed") { stopReason = "expectation_failed"; stoppedAt = finalStepIndex; }
+			else if (finalExpectation.status === "unverifiable") { stopReason = waitStopReason(result) ?? "control_flow"; stoppedAt = finalStepIndex; }
+		} catch (error) {
+			finalExpectation = { status: "unverifiable", details: [message(error)] };
+			stopReason = "control_flow"; stoppedAt = finalStepIndex;
+		}
+	}
+
+	let successor: BrowserActResult["successor"] | undefined;
+	let successorError: unknown;
+	for (let attempt = 0; attempt < 3 && !successor; attempt += 1) {
+		try {
+			const observed = await runtime.observe(action.tab_id);
+			const successorTargets = await runtime.targetIds();
+			const lateBoundary = boundary(current, observed, targets, successorTargets, dialogs, runtime);
+			current = observed; targets = successorTargets; dialogs = runtime.dialogCount();
+			if (lateBoundary) {
+				stopReason ??= lateBoundary;
+				successorError = new Error(`${lateBoundary} changed successor observation`);
+				continue;
+			}
+			if (!observed.complete) throw new Error("successor observation incomplete");
+			if (action.expect && finalExpectation && finalExpectation.status !== "failed") {
+				const evaluation = runtime.evaluate(action.expect, observed, baseline);
+				finalExpectation = {
+					status: evaluation.truth === true ? finalExpectation.before === true ? "preexisting" : "newly_verified" : evaluation.truth === false ? "failed" : "unverifiable",
+					before: finalExpectation.before, after: evaluation.truth,
+					details: [...finalExpectation.details, ...evaluation.details.map((detail) => `successor: ${detail}`)],
+				};
+				if (evaluation.truth !== true) {
+					stopReason = evaluation.truth === false ? "expectation_failed" : "control_flow";
+					stoppedAt = finalStepIndex;
+				} else if (stopReason === "control_flow" && stoppedAt === finalStepIndex) {
+					stopReason = undefined;
+					stoppedAt = undefined;
+				}
+			}
+			const complete: CuaActionBrowserSnapshot = { type: "browser_snapshot", tab_id: action.tab_id, depth: Number.MAX_SAFE_INTEGER };
+			const presentation = runtime.present(observed, { type: "browser_snapshot", tab_id: action.tab_id, ...action.successor });
+			successor = {
+				status: "observed",
+				text: runtime.render(presentation),
+				url: observed.url,
+				title: observed.title,
+				diff: diffObservations(runtime.present(baseline, complete), runtime.present(observed, complete)),
+			};
+		} catch (error) {
+			successorError = error;
+		}
+	}
+	if (!successor) successor = { status: "unavailable", error: message(successorError ?? new Error("successor observation unavailable")) };
+
+	const failed = steps.some((step) => step.outcome === "didnt") || finalExpectation?.status === "failed";
+	const verified = action.expect
+		? finalExpectation?.status === "newly_verified"
+		: steps.length === action.steps.length && steps.every((step) => step.outcome === "worked");
+	const verifiedNavigation = stopReason === "navigation" && steps.length === action.steps.length && steps.at(-1)?.outcome === "worked";
+	return {
+		outcome: failed ? "didnt" : !timedOut && verified && (!stopReason || verifiedNavigation) ? "worked" : "unknown",
+		steps,
+		...(stoppedAt === undefined ? {} : { stopped_at: stoppedAt }),
+		...(stopReason ? { stop_reason: stopReason } : {}),
+		...(finalExpectation ? { final_expectation: finalExpectation } : {}),
+		successor,
+	};
+}
+
+function expectationEvidence(result: BrowserWaitForResult): BrowserActExpectationEvidence {
+	return { status: result.evidence, before: result.initial.truth, after: result.final.truth, details: result.details };
+}
+
+function boundary(before: BrowserObservation, after: BrowserObservation, oldTargets: readonly string[], newTargets: readonly string[], dialogs: number, runtime: BrowserActRuntime): BrowserActResult["stop_reason"] {
+	if (runtime.dialogCount() > dialogs) return "dialog";
+	if (oldTargets.length !== newTargets.length || oldTargets.some((id, index) => id !== newTargets[index])) return "control_flow";
+	if (before.targetId !== after.targetId || before.navigationEpoch !== after.navigationEpoch || runtime.liveNavigationEpoch(after.targetId) !== after.navigationEpoch) return "navigation";
+	if (before.generations.size !== after.generations.size) return "navigation";
+	for (const [frame, generation] of after.generations) {
+		if (runtime.liveGeneration(frame) !== generation || before.generations.get(frame) !== generation) return "navigation";
+	}
+	return undefined;
+}
+
+function waitStopReason(result?: BrowserWaitForResult): BrowserActResult["stop_reason"] {
+	if (!result?.reason) return undefined;
+	if (result.reason === "navigation" || result.reason === "dialog" || result.reason === "stale_ref") return result.reason;
+	return "control_flow";
+}
+
+function stepResult(index: number, step: CuaBrowserActStep, outcome: BrowserActStepResult["outcome"], evidence: string[], expectation?: BrowserActExpectationEvidence): BrowserActStepResult {
+	return { index, type: step.type, outcome, evidence, ...(expectation ? { expectation } : {}) };
+}
+
+function unavailable(error: unknown, stoppedAt: number): BrowserActResult {
+	return { outcome: "unknown", steps: [], stopped_at: stoppedAt, stop_reason: "control_flow", successor: { status: "unavailable", error: message(error) } };
+}
+
+function isStale(error: unknown): boolean {
+	return !!error && /(?:stale.*ref|ref.*stale)/i.test(message(error));
+}
+
+function message(error: unknown): string {
+	return error instanceof Error ? error.message : String(error);
+}

packages/agent/src/translator/browser-observation.ts

@@ -1,3 +1,5 @@
+import type { BrowserObservationDiff } from "./types";
+
 /** Minimal CDP accessibility node used by browser observations. */
 export interface AXNode {
 	nodeId: string;
@@ -71,6 +73,27 @@ export interface BrowserPresentation {
 	shape: string;
 }
 
+/** Compare complete presentations while normalizing away snapshot-scoped refs. */
+export function diffObservations(before: BrowserPresentation, after: BrowserPresentation): BrowserObservationDiff {
+	const normalize = (line: string) => line.replace(/\u0000/g, "ref");
+	const counts = new Map<string, number>();
+	for (const { text } of before.lines) {
+		const line = normalize(text);
+		counts.set(line, (counts.get(line) ?? 0) + 1);
+	}
+	const added: string[] = [];
+	for (const entry of after.lines) {
+		const line = normalize(entry.text);
+		const count = counts.get(line) ?? 0;
+		if (count === 0) added.push(line);
+		else counts.set(line, count - 1);
+	}
+	const removed = [...counts].flatMap(([line, count]) => Array.from({ length: count }, () => line));
+	const url = before.observation.url === after.observation.url ? undefined : { before: before.observation.url, after: after.observation.url };
+	const title = before.observation.title === after.observation.title ? undefined : { before: before.observation.title, after: after.observation.title };
+	return { changed: added.length > 0 || removed.length > 0 || !!url || !!title, added, removed, ...(url ? { url } : {}), ...(title ? { title } : {}) };
+}
+
 /** Build the lookup key for an iframe node's stitched child tree. */
 export function frameStitchKey(parentFrameKey: string, backendNodeId: number): string {
 	return `${parentFrameKey}\u0000${backendNodeId}`;