Add semantic browser waits

packages/agent/src/index.ts

@@ -6,7 +6,12 @@ export { InternalComputerTranslator } from "./translator/translator";
 export { CdpConnection } from "./translator/cdp";
 export { BrowserExecutor } from "./translator/browser";
 export type { BrowserFindCandidate, BrowserRefState } from "./translator/browser";
-export type { BatchExecutionResult, BatchReadResult } from "./translator/types";
+export type {
+	BatchExecutionResult,
+	BatchReadResult,
+	BrowserExpectationEvidence,
+	BrowserWaitForResult,
+} from "./translator/types";
 export { createCuaComputerTools } from "./tools";
 export type {
 	BatchDetails,

packages/agent/src/tools.ts

@@ -16,6 +16,7 @@ import {
 } from "@onkernel/cua-ai";
 import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator";
 import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core";
+import type { BrowserWaitForResult } from "./translator/types";
 
 export interface ComputerToolOptions {
 	browser: KernelBrowser;
@@ -37,6 +38,7 @@ export interface BatchDetails {
 		| { type: "screenshot"; bytes: number }
 		| { type: "cursor_position"; x: number; y: number }
 		| { type: "browser_text"; label: string; bytes: number }
+		| { type: "browser_wait_for"; result: BrowserWaitForResult }
 	>;
 }
 
@@ -170,6 +172,9 @@ async function executeBatchTool(
 			} else if (read.type === "browser_text") {
 				readResults.push({ type: "browser_text", label: read.label, bytes: read.text.length });
 				content.push({ type: "text", text: read.text });
+			} else if (read.type === "browser_wait_for") {
+				readResults.push(read);
+				content.push({ type: "text", text: formatBrowserWaitResult(read.result) });
 			} else {
 				readResults.push({ type: "screenshot", bytes: read.data.length });
 				content.push({ type: "image", data: read.data.toString("base64"), mimeType: read.mimeType });
@@ -185,7 +190,14 @@ async function executeBatchTool(
 	} catch (err) {
 		throw new Error(`Actions failed: ${errorMessage(err)}`, { cause: err });
 	}
-	return { content, details: { statusText: "Actions executed successfully.", readResults } };
+	const waits = readResults.flatMap((read) => read.type === "browser_wait_for" ? [read.result] : []);
+	const failedWait = ["interrupted", "timed_out", "unverifiable"].find((status) => waits.some((wait) => wait.status === status));
+	const statusText = waits.length === 0
+		? "Actions executed successfully."
+		: failedWait
+			? `Browser condition ${failedWait}.`
+			: "Browser condition satisfied.";
+	return { content, details: { statusText, readResults } };
 }
 
 async function executeNavigationTool(
@@ -257,6 +269,11 @@ async function executePlaywrightTool(translator: InternalComputerTranslator, par
 	}
 }
 
+function formatBrowserWaitResult(result: BrowserWaitForResult): string {
+	const reason = result.reason ? ` (${result.reason})` : "";
+	return [`wait_for: ${result.status}/${result.evidence}${reason} after ${result.elapsed_ms}ms`, ...result.details].join("\n");
+}
+
 function formatPlaywrightResult(result: unknown): string {
 	return typeof result === "string" ? result : JSON.stringify(result);
 }

packages/agent/src/translator/browser-observation.ts

@@ -21,6 +21,8 @@ export interface NthIndex {
 export interface RenderContext {
 	targetId: string;
 	frameKey: string;
+	/** Target whose CDP session owns this frame (page target or OOPIF target). */
+	sessionTargetId: string;
 	sessionId: string;
 	generation: number;
 	interactiveOnly: boolean;
@@ -51,12 +53,14 @@ export interface ObservedNode {
 /** Stable structured browser state collected before presentation filtering. */
 export interface BrowserObservation {
 	targetId: string;
+	navigationEpoch: number;
 	tree: FrameStitch;
-	stitches: Map<number, FrameStitch>;
+	stitches: Map<string, FrameStitch>;
 	nodes: ObservedNode[];
 	url: string;
 	title: string;
 	generations: Map<string, number>;
+	complete: boolean;
 }
 
 /** Render-ready projection of one structured browser observation. */
@@ -67,6 +71,11 @@ export interface BrowserPresentation {
 	shape: string;
 }
 
+/** Build the lookup key for an iframe node's stitched child tree. */
+export function frameStitchKey(parentFrameKey: string, backendNodeId: number): string {
+	return `${parentFrameKey}\u0000${backendNodeId}`;
+}
+
 /** Signals that browser state changed while an observation was collected. */
 export class ObservationChangedError extends Error {
 	constructor(message = "Browser observation changed during collection") {

packages/agent/src/translator/browser-wait.ts

@@ -0,0 +1,185 @@
+import type { CuaBrowserExpectation } from "@onkernel/cua-ai";
+import { staticTextRun, type AXNode, type BrowserObservation } from "./browser-observation";
+import type { BrowserExpectationEvidence, BrowserWaitForResult, BrowserWaitReason } from "./types";
+
+/** Internal expectation truth and optional lifecycle reason. */
+export interface BrowserExpectationEvaluation extends BrowserExpectationEvidence {
+	reason?: BrowserWaitReason;
+}
+
+type RefExpectation = Extract<CuaBrowserExpectation, { type: "ref" }>;
+/** Resolve a ref expectation against one structured observation. */
+export type BrowserRefResolver = (expectation: RefExpectation, observation: BrowserObservation) => BrowserExpectationEvaluation;
+
+/** Browser capabilities required by the semantic wait engine. */
+export interface BrowserWaitRuntime {
+	selectTarget(tabId?: string): Promise<string>;
+	observeTarget(targetId: string): Promise<BrowserObservation>;
+	dialogCount(): number;
+	targetExists(targetId: string): Promise<boolean>;
+	liveGeneration(frameId: string): number;
+	resolveRef: BrowserRefResolver;
+	now?(): number;
+	delay?(ms: number): Promise<void>;
+}
+
+/** Timeout, polling, target, and condition options for a semantic wait. */
+export interface BrowserWaitOptions {
+	expect: CuaBrowserExpectation;
+	timeoutMs?: number;
+	pollMs?: number;
+	tabId?: string;
+}
+
+function expectationNodes(observation: BrowserObservation): AXNode[] {
+	const nodes = observation.nodes.map(({ node }) => node);
+	for (const tree of [observation.tree, ...observation.stitches.values()]) {
+		for (const node of tree.byId.values()) {
+			const children = node.childIds ?? [];
+			for (let index = 0; index < children.length; index += 1) {
+				const run = staticTextRun(tree.byId, children, index);
+				if (run) { nodes.push(run.node); index = run.end; }
+			}
+		}
+	}
+	return nodes;
+}
+
+/** Evaluate a semantic condition without minting refs. Unknown means the observation cannot prove the claim. */
+export function evaluateBrowserExpectation(
+	expectation: CuaBrowserExpectation,
+	observation: BrowserObservation,
+	baseline: BrowserObservation,
+	resolveRef: BrowserRefResolver,
+): BrowserExpectationEvaluation {
+	if ("all" in expectation || "any" in expectation) {
+		const all = "all" in expectation;
+		const children = (all ? expectation.all : expectation.any).map((child) => evaluateBrowserExpectation(child, observation, baseline, resolveRef));
+		const truth = all
+			? children.some((child) => child.truth === false) ? false : children.some((child) => child.truth === undefined) ? undefined : true
+			: children.some((child) => child.truth === true) ? true : children.some((child) => child.truth === undefined) ? undefined : false;
+		return {
+			truth,
+			details: children.flatMap((child) => child.details),
+			...(truth === undefined ? { reason: children.find((child) => child.truth === undefined)?.reason } : {}),
+		};
+	}
+	if (expectation.type === "text" || expectation.type === "role_name") {
+		const found = expectationNodes(observation).some((node) => !node.ignored && (expectation.type === "text"
+			? (node.name?.value ?? "").replace(/\s+/g, " ").toLowerCase().includes(expectation.text.replace(/\s+/g, " ").toLowerCase())
+			: (expectation.role === undefined || node.role?.value === expectation.role) && (expectation.name === undefined || node.name?.value === expectation.name)));
+		const expected = expectation.exists ?? true;
+		const truth = !found && !observation.complete ? undefined : found === expected;
+		return { truth, details: [`${expectation.type} ${found ? "present" : "absent"}${observation.complete ? "" : "; observation incomplete"}`], ...(!observation.complete && !found ? { reason: "incomplete_observation" as const } : {}) };
+	}
+	if (expectation.type === "url" || expectation.type === "title") {
+		const value = observation[expectation.type];
+		const initial = baseline[expectation.type];
+		const truth = (expectation.equals === undefined || value === expectation.equals) &&
+			(expectation.contains === undefined || value.includes(expectation.contains)) &&
+			(expectation.changed === undefined || (value !== initial) === expectation.changed);
+		return { truth, details: [`${expectation.type}=${JSON.stringify(value)}`] };
+	}
+	if (expectation.type === "ref") return resolveRef(expectation, observation);
+	return { truth: undefined, details: ["unsupported expectation"] };
+}
+
+/** Poll a semantic browser condition; in-flight browser reads settle before timeout is reported. */
+export async function waitForBrowserExpectation(runtime: BrowserWaitRuntime, options: BrowserWaitOptions): Promise<BrowserWaitForResult> {
+	const now = runtime.now ?? Date.now;
+	const sleep = runtime.delay ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+	const started = now();
+	const timeout = options.timeoutMs ?? 2_000;
+	const poll = options.pollMs ?? 50;
+	const remaining = () => Math.max(0, timeout - (now() - started));
+	const expired = () => now() - started >= timeout;
+	let targetId: string;
+	let baseline: BrowserObservation;
+	try {
+		targetId = await beforeDeadline(() => runtime.selectTarget(options.tabId), remaining(), now);
+		if (expired()) return timedOut(started, now());
+		baseline = await runtime.observeTarget(targetId);
+		if (expired()) return timedOut(started, now());
+	} catch (error) {
+		if (error instanceof WaitDeadlineError) return timedOut(started, now());
+		return failedObservation(started, now(), error);
+	}
+	const initial = evaluateBrowserExpectation(options.expect, baseline, baseline, runtime.resolveRef);
+	if (!baseline.complete) return terminal("unverifiable", "unverifiable", initial, initial, started, now(), "incomplete_observation");
+	if (expired()) return timedOut(started, now());
+	if (initial.reason === "stale_ref") return terminal("interrupted", "unverifiable", initial, initial, started, now(), initial.reason);
+	if (initial.truth === true) return terminal("satisfied", "preexisting", initial, initial, started, now());
+	const dialogs = runtime.dialogCount();
+	let final = initial;
+	while (!expired()) {
+		await sleep(Math.min(poll, remaining()));
+		if (expired()) break;
+		let exists: boolean;
+		try { exists = await beforeDeadline(() => runtime.targetExists(targetId), remaining(), now); }
+		catch (error) { if (error instanceof WaitDeadlineError) break; return failedObservation(started, now(), error, initial, final); }
+		if (expired()) break;
+		if (!exists) return terminal("interrupted", "unverifiable", initial, final, started, now(), "target_detached");
+		let observation: BrowserObservation;
+		try { observation = await runtime.observeTarget(targetId); }
+		catch (error) { return failedObservation(started, now(), error, initial, final); }
+		if (observation.targetId !== targetId) return terminal("interrupted", "unverifiable", initial, final, started, now(), "target_changed");
+		if (runtime.dialogCount() > dialogs) return terminal("interrupted", "unverifiable", initial, final, started, now(), "dialog");
+		const navigated = observation.navigationEpoch !== baseline.navigationEpoch;
+		if (navigated) {
+			if (isLocationExpectation(options.expect)) {
+				final = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef);
+				if (final.truth === true && !expired()) return terminal("satisfied", "newly_verified", initial, final, started, now());
+			}
+			return terminal("interrupted", "unverifiable", initial, final, started, now(), "navigation");
+		}
+		if (!observation.complete) {
+			const partial = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef);
+			final = { ...partial, truth: undefined, reason: "incomplete_observation" };
+			continue;
+		}
+		const removedFrame = [...baseline.generations.keys()].some((key) => !observation.generations.has(key));
+		if (removedFrame) return terminal("unverifiable", "unverifiable", initial, final, started, now(), "incomplete_observation");
+		if (expired()) break;
+		final = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef);
+		if (expired()) break;
+		if (final.reason === "stale_ref") return terminal("interrupted", "unverifiable", initial, final, started, now(), final.reason);
+		if (final.truth === true) return terminal("satisfied", "newly_verified", initial, final, started, now());
+	}
+	if (final.truth === undefined) return terminal("unverifiable", "unverifiable", initial, final, started, now(), final.reason ?? "incomplete_observation");
+	return terminal("timed_out", "failed", initial, final, started, now());
+}
+
+class WaitDeadlineError extends Error {}
+
+async function beforeDeadline<T>(operation: () => Promise<T>, remaining: number, now: () => number): Promise<T> {
+	if (remaining <= 0) throw new WaitDeadlineError();
+	const started = now();
+	try {
+		const result = await operation();
+		if (Number.isFinite(remaining) && now() - started >= remaining) throw new WaitDeadlineError();
+		return result;
+	} catch (error) {
+		if (error instanceof WaitDeadlineError || Number.isFinite(remaining) && now() - started >= remaining) throw new WaitDeadlineError();
+		throw error;
+	}
+}
+
+function isLocationExpectation(expectation: CuaBrowserExpectation): boolean {
+	if ("all" in expectation) return expectation.all.every(isLocationExpectation);
+	if ("any" in expectation) return expectation.any.every(isLocationExpectation);
+	return expectation.type === "url" || expectation.type === "title";
+}
+
+function timedOut(started: number, ended: number): BrowserWaitForResult {
+	const evidence = { truth: undefined, details: [] };
+	return terminal("timed_out", "failed", evidence, evidence, started, ended);
+}
+
+function terminal(status: BrowserWaitForResult["status"], evidence: BrowserWaitForResult["evidence"], initial: BrowserExpectationEvidence, final: BrowserExpectationEvidence, started: number, ended: number, reason?: BrowserWaitReason): BrowserWaitForResult {
+	return { status, evidence, initial, final, elapsed_ms: Math.max(0, ended - started), ...(reason ? { reason } : {}), details: [...initial.details.map((detail) => `initial: ${detail}`), ...final.details.map((detail) => `final: ${detail}`)] };
+}
+
+function failedObservation(started: number, ended: number, error: unknown, initial: BrowserExpectationEvidence = { truth: undefined, details: [] }, final = initial): BrowserWaitForResult {
+	const message = error instanceof Error ? error.message : String(error);
+	return { ...terminal("unverifiable", "unverifiable", initial, final, started, ended, "observation_failed"), details: [...initial.details, message] };
+}