diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index 10fe14dd..6536688b 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -6,7 +6,7 @@ import { BrowserArtifactSession } from "./browser-artifact-session.js" import { BrowserCommandArtifactError, isBrowserCommandArtifactError } from "./browser-command-artifact-error.js" import { runBrowserMultiActorScenarioCommand } from "./browser-multi-actor-scenario-runner.js" import type { BrowserArtifact, BrowserProbeAuthSummary, BrowserProbeErrorRecord, BrowserProbeNetworkRecord, BrowserProbeViewport, BrowserProbeWebSocketRecord, BrowserStepRecord } from "./browser-artifacts.js" -import { attachBrowserCaptureListeners, launchChromiumBrowser, settleBrowserNetworkTasks } from "./browser-capture-session.js" +import { attachBrowserCaptureListeners, captureBrowserPageHtml, launchChromiumBrowser, settleBrowserNetworkTasks, trackBrowserNavigation, type BrowserNavigationTracker } from "./browser-capture-session.js" import { captureBrowserDomSnapshot, type BrowserDomSnapshotArtifact } from "./browser-dom-snapshot.js" import { browserAssertionsSummary, browserStepRecord, executeBrowserInteractionStep } from "./browser-interactions.js" import { browserCommandLivenessPolicy, isBrowserCommandLivenessError, withBrowserCommandLiveness } from "./browser-liveness.js" @@ -159,6 +159,9 @@ export async function runBrowserActionsCommand({ let environmentEvidence: BrowserArtifact["summary"]["environment"] | undefined let resolvedEnvironment: Awaited> | undefined let activePage: Page | undefined + let navigationTracker: BrowserNavigationTracker | undefined + let adaptiveCaptureNavigationUnsettled = false + let adaptiveCaptureBudgetMs = 0 let installedTransportFaults: InstalledBrowserTransportFaults | undefined let transportFaultReport: BrowserTransportFaultReport | undefined const abortHandler = () => { @@ -205,6 +208,7 @@ export async function runBrowserActionsCommand({ } if (context && runPlan.transportFaults) installedTransportFaults = await installBrowserTransportFaults(context, runPlan.transportFaults, { policy: browserPreviewTransportFaultPolicy(networkPolicy, topology.origins.localProxyOrigin), serviceWorkersBlocked: true }) const page = activePage = environmentRuntime?.page ?? await browser.newPage() + navigationTracker = trackBrowserNavigation(page) if (onProgress) { await page.exposeFunction("__wpCodeboxProbeCheckpointEvent", (checkpoint: unknown) => { const normalized = normalizeBrowserProbeScriptCheckpoint(checkpoint) @@ -433,9 +437,30 @@ export async function runBrowserActionsCommand({ if (capture.has("html")) { try { - const html = await page.content() - await artifactSession.writeText("html", "snapshot.html", html) - htmlSha256 = sha256(Buffer.from(html, "utf8")) + const captureBudgetMs = Math.min( + runPlan.adaptiveExploration?.stabilization.maxWaitMs ?? stepTimeoutMs, + livenessRemainingWallTimeMs(startedAtMs, totalTimeoutMs), + ) + adaptiveCaptureBudgetMs = captureBudgetMs + const captureResult = await captureBrowserPageHtml(page, navigationTracker, captureBudgetMs) + if (captureResult.status === "captured") { + await artifactSession.writeText("html", "snapshot.html", captureResult.html) + htmlSha256 = sha256(Buffer.from(captureResult.html, "utf8")) + } else if (adaptiveExplorationArtifact) { + adaptiveCaptureNavigationUnsettled = true + adaptiveExplorationArtifact.result.status = "incomplete" + adaptiveExplorationArtifact.result.diagnostics.unshift({ + code: "browser_adaptive_capture_navigation_unsettled", + message: "Adaptive exploration ended while document navigation remained active; HTML capture was omitted and partial browser evidence was retained.", + metadata: { attempts: captureResult.attempts, budgetMs: captureBudgetMs, waitedMs: captureResult.waitedMs, reason: captureResult.reason }, + }) + adaptiveExplorationArtifact.result.diagnostics.splice(adaptiveExplorationArtifact.contract.descriptorLimits.maxDiagnostics) + boundAdaptiveExplorationArtifact(adaptiveExplorationArtifact, Math.max(512, Math.floor(adaptiveExplorationArtifact.contract.budgets.maxArtifactBytes / 2))) + await artifactSession.writeJson("adaptiveExploration", "adaptive-exploration.json", adaptiveExplorationArtifact) + if (adaptiveExplorationSummary) adaptiveExplorationSummary.status = "incomplete" + } else { + throw new Error(captureResult.reason) + } } catch (error) { const serialized = serializeBrowserError("probe-error", error) errors.push(serialized) @@ -447,7 +472,19 @@ export async function runBrowserActionsCommand({ if (capture.has("screenshot")) { try { - await artifactSession.writeGenerated("screenshot", "screenshot.png", (path) => page.screenshot({ path, fullPage: true }).then(() => undefined)) + const screenshotCaptureBudgetMs = adaptiveCaptureNavigationUnsettled ? Math.min(adaptiveCaptureBudgetMs, livenessRemainingWallTimeMs(startedAtMs, totalTimeoutMs)) : 0 + if (adaptiveCaptureNavigationUnsettled && screenshotCaptureBudgetMs <= 0) throw new Error("Adaptive screenshot capture budget was exhausted before capture started.") + const screenshotCapture = artifactSession.writeGenerated("screenshot", "screenshot.png", (path) => page.screenshot({ path, fullPage: true }).then(() => undefined)) + if (adaptiveCaptureNavigationUnsettled) { + await withBrowserCommandLiveness({ + command: "wordpress.browser-actions", + phase: "adaptive partial screenshot capture", + operation: screenshotCapture, + policy: { wallTimeoutMs: screenshotCaptureBudgetMs, idleTimeoutMs: 0 }, + }) + } else { + await screenshotCapture + } screenshotSha256 = await fileSha256(screenshotPath) if (capture.has("dom-snapshot")) { domSnapshots.push(await captureBrowserActionDomSnapshot({ @@ -463,7 +500,16 @@ export async function runBrowserActionsCommand({ } catch (error) { const serialized = serializeBrowserError("probe-error", error) errors.push(serialized) - if (!pendingError) { + if (adaptiveCaptureNavigationUnsettled && adaptiveExplorationArtifact) { + adaptiveExplorationArtifact.result.diagnostics.unshift({ + code: "browser_adaptive_capture_screenshot_unavailable", + message: "Screenshot capture did not settle inside the remaining adaptive capture budget; other partial browser evidence was retained.", + metadata: { budgetMs: adaptiveCaptureBudgetMs, reason: error instanceof Error ? error.message : String(error) }, + }) + adaptiveExplorationArtifact.result.diagnostics.splice(adaptiveExplorationArtifact.contract.descriptorLimits.maxDiagnostics) + boundAdaptiveExplorationArtifact(adaptiveExplorationArtifact, Math.max(512, Math.floor(adaptiveExplorationArtifact.contract.budgets.maxArtifactBytes / 2))) + await artifactSession.writeJson("adaptiveExploration", "adaptive-exploration.json", adaptiveExplorationArtifact) + } else if (!pendingError) { pendingError = error instanceof Error ? error : new Error(String(error)) } } @@ -547,7 +593,7 @@ export async function runBrowserActionsCommand({ ...(capture.has("network") ? { waterfall: "files/browser/waterfall.json" } : {}), ...(capture.has("websocket") ? { websocket: "files/browser/websocket.json" } : {}), ...(redirectDiagnostics ? { redirectDiagnostics: "files/browser/redirect-diagnostics.json" } : {}), - ...(capture.has("screenshot") ? { screenshot: "files/browser/screenshot.png" } : {}), + ...(screenshotSha256 ? { screenshot: "files/browser/screenshot.png" } : {}), ...(screenshots.length > 0 ? { screenshots } : {}), ...(domSnapshots.length > 0 ? { domSnapshots: domSnapshots.map((snapshot) => snapshot.snapshot) } : {}), ...(verifierResults.length > 0 ? { verifierResults: verifierResults.map((result) => result.artifact) } : {}), @@ -578,7 +624,7 @@ export async function runBrowserActionsCommand({ ...(wordpressDiagnosticsSummary ? { wordpressDiagnostics: wordpressDiagnosticsSummary } : {}), ...(transportFaultReport ? { transportFaults: browserTransportFaultSummary(transportFaultReport) } : {}), replayability: browserProbeReplayability(capture), - screenshot: capture.has("screenshot"), + screenshot: Boolean(screenshotSha256), auth: authSummary, environment: environmentEvidence, viewport, @@ -617,6 +663,7 @@ export async function runBrowserActionsCommand({ summary: artifact.summary, }) abortSignal?.removeEventListener("abort", abortHandler) + navigationTracker?.dispose() } if (pendingError) { diff --git a/packages/runtime-playground/src/browser-capture-session.ts b/packages/runtime-playground/src/browser-capture-session.ts index 35f06c82..205966a0 100644 --- a/packages/runtime-playground/src/browser-capture-session.ts +++ b/packages/runtime-playground/src/browser-capture-session.ts @@ -2,7 +2,7 @@ import { redactString } from "@automattic/wp-codebox-core" import type { BrowserProbeErrorRecord, BrowserProbeNetworkRecord, BrowserProbeWebSocketRecord } from "./browser-artifacts.js" import { browserCommandLivenessPolicy } from "./browser-liveness.js" import { serializeBrowserConsoleMessage, serializeBrowserError, serializeBrowserFinishedRequest, serializeBrowserRequestFailure } from "./browser-metrics.js" -import type { Browser, Page } from "playwright" +import type { Browser, Page, Request } from "playwright" import { assertPlaywrightBrowserReady } from "./playwright-browser-provenance.js" export async function launchChromiumBrowser(): Promise { @@ -23,6 +23,139 @@ export function chromiumBrowserMetadata(browser: Browser): { name: "chromium"; c } } +export interface BrowserNavigationTracker { + navigating(): boolean + waitForSettlement(timeoutMs: number): Promise + dispose(): void +} + +export type BrowserHtmlCaptureResult = { + status: "captured" + html: string + attempts: number + waitedMs: number + navigationObserved: boolean +} | { + status: "navigation_unsettled" + attempts: number + waitedMs: number + navigationObserved: true + reason: string +} + +export function trackBrowserNavigation(page: Page): BrowserNavigationTracker { + const active = new Set() + const waiters = new Set<() => void>() + const notify = () => { + if (active.size > 0) return + for (const resolve of waiters) resolve() + waiters.clear() + } + const onRequest = (request: Request) => { + if (request.isNavigationRequest() && request.frame() === page.mainFrame()) active.add(request) + } + const onRequestFailed = (request: Request) => { + if (!active.has(request)) return + active.clear() + notify() + } + const onDomContentLoaded = () => { + active.clear() + notify() + } + page.on("request", onRequest) + page.on("requestfailed", onRequestFailed) + page.on("domcontentloaded", onDomContentLoaded) + + return { + navigating: () => active.size > 0, + async waitForSettlement(timeoutMs) { + if (active.size === 0) return true + if (timeoutMs <= 0) return false + return await new Promise((resolve) => { + let timeout: ReturnType | undefined + const settled = () => { + if (timeout) clearTimeout(timeout) + waiters.delete(settled) + resolve(true) + } + waiters.add(settled) + if (active.size === 0) { + settled() + return + } + timeout = setTimeout(() => { + waiters.delete(settled) + resolve(false) + }, timeoutMs) + }) + }, + dispose() { + page.off("request", onRequest) + page.off("requestfailed", onRequestFailed) + page.off("domcontentloaded", onDomContentLoaded) + active.clear() + notify() + }, + } +} + +export async function captureBrowserPageHtml(page: Page, navigation: BrowserNavigationTracker, timeoutMs: number): Promise { + const startedAt = Date.now() + const deadline = startedAt + Math.max(0, timeoutMs) + let attempts = 0 + let navigationObserved = navigation.navigating() + let reason = "Navigation did not settle before the browser capture budget expired." + + while (true) { + if (navigation.navigating()) { + navigationObserved = true + const settled = await navigation.waitForSettlement(Math.max(0, deadline - Date.now())) + if (!settled) return { status: "navigation_unsettled", attempts, waitedMs: Date.now() - startedAt, navigationObserved: true, reason } + } + + attempts += 1 + try { + const content = await captureBrowserContentWithin(page, Math.max(0, deadline - Date.now())) + if (content.status === "timeout") { + return { status: "navigation_unsettled", attempts, waitedMs: Date.now() - startedAt, navigationObserved: true, reason: "page.content did not settle before the browser capture budget expired." } + } + return { status: "captured", html: content.html, attempts, waitedMs: Date.now() - startedAt, navigationObserved } + } catch (error) { + if (!browserContentNavigationRace(error)) throw error + navigationObserved = true + reason = error instanceof Error ? error.message : String(error) + const remainingMs = Math.max(0, deadline - Date.now()) + if (remainingMs <= 0) return { status: "navigation_unsettled", attempts, waitedMs: Date.now() - startedAt, navigationObserved: true, reason } + if (!navigation.navigating()) await page.waitForTimeout(Math.min(10, remainingMs)) + const settled = await navigation.waitForSettlement(remainingMs) + if (!settled || Date.now() >= deadline) return { status: "navigation_unsettled", attempts, waitedMs: Date.now() - startedAt, navigationObserved: true, reason } + } + } +} + +async function captureBrowserContentWithin(page: Page, timeoutMs: number): Promise<{ status: "captured"; html: string } | { status: "timeout" }> { + if (timeoutMs <= 0) return { status: "timeout" } + const content = page.content() + content.catch(() => undefined) + let timeout: ReturnType | undefined + try { + return await Promise.race([ + content.then((html) => ({ status: "captured" as const, html })), + new Promise<{ status: "timeout" }>((resolve) => { + timeout = setTimeout(() => resolve({ status: "timeout" }), timeoutMs) + }), + ]) + } finally { + if (timeout) clearTimeout(timeout) + } +} + +function browserContentNavigationRace(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error) + return /page\.content: Unable to retrieve content because the page is navigating and changing the content\./i.test(message) +} + export function attachBrowserCaptureListeners({ captureConsole, captureErrors, diff --git a/tests/browser-actions-navigation-capture.browser.test.ts b/tests/browser-actions-navigation-capture.browser.test.ts new file mode 100644 index 00000000..fe8f7fef --- /dev/null +++ b/tests/browser-actions-navigation-capture.browser.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict" +import { access, mkdtemp, readFile, rm } from "node:fs/promises" +import { createServer } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import { browserAdaptiveExplorationContract, type RuntimeCreateSpec } from "../packages/runtime-core/src/index.js" +import { runBrowserActionsCommand } from "../packages/runtime-playground/src/browser-actions-runner.js" +import { captureBrowserPageHtml, trackBrowserNavigation, type BrowserNavigationTracker } from "../packages/runtime-playground/src/browser-capture-session.js" +import { browserEnvironmentCell, resolvePlaywrightBrowserEnvironment } from "../packages/runtime-playground/src/browser-environment-matrix.js" +import { chromium } from "playwright" + +const runtimeSpec: RuntimeCreateSpec = { + backend: "wordpress-playground", + environment: {}, + policy: { network: "deny", filesystem: "sandbox", commands: ["wordpress.browser-actions"], secrets: "none", approvals: "never" }, +} + +test("HTML capture remains unchanged when no navigation is active", async () => { + const browser = await chromium.launch({ headless: true }) + const page = await browser.newPage() + const navigation = trackBrowserNavigation(page) + try { + await page.setContent("stable capture
ready
") + const result = await captureBrowserPageHtml(page, navigation, 50) + assert.equal(result.status, "captured") + assert.equal(result.attempts, 1) + assert.equal(result.navigationObserved, false) + assert.match(result.html, /stable capture/) + } finally { + navigation.dispose() + await browser.close() + } +}) + +test("HTML capture waits boundedly for an active navigation and captures the settled document", async () => { + const fixture = await navigationFixture(75) + const browser = await chromium.launch({ headless: true }) + const page = await browser.newPage() + const navigation = trackBrowserNavigation(page) + try { + await page.goto(fixture.url) + const navigated = page.goto(`${fixture.url}/slow`) + await fixture.navigationStarted + const result = await captureBrowserPageHtml(page, navigation, 500) + assert.equal(result.status, "captured") + assert.equal(result.navigationObserved, true) + assert.match(result.html, /settled destination/) + await navigated + } finally { + navigation.dispose() + await browser.close() + await fixture.close() + } +}) + +test("HTML capture contains a page.content navigation race and retries after settlement", async () => { + let attempts = 0 + const page = { + async waitForTimeout() {}, + async content() { + attempts += 1 + if (attempts === 1) throw new Error("page.content: Unable to retrieve content because the page is navigating and changing the content.") + return "settled retry" + }, + } + const navigation: BrowserNavigationTracker = { + navigating: () => false, + waitForSettlement: async () => true, + dispose() {}, + } + const result = await captureBrowserPageHtml(page as never, navigation, 50) + assert.equal(result.status, "captured") + assert.equal(result.attempts, 2) + assert.equal(result.navigationObserved, true) +}) + +test("HTML capture bounds a page.content call that never settles", async () => { + const page = { + async content() { return await new Promise(() => {}) }, + } + const navigation: BrowserNavigationTracker = { + navigating: () => false, + waitForSettlement: async () => true, + dispose() {}, + } + const startedAt = Date.now() + const result = await captureBrowserPageHtml(page as never, navigation, 25) + assert.equal(result.status, "navigation_unsettled") + assert(Date.now() - startedAt < 150) +}) + +test("adaptive capture classifies unresolved navigation and retains partial evidence", async () => { + const fixture = await navigationFixture(10) + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-adaptive-navigation-capture-")) + const browser = await chromium.launch({ headless: true }) + const context = await browser.newContext() + const page = await context.newPage() + await page.addInitScript("globalThis.__name = value => value") + ;(page as unknown as { content(): Promise }).content = async () => { + throw new Error("page.content: Unable to retrieve content because the page is navigating and changing the content.") + } + const resolved = await resolvePlaywrightBrowserEnvironment(browserEnvironmentCell({}), browser) + const startedAt = Date.now() + try { + const result = await runBrowserActionsCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + session: { browser, requested: {}, resolved, runtime: { context, page, close: async () => context.close() } }, + spec: { command: "wordpress.browser-actions", args: [] }, + plan: { + steps: [], + capture: new Set(["steps", "html", "network", "screenshot"]), + stepTimeoutMs: 500, + totalTimeoutMs: 2_000, + networkSettleTimeoutMs: 100, + maxDomSnapshotElements: 20, + adaptiveExploration: browserAdaptiveExplorationContract({ + seed: "active-navigation-capture", + startUrl: fixture.url, + actionFamilies: ["click"], + resetPolicy: { mode: "none" }, + failOnFinding: false, + budgets: { maxActions: 1, maxStates: 2, maxTransitions: 1, maxDurationMs: 1_000, maxArtifactBytes: 100_000, maxErrors: 5 }, + stabilization: { pollIntervalMs: 10, quietWindowMs: 10, maxWaitMs: 40, maxMutationRecords: 20 }, + }), + }, + }) + const adaptive = JSON.parse(await readFile(join(artifactRoot, "files/browser/adaptive-exploration.json"), "utf8")) + assert.equal(result.artifact.summary.adaptiveExploration?.status, "incomplete") + assert.equal(result.artifact.summary.htmlSnapshot, false) + assert.equal(adaptive.result.status, "incomplete") + assert(adaptive.result.diagnostics.some(({ code }: { code: string }) => code === "browser_adaptive_capture_navigation_unsettled")) + assert(Date.now() - startedAt < 1_500, "capture settlement must remain inside the command budget") + await access(join(artifactRoot, "files/browser/steps.jsonl")) + await access(join(artifactRoot, "files/browser/network.jsonl")) + await access(join(artifactRoot, "files/browser/screenshot.png")) + await assert.rejects(access(join(artifactRoot, "files/browser/snapshot.html"))) + } finally { + await context.close() + await browser.close() + await rm(artifactRoot, { recursive: true, force: true }) + await fixture.close() + } +}) + +async function navigationFixture(delayMs: number) { + let resolveNavigationStarted!: () => void + const navigationStarted = new Promise((resolve) => { resolveNavigationStarted = resolve }) + const httpServer = createServer((request, response) => { + if (request.url === "/slow") { + resolveNavigationStarted() + setTimeout(() => { + response.setHeader("content-type", "text/html") + response.end("settled destination") + }, delayMs) + return + } + response.setHeader("content-type", "text/html") + response.end("starting document") + }) + await new Promise((resolve) => httpServer.listen(0, "127.0.0.1", resolve)) + const address = httpServer.address() + assert(address && typeof address === "object") + const url = `http://127.0.0.1:${address.port}` + return { + url, + navigationStarted, + server: { + serverUrl: url, + playground: { async run() { return { text: "", exitCode: 0 } } }, + async [Symbol.asyncDispose]() {}, + }, + close: () => new Promise((resolve, reject) => httpServer.close((error) => error ? reject(error) : resolve())), + } +}