From 6e6abed969eac9eefa258d990aea9e267706afc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 19 Aug 2026 12:13:13 +0200 Subject: [PATCH] refactor: migrate screenshot to request-bound runtime Retires the last dispatchCommand edges for screen capture: the generic-route command, the sparse-snapshot fallback, and the Android snapshot-timeout evidence capture all admit exact owner facts and bind once (ADR 0019, cutover rule R39). --overlay-refs becomes part of the declared use, so a target that can capture pixels but not a tree is refused before anything is written to disk. --- packages/contracts/src/facades/platform.ts | 16 +- .../src/platform-runtime-operations.ts | 37 + .../src/platform-runtime-unavailable.test.ts | 6 + .../src/platform-runtime-unavailable.ts | 27 +- .../contracts/src/screenshot-runtime.test.ts | 56 ++ packages/contracts/src/screenshot-runtime.ts | 88 +++ packages/platform-android/src/runtime.test.ts | 2 + packages/platform-android/src/runtime.ts | 17 + packages/platform-apple/src/runtime.test.ts | 14 + packages/platform-apple/src/runtime.ts | 30 + .../platform-harmonyos/src/runtime.test.ts | 2 + packages/platform-harmonyos/src/runtime.ts | 20 + packages/platform-linux/src/runtime.test.ts | 4 + packages/platform-linux/src/runtime.ts | 17 + packages/platform-vega/src/runtime.test.ts | 5 + packages/platform-vega/src/runtime.ts | 6 + packages/platform-web/src/runtime.test.ts | 4 + packages/platform-web/src/runtime.ts | 20 +- .../src/app-log-runtime.test.ts | 3 + .../provider-limrun/src/app-log-runtime.ts | 10 + .../src/platform-runtime.test.ts | 15 +- .../src/platform-runtime.ts | 82 +- packages/provider-webdriver/src/runtime.ts | 1 + scripts/layering/daemon-modularity.test.ts | 19 +- scripts/layering/daemon-modularity.ts | 8 +- .../layering/runtime-command-cutover-table.ts | 24 + src/__tests__/test-file-size-ratchet.test.ts | 2 +- .../test-utils/runtime-operation-facts.ts | 2 + .../__tests__/command-explain.test.ts | 3 +- .../capability-plugin-routing-parity.test.ts | 1 - .../__tests__/dispatch-screenshot.test.ts | 42 + src/core/capabilities.ts | 3 +- .../__tests__/parity.test.ts | 2 + .../screenshot-runtime-execution.test.ts | 18 + src/core/command-descriptor/registry.ts | 26 +- src/core/dispatch.ts | 28 - .../request-router-screenshot.test.ts | 748 ++++++------------ .../__tests__/screenshot-runtime-fixture.ts | 119 +++ .../__tests__/screenshot-runtime.test.ts | 198 +++++ .../__tests__/snapshot-runtime-fixture.ts | 16 + .../sparse-fallback-screenshot.test.ts | 87 +- src/daemon/__tests__/viewport-runtime.test.ts | 49 +- .../android-snapshot-timeout-evidence.ts | 90 ++- src/daemon/generic-runtime-execution.ts | 28 + .../handlers/__tests__/install-source.test.ts | 2 + .../session-capabilities.fixtures.ts | 24 +- .../__tests__/session-capabilities.test.ts | 7 + .../__tests__/session-command-harness.ts | 6 + .../handlers/__tests__/session-state.test.ts | 2 + .../__tests__/snapshot-handler.test.ts | 28 +- src/daemon/handlers/session-inventory.ts | 1 + src/daemon/request-generic-dispatch.ts | 311 +++----- src/daemon/request-router.ts | 22 +- src/daemon/screenshot-overlay-draw.ts | 153 ++++ src/daemon/screenshot-overlay-rects.ts | 5 + src/daemon/screenshot-overlay.ts | 153 +--- src/daemon/screenshot-runtime-binding.ts | 118 +++ src/daemon/screenshot-runtime.ts | 277 ++++++- src/daemon/snapshot-command-runtime.ts | 2 + src/daemon/snapshot-runtime.ts | 2 + src/daemon/sparse-fallback-screenshot.ts | 51 +- src/daemon/viewport-runtime.ts | 14 +- src/platform-runtime-gateway.test.ts | 78 ++ src/platform-runtime-gateway.ts | 2 + 64 files changed, 2103 insertions(+), 1150 deletions(-) create mode 100644 packages/contracts/src/screenshot-runtime.test.ts create mode 100644 packages/contracts/src/screenshot-runtime.ts create mode 100644 src/core/__tests__/dispatch-screenshot.test.ts create mode 100644 src/core/command-descriptor/__tests__/screenshot-runtime-execution.test.ts create mode 100644 src/daemon/__tests__/screenshot-runtime-fixture.ts create mode 100644 src/daemon/__tests__/screenshot-runtime.test.ts create mode 100644 src/daemon/generic-runtime-execution.ts create mode 100644 src/daemon/screenshot-overlay-draw.ts create mode 100644 src/daemon/screenshot-runtime-binding.ts diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index 2cdb0918b8..6ac3c411d9 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -213,11 +213,13 @@ export { appsRuntimeUse, captureSnapshotUse, defineUse, + resolveScreenshotRuntimePlan, resolveSnapshotRuntimePlan, + screenshotRuntimePlanUses, snapshotRuntimePlanUses, viewportRuntimeUse, } from '../platform-runtime-operations.ts'; -export type { SnapshotRuntimePlan } from '../platform-runtime-operations.ts'; +export type { ScreenshotRuntimePlan, SnapshotRuntimePlan } from '../platform-runtime-operations.ts'; export type { PlatformRuntimeHost, PlatformRuntimeModule, @@ -235,6 +237,18 @@ export { shutdownTargetUse, } from '../platform-runtime-operations.ts'; export type { DeviceReadinessRuntimePlan } from '../platform-runtime-operations.ts'; +export { + bindLocalScreenshotInteractor, + bindProviderScreenshotInteractor, + screenshotRuntimeOperationFacts, +} from '../screenshot-runtime.ts'; +export type { + CaptureScreenshotInput, + ScreenshotOptions, + ScreenshotRuntimeExecution, + ScreenshotRuntimeOperations, + ScreenshotRuntimeOperationFacts, +} from '../screenshot-runtime.ts'; export { bindLocalSnapshotInteractor, bindProviderSnapshotInteractor, diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index cd96c85b76..53f4e0d047 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -12,6 +12,7 @@ import type { AppStateRuntimeHost, AppStateRuntimeOperations } from './app-state import type { NetworkRuntimeHost, NetworkRuntimeOperations } from './network-runtime.ts'; import type { ScreenRecordingRuntimeHost } from './screen-recording-runtime-host.ts'; import type { ScreenRecordingRuntimeOperations } from './screen-recording-runtime.ts'; +import type { ScreenshotRuntimeOperations } from './screenshot-runtime.ts'; import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; import type { @@ -43,6 +44,7 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & AppStateRuntimeOperations & NetworkRuntimeOperations & ScreenRecordingRuntimeOperations & + ScreenshotRuntimeOperations & SnapshotRuntimeOperations & ViewportRuntimeOperations & DeviceReadinessRuntimeOperations & @@ -133,6 +135,41 @@ export function resolveSnapshotRuntimePlan(input: { use: captureSnapshotWithoutActiveAppUse, }); } + +const captureScreenshotUse = defineUse({ required: ['captureScreenshot'] }); +/** + * `--overlay-refs` annotates the capture with the refs of a snapshot taken in the same request, so + * the snapshot is part of what the command requires — not something to discover after the PNG is + * already on disk. Declaring it in the use is what lets admission refuse the whole request up + * front on a target that can capture pixels but not a tree. + */ +const captureScreenshotWithOverlayRefsUse = defineUse({ + required: ['captureScreenshot', 'captureSnapshot'], +}); + +export const screenshotRuntimePlanUses = Object.freeze([ + captureScreenshotUse, + captureScreenshotWithOverlayRefsUse, +] as const); + +export type ScreenshotRuntimePlan = + | Readonly<{ kind: 'capture'; use: typeof captureScreenshotUse }> + | Readonly<{ + kind: 'capture-with-overlay-refs'; + use: typeof captureScreenshotWithOverlayRefsUse; + }>; + +/** Selects one owner-fact-backed capture plan from normalized command intent. */ +export function resolveScreenshotRuntimePlan( + input: Readonly<{ overlayRefs: boolean }>, +): ScreenshotRuntimePlan { + return input.overlayRefs + ? Object.freeze({ + kind: 'capture-with-overlay-refs', + use: captureScreenshotWithOverlayRefsUse, + }) + : Object.freeze({ kind: 'capture', use: captureScreenshotUse }); +} export const deviceBootRuntimeUses = Object.freeze([bootTargetUse, bootTargetHeadlessUse] as const); export type DeviceReadinessRuntimePlan = diff --git a/packages/contracts/src/platform-runtime-unavailable.test.ts b/packages/contracts/src/platform-runtime-unavailable.test.ts index 3c207e1ed6..e9f15472a5 100644 --- a/packages/contracts/src/platform-runtime-unavailable.test.ts +++ b/packages/contracts/src/platform-runtime-unavailable.test.ts @@ -29,6 +29,7 @@ test('generic unavailable binding preserves exact provider ownership and mode', const binding = createUnavailablePlatformRuntimeBinding(device, owner, { appLog: { available: false, reason: 'unsupported-provider-mode' }, network: { available: false, reason: 'owner-capability-missing' }, + screenshot: { available: false, reason: 'unsupported-device-kind' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, lifecycle, }); @@ -39,6 +40,11 @@ test('generic unavailable binding preserves exact provider ownership and mode', available: false, reason: 'unsupported-platform-leaf', }); + // Both capture cells are owner-stated, so neither inherits the network gap's reason. + assert.deepEqual(binding.facts.operations.captureScreenshot, { + available: false, + reason: 'unsupported-device-kind', + }); assert.deepEqual(binding.operations, {}); await binding[Symbol.asyncDispose](); }); diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 5d82e4b4a3..4e34870f6c 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -10,6 +10,7 @@ import type { RuntimeOperationUnavailability, RuntimeOwnerRef, } from './platform-runtime.ts'; +import { screenshotRuntimeOperationFacts } from './screenshot-runtime.ts'; import { snapshotRuntimeOperationFacts } from './snapshot-runtime.ts'; import { viewportRuntimeOperationFacts } from './viewport-runtime.ts'; @@ -24,6 +25,7 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{ appState?: RuntimeOperationUnavailability; network: RuntimeOperationUnavailability; screenRecording?: RuntimeOperationUnavailability; + screenshot: RuntimeOperationUnavailability; snapshot?: RuntimeOperationUnavailability; viewport: RuntimeOperationUnavailability; readiness?: RuntimeOperationUnavailability; @@ -38,6 +40,7 @@ type FrozenUnavailablePlatformRuntimeFacts = Readonly<{ appState: RuntimeOperationUnavailability; network: RuntimeOperationUnavailability; screenRecording: RuntimeOperationUnavailability; + screenshot: RuntimeOperationUnavailability; snapshot: RuntimeOperationUnavailability; viewport: RuntimeOperationUnavailability; readiness: RuntimeOperationUnavailability; @@ -71,6 +74,7 @@ export function createUnavailablePlatformRuntimeFacts( appState, network, screenRecording, + screenshot, snapshot, viewport, readiness, @@ -98,6 +102,7 @@ export function createUnavailablePlatformRuntimeFacts( screenRecordingStart: screenRecording, screenRecordingReattach: screenRecording, screenRecordingCleanup: screenRecording, + ...screenshotRuntimeOperationFacts({ capture: screenshot }), ...snapshotRuntimeOperationFacts({ capture: snapshot, customActions: snapshot, @@ -116,19 +121,23 @@ export function createUnavailablePlatformRuntimeFacts( function freezeUnavailableFacts( unavailable: UnavailablePlatformRuntimeFacts, ): FrozenUnavailablePlatformRuntimeFacts { + // Every optional cell falls back to the caller's network gap: an owner that did not classify a + // family has, by construction, the same reason its transport does. + const orNetwork = (fact: RuntimeOperationUnavailability | undefined) => + Object.freeze({ ...(fact ?? unavailable.network) }); return Object.freeze({ appLog: Object.freeze({ ...unavailable.appLog }), - apps: Object.freeze({ ...(unavailable.apps ?? unavailable.network) }), - appDeployment: Object.freeze({ ...(unavailable.appDeployment ?? unavailable.network) }), - appState: Object.freeze({ ...(unavailable.appState ?? unavailable.network) }), + apps: orNetwork(unavailable.apps), + appDeployment: orNetwork(unavailable.appDeployment), + appState: orNetwork(unavailable.appState), network: Object.freeze({ ...unavailable.network }), - screenRecording: Object.freeze({ - ...(unavailable.screenRecording ?? unavailable.network), - }), - snapshot: Object.freeze({ ...(unavailable.snapshot ?? unavailable.network) }), + screenRecording: orNetwork(unavailable.screenRecording), + // Capture cells are stated by their owner, never inherited from the transport gap (#1873). + screenshot: Object.freeze({ ...unavailable.screenshot }), + snapshot: orNetwork(unavailable.snapshot), viewport: Object.freeze({ ...unavailable.viewport }), - readiness: Object.freeze({ ...(unavailable.readiness ?? unavailable.network) }), - shutdown: Object.freeze({ ...(unavailable.shutdown ?? unavailable.network) }), + readiness: orNetwork(unavailable.readiness), + shutdown: orNetwork(unavailable.shutdown), lifecycle: applicationLifecycleOperationFacts(unavailable.lifecycle), }); } diff --git a/packages/contracts/src/screenshot-runtime.test.ts b/packages/contracts/src/screenshot-runtime.test.ts new file mode 100644 index 0000000000..a4e884ae4a --- /dev/null +++ b/packages/contracts/src/screenshot-runtime.test.ts @@ -0,0 +1,56 @@ +import { expect, test, vi } from 'vitest'; +import type { Interactor } from './interactor-types.ts'; +import { + bindLocalScreenshotInteractor, + bindProviderScreenshotInteractor, + screenshotRuntimeOperationFacts, +} from './screenshot-runtime.ts'; + +const device = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +} as const; + +test('builds the exact screenshot operation fact catalog', () => { + const capture = { available: true } as const; + expect(screenshotRuntimeOperationFacts({ capture })).toEqual({ captureScreenshot: capture }); +}); + +test('a local binding hands the interactor the destination, options, and the request signal', async () => { + const screenshot = vi.fn(async () => {}); + const resolveInteractor = vi.fn(async () => ({ screenshot }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindLocalScreenshotInteractor({ device, signal, resolveInteractor }); + await operations.captureScreenshot({ + outPath: '/tmp/out.png', + options: { appBundleId: 'com.example.app', fullscreen: true }, + execution: { logPath: '/tmp/daemon.log' }, + }); + + expect(resolveInteractor).toHaveBeenCalledWith(device, { + logPath: '/tmp/daemon.log', + appBundleId: 'com.example.app', + signal, + }); + expect(screenshot).toHaveBeenCalledWith('/tmp/out.png', { + appBundleId: 'com.example.app', + fullscreen: true, + }); +}); + +test('a provider binding fails closed when its exact owner exposes no interactor', async () => { + const operations = bindProviderScreenshotInteractor({ + device, + signal: new AbortController().signal, + resolveInteractor: () => undefined, + }); + + await expect(operations.captureScreenshot({ outPath: '/tmp/out.png' })).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'provider-runtime-interactor-missing' }, + }); +}); diff --git a/packages/contracts/src/screenshot-runtime.ts b/packages/contracts/src/screenshot-runtime.ts new file mode 100644 index 0000000000..f99eb680bf --- /dev/null +++ b/packages/contracts/src/screenshot-runtime.ts @@ -0,0 +1,88 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import type { Interactor, RunnerContext, ScreenshotOptions } from './interactor-types.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; + +export type { ScreenshotOptions } from './interactor-types.ts'; + +/** Runner metadata needed by the selected capture implementation, without request-owned state. */ +export type ScreenshotRuntimeExecution = Readonly>; + +/** + * Neutral capture intent. The destination is the caller's already-reserved artifact path — the + * runtime never invents one — and the request binding supplies cancellation and exact-owner + * authority. + */ +export type CaptureScreenshotInput = Readonly<{ + outPath: string; + options?: Readonly; + execution?: ScreenshotRuntimeExecution; +}>; + +export type ScreenshotRuntimeOperations = Readonly<{ + captureScreenshot(input: CaptureScreenshotInput): Promise; +}>; + +export type ScreenshotRuntimeOperationFacts = Readonly<{ + captureScreenshot: RuntimeOperationFact; +}>; + +/** Builds the owner claim for the single capture requirement. */ +export function screenshotRuntimeOperationFacts( + input: Readonly<{ capture: RuntimeOperationFact }>, +): ScreenshotRuntimeOperationFacts { + return Object.freeze({ captureScreenshot: input.capture }); +} + +/** + * Captures one selected owner's interactor authority for the lifetime of a request binding. The + * owner is already chosen by the time a binder is called, so each entry point supplies its own + * resolution and this holds only what both share: the runner context and the capture itself. + */ +function bindScreenshotCapture( + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, +): ScreenshotRuntimeOperations { + return Object.freeze({ + captureScreenshot: async (input: CaptureScreenshotInput) => { + const interactor = await resolveInteractor({ + ...input.execution, + appBundleId: input.options?.appBundleId, + signal, + }); + await interactor.screenshot(input.outPath, input.options); + }, + }); +} + +export function bindLocalScreenshotInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: (device: DeviceInfo, runner: RunnerContext) => Promise; + }>, +): ScreenshotRuntimeOperations { + return bindScreenshotCapture( + params.signal, + async (runner) => await params.resolveInteractor(params.device, runner), + ); +} + +/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */ +export function bindProviderScreenshotInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: (runner: RunnerContext) => Interactor | undefined; + }>, +): ScreenshotRuntimeOperations { + return bindScreenshotCapture(params.signal, async (runner) => { + const interactor = params.resolveInteractor(runner); + if (interactor) return interactor; + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Provider-owned screenshot operation has no bound provider interactor.', + { reason: 'provider-runtime-interactor-missing', deviceId: params.device.id }, + ); + }); +} diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 6756499b58..f34ac21c7e 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -98,6 +98,8 @@ test.each([ expect(facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(facts.operations.setViewport).toMatchObject({ available: false }); expect(binding.operations.setViewport).toBeUndefined(); + expect(facts.operations.captureScreenshot).toEqual({ available: true }); + expect(binding.operations.captureScreenshot).toBeTypeOf('function'); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); await expect(binding.operations.ensureReady?.({})).resolves.toMatchObject({ diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 1fcfdb6ba6..585c2e571c 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -9,8 +9,10 @@ import type { import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, + bindLocalScreenshotInteractor, bindLocalSnapshotInteractor, localRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, } from '@agent-device/contracts/platform'; @@ -68,6 +70,11 @@ const shutdownKindUnavailable = Object.freeze({ reason: 'unsupported-device-kind', hint: 'shutdown is supported only for Apple simulators and Android emulators.', } as const); +const screenshotKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: 'screenshot is supported only for Android emulators and devices.', +} as const); const snapshotKindUnavailable = Object.freeze({ available: false, reason: 'unsupported-device-kind', @@ -137,6 +144,9 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor customActions: snapshotCustomActionsUnavailable, withoutActiveApp: device.kind === 'simulator' ? snapshotKindUnavailable : available, }), + ...screenshotRuntimeOperationFacts({ + capture: device.kind === 'simulator' ? screenshotKindUnavailable : available, + }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ensureReady: available, bootTarget: available, @@ -191,6 +201,13 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor resolveInteractor: host.localInteractors.resolve, }) : {}), + ...(facts.operations.captureScreenshot.available + ? bindLocalScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ensureReady: async (input: EnsureReadyInput) => await ensureAndroidReady( host, diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 63e83e7b0b..84431a2838 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -104,9 +104,23 @@ test.each([ hint: 'viewport resizes web targets only (--platform web). Apple screen geometry is fixed by the selected simulator or device type — open a different simulator to test another screen size.', }); expect(binding.operations.setViewport).toBeUndefined(); + expectAppleCaptureAvailability(binding, device); expectAppleSnapshotAvailability(binding, device); }); +/** + * watchOS is admitted by no capture cell: the Apple interactor cannot even be constructed for it, + * so the refusal is a fact rather than a throw from inside the leaf. + */ +function expectAppleCaptureAvailability( + binding: DeviceBinding, + device: DeviceInfo, +): void { + const available = device.appleOs !== 'watchos'; + expect(binding.facts.operations.captureScreenshot.available).toBe(available); + expect(binding.operations.captureScreenshot).toBeTypeOf(available ? 'function' : 'undefined'); +} + function expectAppleSnapshotAvailability( binding: DeviceBinding, device: DeviceInfo, diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index ee3371dd99..8903e1664c 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -8,7 +8,9 @@ import type { import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, + bindLocalScreenshotInteractor, localRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, } from '@agent-device/contracts/platform'; @@ -101,6 +103,14 @@ const snapshotCustomActionsUnavailable = Object.freeze({ reason: 'unsupported-platform-leaf', hint: 'Re-run without --actions, or target an iOS simulator.', } as const); +const screenshotWatchOsUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'screenshot is not supported on watchOS because XCUITest cannot drive watchOS UI.', +} as const); +const screenshotKindUnavailable = unsupportedAppleDeviceKind( + 'screenshot is supported only for Apple simulators and devices.', +); const snapshotActiveAppRequired = Object.freeze({ available: false, reason: 'owner-capability-missing', @@ -212,6 +222,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR screenRecordingReattach: recordingFacts, screenRecordingCleanup: recordingFacts, ...appleSnapshotFacts(device), + ...screenshotRuntimeOperationFacts({ capture: appleScreenshotFact(device) }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ensureReady: readiness, bootTarget: boot, @@ -257,6 +268,13 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR signal: request.scope.signal, }) : {}), + ...(facts.operations.captureScreenshot.available + ? bindLocalScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ...(facts.operations.ensureReady.available ? { ensureReady: async () => @@ -311,6 +329,18 @@ function appleSnapshotFact(device: DeviceInfo) { : snapshotKindUnavailable; } +/** + * macOS surface selection (app window vs desktop/menubar) lives inside the Apple interactor's own + * capture, so the ordinary local interactor binding covers every admitted Apple cell — unlike + * snapshot, whose desktop surfaces come from a separate host port. + */ +function appleScreenshotFact(device: DeviceInfo) { + if (resolveDeviceAppleOs(device) === 'watchos') return screenshotWatchOsUnavailable; + return device.kind === 'simulator' || device.kind === 'device' + ? available + : screenshotKindUnavailable; +} + function appleSnapshotFacts(device: DeviceInfo) { const capture = appleSnapshotFact(device); return snapshotRuntimeOperationFacts({ diff --git a/packages/platform-harmonyos/src/runtime.test.ts b/packages/platform-harmonyos/src/runtime.test.ts index 517b41141d..12d40fd980 100644 --- a/packages/platform-harmonyos/src/runtime.test.ts +++ b/packages/platform-harmonyos/src/runtime.test.ts @@ -70,6 +70,8 @@ test.each([ expect(facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(facts.operations.setViewport).toMatchObject({ available: false }); expect(binding.operations.setViewport).toBeUndefined(); + expect(facts.operations.captureScreenshot).toEqual({ available: true }); + expect(binding.operations.captureScreenshot).toBeTypeOf('function'); await expect(binding.operations.ensureReady?.({})).resolves.toMatchObject({ booted: true }); await expect( binding.operations.listApps?.({ device: runtimeDevice, filter: 'all' }), diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index ac1e002dbb..dafbdabafb 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -7,8 +7,10 @@ import type { import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, + bindLocalScreenshotInteractor, bindLocalSnapshotInteractor, localRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, } from '@agent-device/contracts/platform'; @@ -63,6 +65,11 @@ const snapshotKindUnavailable = Object.freeze({ reason: 'unsupported-device-kind', hint: 'snapshot is supported only for HarmonyOS emulators and devices.', } as const); +const screenshotKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: 'screenshot is supported only for HarmonyOS emulators and devices.', +} as const); const snapshotCustomActionsUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', @@ -129,6 +136,12 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor ? available : snapshotKindUnavailable, }), + ...screenshotRuntimeOperationFacts({ + capture: + device.kind === 'emulator' || device.kind === 'device' + ? available + : screenshotKindUnavailable, + }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ensureReady: available, bootTarget: unavailable, @@ -184,6 +197,13 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor resolveInteractor: host.localInteractors.resolve, }) : {}), + ...(facts.operations.captureScreenshot.available + ? bindLocalScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), listApps: async (input: { device: DeviceInfo; filter: 'all' | 'user-installed' }) => await host.appInventory.harmonyos.listApps( input.device, diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index 3026144f89..56700ea5bf 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -106,6 +106,10 @@ test.each([ ); expect(binding.facts.operations.setViewport).toMatchObject({ available: false }); expect(binding.operations.setViewport).toBeUndefined(); + expect(binding.facts.operations.captureScreenshot.available).toBe(device.kind === 'device'); + expect(binding.operations.captureScreenshot).toBeTypeOf( + device.kind === 'device' ? 'function' : 'undefined', + ); expect(binding.operations.captureSnapshot).toBeTypeOf( device.kind === 'device' ? 'function' : 'undefined', ); diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 5e2f2e86d4..10c95f1c9d 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -10,9 +10,11 @@ import type { import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, + bindLocalScreenshotInteractor, createUnavailablePlatformRuntimeFacts, localRuntimeOwner, sameRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, } from '@agent-device/contracts/platform'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -46,6 +48,10 @@ const snapshotKindUnavailable = unavailableLinuxRuntimeFact( 'unsupported-device-kind', 'snapshot is supported only for the Linux desktop device.', ); +const screenshotKindUnavailable = unavailableLinuxRuntimeFact( + 'unsupported-device-kind', + 'screenshot is supported only for the Linux desktop device.', +); const snapshotCustomActionsUnavailable = unavailableLinuxRuntimeFact( 'unsupported-platform-leaf', 'Re-run without --actions, or target an iOS simulator.', @@ -80,6 +86,13 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR ...(facts.operations.captureSnapshot.available ? linuxSnapshotOperations(host, request) : {}), + ...(facts.operations.captureScreenshot.available + ? bindLocalScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), }), [Symbol.asyncDispose]: async () => undefined, }) satisfies DeviceBinding; @@ -94,6 +107,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts const unavailable = createUnavailablePlatformRuntimeFacts(device, linuxOwner, { appLog: unsupportedPlatformLeaf, network: unsupportedPlatformLeaf, + screenshot: screenshotKindUnavailable, snapshot: snapshotKindUnavailable, viewport: unsupportedPlatformLeaf, readiness: unsupportedPlatformLeaf, @@ -118,6 +132,9 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts customActions: snapshotCustomActionsUnavailable, withoutActiveApp: device.kind === 'device' ? supported : snapshotKindUnavailable, }), + ...screenshotRuntimeOperationFacts({ + capture: device.kind === 'device' ? supported : screenshotKindUnavailable, + }), }, }); } diff --git a/packages/platform-vega/src/runtime.test.ts b/packages/platform-vega/src/runtime.test.ts index 5fe9c6036a..8859b3a7e6 100644 --- a/packages/platform-vega/src/runtime.test.ts +++ b/packages/platform-vega/src/runtime.test.ts @@ -136,6 +136,11 @@ test.each([ }); expect(binding.operations.captureSnapshot).toBeUndefined(); expect(binding.facts.operations.setViewport).toMatchObject({ available: false }); + expect(binding.facts.operations.captureScreenshot).toMatchObject({ + available: false, + hint: 'screenshot is not supported on Vega OS: the Vega runtime exposes remote navigation only.', + }); + expect(binding.operations.captureScreenshot).toBeUndefined(); expect(binding.operations.setViewport).toBeUndefined(); expectLifecycleFacts(binding, legacy); }, diff --git a/packages/platform-vega/src/runtime.ts b/packages/platform-vega/src/runtime.ts index 185a08772b..ebd37d6ceb 100644 --- a/packages/platform-vega/src/runtime.ts +++ b/packages/platform-vega/src/runtime.ts @@ -75,6 +75,11 @@ export function createVegaPlatformRuntime(host: PlatformRuntimeHost): PlatformRu }); } +const screenshotUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + 'screenshot is not supported on Vega OS: the Vega runtime exposes remote navigation only.', +); + function vegaFacts(device: DeviceInfo): RuntimeFacts { const supported = device.kind === 'emulator' && device.target === 'tv'; const openTarget = supported ? lifecycleAvailable : openTargetUnavailable; @@ -82,6 +87,7 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts return createUnavailablePlatformRuntimeFacts(device, vegaOwner, { appLog: unsupportedPlatformLeaf, network: unsupportedPlatformLeaf, + screenshot: screenshotUnavailable, snapshot: unsupportedPlatformLeaf, viewport: unsupportedPlatformLeaf, readiness: unsupportedPlatformLeaf, diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index e2164db396..9b9c4f6014 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -52,6 +52,8 @@ test('preserves a narrow web provider dump including empty successful entries', expect(binding.facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(binding.facts.operations.setViewport).toEqual({ available: true }); expect(binding.operations.setViewport).toBeTypeOf('function'); + expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); + expect(binding.operations.captureScreenshot).toBeTypeOf('function'); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); expectLifecycleFacts(binding); }); @@ -137,6 +139,8 @@ test.each([ expect(binding.facts.operations.captureSnapshot.available).toBe(false); expect(binding.facts.operations.setViewport.available).toBe(false); expect(binding.operations.setViewport).toBeUndefined(); + expect(binding.facts.operations.captureScreenshot.available).toBe(false); + expect(binding.operations.captureScreenshot).toBeUndefined(); expect(binding.operations.captureSnapshot).toBeUndefined(); }, ); diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index 2777a04439..5e301ffebb 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -8,8 +8,10 @@ import type { import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, + bindLocalScreenshotInteractor, bindLocalSnapshotInteractor, localRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, sameRuntimeOwner, viewportRuntimeOperationFacts, @@ -168,6 +170,13 @@ function bindWebRuntime( resolveInteractor: host.localInteractors.resolve, }) : {}), + ...(facts.operations.captureScreenshot.available + ? bindLocalScreenshotInteractor({ + device, + signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ...(facts.operations.setViewport.available ? { setViewport: async (input) => { @@ -208,6 +217,8 @@ function webRuntimeFacts( reason: 'owner-capability-missing', hint: 'network is not supported by this web provider', } as const); + // One browser-device cell, read by every operation this runtime binds through the interactor. + const browserDevice = device.kind === 'device' ? available : openTargetKindUnavailable; return Object.freeze({ device: { family: 'web', @@ -227,13 +238,12 @@ function webRuntimeFacts( screenRecordingReattach: recordingAvailable ? available : recordingUnavailable, screenRecordingCleanup: recordingAvailable ? available : recordingUnavailable, ...snapshotRuntimeOperationFacts({ - capture: device.kind === 'device' ? available : openTargetKindUnavailable, + capture: browserDevice, customActions: snapshotCustomActionsUnavailable, - withoutActiveApp: device.kind === 'device' ? available : openTargetKindUnavailable, - }), - ...viewportRuntimeOperationFacts({ - setViewport: device.kind === 'device' ? available : openTargetKindUnavailable, + withoutActiveApp: browserDevice, }), + ...screenshotRuntimeOperationFacts({ capture: browserDevice }), + ...viewportRuntimeOperationFacts({ setViewport: browserDevice }), ensureReady: readinessUnavailable, bootTarget: readinessUnavailable, bootTargetHeadless: readinessUnavailable, diff --git a/packages/provider-limrun/src/app-log-runtime.test.ts b/packages/provider-limrun/src/app-log-runtime.test.ts index 103c4f10d0..c372c1c91d 100644 --- a/packages/provider-limrun/src/app-log-runtime.test.ts +++ b/packages/provider-limrun/src/app-log-runtime.test.ts @@ -281,6 +281,8 @@ test.each([ expect(binding.facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(binding.facts.operations.setViewport).toMatchObject({ available: false }); expect(binding.operations.setViewport).toBeUndefined(); + expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); + expect(binding.operations.captureScreenshot).toBeTypeOf('function'); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); await expect( binding.operations.listApps?.({ device: runtimeDevice, filter: 'all' }), @@ -339,6 +341,7 @@ test('fails closed for a stale Android identity before exposing facts or binding expect(facts.operations.captureSnapshotWithCustomActions).toMatchObject({ available: false }); expect(facts.operations.captureSnapshotWithoutActiveApp).toMatchObject({ available: false }); expect(facts.operations.setViewport).toMatchObject({ available: false }); + expect(facts.operations.captureScreenshot).toMatchObject({ available: false }); await expect( owner.bind({ device: staleDevice, intent: { kind: 'ordinary' }, scope }), ).rejects.toMatchObject({ diff --git a/packages/provider-limrun/src/app-log-runtime.ts b/packages/provider-limrun/src/app-log-runtime.ts index 7f4eb56d29..76d8cb01be 100644 --- a/packages/provider-limrun/src/app-log-runtime.ts +++ b/packages/provider-limrun/src/app-log-runtime.ts @@ -21,10 +21,12 @@ import { import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, + bindProviderScreenshotInteractor, bindProviderSnapshotInteractor, createUnavailablePlatformRuntimeFacts, providerRuntimeOwner, sameRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, } from '@agent-device/contracts/platform'; @@ -187,6 +189,7 @@ export function createLimrunPlatformRuntimeOwner( appState: liveSessionUnavailable, appDeployment: liveSessionUnavailable, network: liveSessionUnavailable, + screenshot: liveSessionUnavailable, viewport: liveSessionUnavailable, readiness: liveSessionUnavailable, shutdown: liveSessionUnavailable, @@ -353,6 +356,11 @@ function bindLimrunAppLogs( signal, resolveInteractor: (runner) => options.getInteractor(device, runner), }), + ...bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor: (runner) => options.getInteractor(device, runner), + }), ...createLimrunAppDeploymentOperations( deploymentOptions(options), device, @@ -432,6 +440,7 @@ function facts( : customSnapshotUnavailable, withoutActiveApp: available, }), + ...screenshotRuntimeOperationFacts({ capture: available }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ensureReady: available, bootTarget: available, @@ -471,6 +480,7 @@ function recoveryFacts( customActions: liveSessionUnavailable, withoutActiveApp: liveSessionUnavailable, }), + ...screenshotRuntimeOperationFacts({ capture: liveSessionUnavailable }), ...viewportRuntimeOperationFacts({ setViewport: liveSessionUnavailable }), ensureReady: liveSessionUnavailable, bootTarget: liveSessionUnavailable, diff --git a/packages/provider-webdriver/src/platform-runtime.test.ts b/packages/provider-webdriver/src/platform-runtime.test.ts index 85d2d56f6d..c14d401e33 100644 --- a/packages/provider-webdriver/src/platform-runtime.test.ts +++ b/packages/provider-webdriver/src/platform-runtime.test.ts @@ -237,6 +237,8 @@ test('captures through only the active exact WebDriver interactor', async () => expect(binding.facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(binding.facts.operations.setViewport).toMatchObject({ available: false }); expect(binding.operations.setViewport).toBeUndefined(); + expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); + expect(binding.operations.captureScreenshot).toBeTypeOf('function'); await expect( binding.operations.captureSnapshot?.({ options: { interactiveOnly: true } }), ).resolves.toEqual({ backend: 'android', nodes: [] }); @@ -248,9 +250,15 @@ test('captures through only the active exact WebDriver interactor', async () => }); test.each([ - ['inactive session', { isSessionActive: () => false, snapshotAvailable: true }], - ['unsupported capability', { isSessionActive: () => true, snapshotAvailable: false }], -] as const)('fails closed for an %s snapshot owner cell', async (_name, state) => { + [ + 'inactive session', + { isSessionActive: () => false, snapshotAvailable: true, screenshotAvailable: true }, + ], + [ + 'unsupported capability', + { isSessionActive: () => true, snapshotAvailable: false, screenshotAvailable: false }, + ], +] as const)('fails closed for an %s capture owner cell', async (_name, state) => { const getInteractor = vi.fn(() => ({}) as unknown as Interactor); const owner = createWebDriverPlatformRuntimeOwner({ host: host(async () => ({ stdout: '', stderr: '', exitCode: 0 })), @@ -264,6 +272,7 @@ test.each([ expect(facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(facts.operations.captureSnapshotWithoutActiveApp.available).toBe(false); expect(facts.operations.setViewport.available).toBe(false); + expect(facts.operations.captureScreenshot.available).toBe(false); if (state.isSessionActive()) { const binding = await owner.bind({ device, diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index a09b64154f..813b04ff18 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -1,9 +1,11 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, + bindProviderScreenshotInteractor, bindProviderSnapshotInteractor, createUnavailablePlatformRuntimeFacts, sameRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, type AppDeploymentInput, @@ -69,6 +71,11 @@ const snapshotUnavailable = Object.freeze({ reason: 'unsupported-provider-mode', hint: 'This WebDriver provider runtime does not expose snapshot capture for this device.', } as const); +const screenshotUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'This WebDriver provider runtime does not expose screenshot capture for this device.', +} as const); const snapshotCustomActionsUnavailable = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -132,16 +139,20 @@ function webDriverLifecycleFacts(device: DeviceInfo) { configureProviderPortReverse: portReverseUnavailable, }); } +/** How this provider instance was configured: identity, session liveness, and declared capture. */ +export type WebDriverPlatformRuntimeOptions = Readonly<{ + host: PlatformRuntimeHost; + owner: Extract; + ownsDevice(device: DeviceInfo): boolean; + isSessionActive?(device: DeviceInfo): boolean; + deployment?: WebDriverPlatformDeploymentRuntime; + screenshotAvailable?: boolean; + snapshotAvailable?: boolean; + getInteractor?(device: DeviceInfo, runner?: RunnerContext): Interactor | undefined; +}>; + export function createWebDriverPlatformRuntimeOwner( - options: Readonly<{ - host: PlatformRuntimeHost; - owner: Extract; - ownsDevice(device: DeviceInfo): boolean; - isSessionActive?(device: DeviceInfo): boolean; - deployment?: WebDriverPlatformDeploymentRuntime; - snapshotAvailable?: boolean; - getInteractor?(device: DeviceInfo, runner?: RunnerContext): Interactor | undefined; - }>, + options: WebDriverPlatformRuntimeOptions, ): PlatformRuntimeOwner { return Object.freeze({ owner: options.owner, @@ -173,15 +184,7 @@ export function createWebDriverPlatformRuntimeOwner( } function bindWebDriverPlatformRuntime( - options: Readonly<{ - host: PlatformRuntimeHost; - owner: Extract; - ownsDevice(device: DeviceInfo): boolean; - isSessionActive?(device: DeviceInfo): boolean; - deployment?: WebDriverPlatformDeploymentRuntime; - snapshotAvailable?: boolean; - getInteractor?(device: DeviceInfo, runner?: RunnerContext): Interactor | undefined; - }>, + options: WebDriverPlatformRuntimeOptions, device: DeviceInfo, signal: AbortSignal, ): DeviceBinding { @@ -206,6 +209,13 @@ function bindWebDriverPlatformRuntime( resolveInteractor: (runner) => options.getInteractor?.(device, runner), }) : {}), + ...(facts.operations.captureScreenshot.available + ? bindProviderScreenshotInteractor({ + device, + signal, + resolveInteractor: (runner) => options.getInteractor?.(device, runner), + }) + : {}), networkDump: async (input) => { const recent = await options.host.appLogs.readRecent(input.sessionId, input.maxScanLines); const dump = readRecentNetworkTrafficFromText(recent.text, { @@ -247,14 +257,7 @@ function bindWebDriverPlatformRuntime( } function webDriverFacts( - options: Readonly<{ - owner: Extract; - ownsDevice(device: DeviceInfo): boolean; - isSessionActive?(device: DeviceInfo): boolean; - deployment?: WebDriverPlatformDeploymentRuntime; - snapshotAvailable?: boolean; - getInteractor?(device: DeviceInfo, runner?: RunnerContext): Interactor | undefined; - }>, + options: Omit, device: DeviceInfo, ): RuntimeFacts { if (!webDriverSessionActive(options, device)) { @@ -263,6 +266,7 @@ function webDriverFacts( appDeployment: inactiveSession, network: inactiveSession, screenRecording: inactiveSession, + screenshot: inactiveSession, viewport: inactiveSession, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: inactiveSession, @@ -283,9 +287,17 @@ function webDriverFacts( appDeployment: deploymentUnavailable, network: appLogUnavailable, screenRecording: recordingUnavailable, + screenshot: screenshotUnavailable, viewport: viewportUnavailable, lifecycle: webDriverLifecycleFacts(device), }); + // Both capture cells need the same reachability: an interactor this provider can drive, on a + // device shape it supports. Each then adds its own declared-capability gate. + const reachable = options.getInteractor !== undefined && webDriverInteractorDevice(device); + const snapshotCell = + reachable && options.snapshotAvailable !== false ? available : snapshotUnavailable; + const screenshotCell = + reachable && options.screenshotAvailable !== false ? available : screenshotUnavailable; return Object.freeze({ device: unavailable.device, operations: { @@ -297,20 +309,11 @@ function webDriverFacts( appState: appStateUnavailable, networkDump: available, ...snapshotRuntimeOperationFacts({ - capture: - options.snapshotAvailable !== false && - options.getInteractor !== undefined && - webDriverSnapshotDevice(device) - ? available - : snapshotUnavailable, + capture: snapshotCell, customActions: snapshotCustomActionsUnavailable, - withoutActiveApp: - options.snapshotAvailable !== false && - options.getInteractor !== undefined && - webDriverSnapshotDevice(device) - ? available - : snapshotUnavailable, + withoutActiveApp: snapshotCell, }), + ...screenshotRuntimeOperationFacts({ capture: screenshotCell }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ensureReady: available, bootTarget: available, @@ -325,7 +328,8 @@ function webDriverFacts( }); } -function webDriverSnapshotDevice(device: DeviceInfo): boolean { +/** The device shapes this provider can reach at all through its own WebDriver interactor. */ +function webDriverInteractorDevice(device: DeviceInfo): boolean { return ( device.kind === 'device' && device.target === 'mobile' && diff --git a/packages/provider-webdriver/src/runtime.ts b/packages/provider-webdriver/src/runtime.ts index 0827199ad8..34d3d7a56e 100644 --- a/packages/provider-webdriver/src/runtime.ts +++ b/packages/provider-webdriver/src/runtime.ts @@ -166,6 +166,7 @@ class CloudWebDriverRuntimeImplementation implements CloudWebDriverRuntime { isSessionActive: (device) => this.sessions.findSessionForDevice(device) !== undefined, deployment: this.deployment, snapshotAvailable: this.capabilities.operations.snapshot.support !== 'unsupported', + screenshotAvailable: this.capabilities.operations.screenshot.support !== 'unsupported', getInteractor: (device) => this.getInteractor(device), }); } diff --git a/scripts/layering/daemon-modularity.test.ts b/scripts/layering/daemon-modularity.test.ts index ae8afd1e6d..843f588e13 100644 --- a/scripts/layering/daemon-modularity.test.ts +++ b/scripts/layering/daemon-modularity.test.ts @@ -73,8 +73,8 @@ test('daemon modularity baseline records the measured R7 ownership pressure', () Object.values(SESSION_STATE_FIELD_OWNERS).reduce((sum, owners) => sum + owners.length, 0), DAEMON_MODULARITY_BASELINE.sessionState.ownerFileClaims, ); - assert.equal(TYPE_CYCLE_BASELINE, 46); - assert.equal(DAEMON_MODULARITY_BASELINE.largestTypeCycle.zoneMembers['daemon-server'], 16); + assert.equal(TYPE_CYCLE_BASELINE, 26); + assert.equal(DAEMON_MODULARITY_BASELINE.largestTypeCycle.zoneMembers['daemon-server'], 14); assert.equal('daemon' in DAEMON_MODULARITY_BASELINE.largestTypeCycle.zoneMembers, false); }); @@ -202,19 +202,20 @@ test('internal trees reject deep imports globally, including from daemon', () => test('R9 records zone ceilings and keeps engine files outside the largest component', () => { // One commands file and one engine file traded for two daemon-server ones, so the total - // stays at the baseline and only the per-zone claims are on trial. + // stays at the baseline and only the per-zone claims are on trial. `commands` left the cycle + // entirely when screenshot stopped threading through generic dispatch, so its ceiling is 0. const zones = DAEMON_MODULARITY_BASELINE.largestTypeCycle.zoneMembers; const violations = checkDaemonModularityRatchets( baselineEdges(), baselineTypeCycleMembers({ - commands: zones.commands + 1, + commands: 1, 'ad-replay': 1, 'daemon-server': zones['daemon-server']! - 2, }), ); assert.equal(violations.length, 3); - assert.ok(violations.some(({ message }) => /contains 15 commands file/.test(message))); + assert.ok(violations.some(({ message }) => /contains 1 commands file/.test(message))); assert.ok(violations.some(({ message }) => /contains 1 ad-replay file/.test(message))); assert.ok(violations.some(({ message }) => /engine file entered/.test(message))); }); @@ -226,7 +227,9 @@ test('R10 zone overflow lists the whole zone so the joining member is visible', const zones = DAEMON_MODULARITY_BASELINE.largestTypeCycle.zoneMembers; // Sorts after the daemon-server probes: the old first-member pick could not name it by luck. const joined = 'src/daemon/snapshot-interactor-capture.ts'; - const members = [...baselineTypeCycleMembers({ commands: zones.commands - 1 }), joined].sort(); + // `commands` left the cycle with the screenshot cutover, so the offsetting removal comes from + // `core` instead: the total stays at the baseline and only the daemon-server claim is on trial. + const members = [...baselineTypeCycleMembers({ core: zones.core! - 1 }), joined].sort(); const daemonMembers = members.filter((member) => member.startsWith('src/daemon/')); assert.notEqual(daemonMembers[0], joined); @@ -236,7 +239,7 @@ test('R10 zone overflow lists the whole zone so the joining member is visible', const [violation] = violations; assert.equal(violation!.rule, 'R10 daemon-modularity'); assert.equal(violation!.file, 'scripts/layering/daemon-modularity.ts'); - assert.match(violation!.message, /contains 17 daemon-server file\(s\) \(baseline 16\)/); + assert.match(violation!.message, /contains 15 daemon-server file\(s\) \(baseline 14\)/); for (const member of daemonMembers) { assert.ok(violation!.message.includes(member), `${member} missing from: ${violation!.message}`); } @@ -254,6 +257,6 @@ test('R9 rejects a baseline left above the measured cycle', () => { assert.equal(violations.length, 1); assert.match(violations[0]!.rule, /^R9 /); - assert.match(violations[0]!.message, /dropped to 45 files \(baseline 46\)/); + assert.match(violations[0]!.message, /dropped to 25 files \(baseline 26\)/); assert.match(violations[0]!.message, /Lower LARGEST_TYPE_CYCLE_ZONE_CEILINGS by the same 1/); }); diff --git a/scripts/layering/daemon-modularity.ts b/scripts/layering/daemon-modularity.ts index 7d908f4978..75202897eb 100644 --- a/scripts/layering/daemon-modularity.ts +++ b/scripts/layering/daemon-modularity.ts @@ -3,11 +3,9 @@ import { targetDagZone, type LayeringViolation, type ResolvedImportEdge } from ' import { SESSION_STATE_FIELD_OWNERS } from './session-state.ts'; const LARGEST_TYPE_CYCLE_ZONE_CEILINGS: Readonly> = { - '(root)': 3, - client: 1, - commands: 14, - core: 10, - 'daemon-server': 16, + '(root)': 2, + core: 8, + 'daemon-server': 14, platforms: 2, }; diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index ed4ad789d0..c856a822da 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -523,6 +523,30 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ operationOwners: { setViewport: ['resolveBoundViewportRuntime'] }, }, }, + { + rule: 'R39 screenshot-runtime-cutover', + command: 'screenshot', + subject: 'screen capture', + tier: 'request-scoped', + execution: 'device-runtime', + legacyRetirement: { + // The command leaf, the daemon adapter that re-entered it, and the evidence capture that + // dispatched it directly. `captureSnapshot` is shared with the snapshot unit and stays. + routeNames: [ + 'handleScreenshotCommand', + 'dispatchScreenshotViaRuntime', + 'executeScreenshotPlatformCommand', + 'resolveScreenshotOutputPlacement', + ], + }, + runtimeTypeNames: ['ScreenshotRuntimeOperations'], + operations: { names: ['captureScreenshot'] }, + singularExecution: { + routes: ['dispatchGenericCommand'], + operations: ['captureScreenshot'], + operationOwners: { captureScreenshot: ['selectScreenshotCapture'] }, + }, + }, ]; function snapshotRetiredDispatchProjectionProof( diff --git a/src/__tests__/test-file-size-ratchet.test.ts b/src/__tests__/test-file-size-ratchet.test.ts index 81cb97ea9d..9ef6efdfea 100644 --- a/src/__tests__/test-file-size-ratchet.test.ts +++ b/src/__tests__/test-file-size-ratchet.test.ts @@ -34,7 +34,7 @@ const TRIPWIRE_LINES = 1_000; // Exact current lengths. Lower a pin when its file shrinks; never raise one — extract instead. const PINNED_TEST_FILE_LINES: Readonly> = Object.freeze({ 'src/__tests__/remote-connection.test.ts': 2973, - 'src/daemon/handlers/__tests__/snapshot-handler.test.ts': 2654, + 'src/daemon/handlers/__tests__/snapshot-handler.test.ts': 2640, 'src/commands/interaction/runtime/settle.test.ts': 2361, 'src/platforms/apple/core/__tests__/runner-session.test.ts': 2083, 'src/daemon/handlers/__tests__/session-replay-runtime-maestro.test.ts': 2031, diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index a44da7437b..8f2b6f3f55 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -1,5 +1,6 @@ import { applicationLifecycleOperationFacts, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, type RuntimeOperationFact, } from '@agent-device/contracts/platform'; @@ -30,6 +31,7 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre withoutActiveApp: unavailable, }), ...unavailableShutdownOperationFacts, + ...screenshotRuntimeOperationFacts({ capture: unavailable }), setViewport: unavailable, }); diff --git a/src/commands/__tests__/command-explain.test.ts b/src/commands/__tests__/command-explain.test.ts index fb86555d55..43706f9c82 100644 --- a/src/commands/__tests__/command-explain.test.ts +++ b/src/commands/__tests__/command-explain.test.ts @@ -149,7 +149,8 @@ describe('explainCommand table-driven coverage', () => { test.each([ ['press', ['src/commands/interaction/index.ts', 'src/daemon/handlers/interaction.ts']], ['apps', ['src/commands/management/app.ts']], - ['screenshot', ['src/commands/capture/screenshot.ts', 'src/core/dispatch.ts']], + // R39: screenshot has no dispatch projection left, so its platform work is named directly. + ['screenshot', ['src/commands/capture/screenshot.ts', 'src/daemon/screenshot-runtime.ts']], ['react-native', ['src/daemon/handlers/react-native.ts']], ['record', ['src/daemon/handlers/record-trace.ts']], ['trace', ['src/daemon/handlers/record-trace.ts']], diff --git a/src/core/__tests__/capability-plugin-routing-parity.test.ts b/src/core/__tests__/capability-plugin-routing-parity.test.ts index b25b2ac3d4..1c6983fc01 100644 --- a/src/core/__tests__/capability-plugin-routing-parity.test.ts +++ b/src/core/__tests__/capability-plugin-routing-parity.test.ts @@ -277,7 +277,6 @@ test('HarmonyOS static capabilities omit runtime-backed command admissions', () 'longpress', 'perf', 'press', - 'screenshot', 'scroll', 'settings', 'swipe', diff --git a/src/core/__tests__/dispatch-screenshot.test.ts b/src/core/__tests__/dispatch-screenshot.test.ts new file mode 100644 index 0000000000..fc5ca7829b --- /dev/null +++ b/src/core/__tests__/dispatch-screenshot.test.ts @@ -0,0 +1,42 @@ +import { expect, test, vi } from 'vitest'; +import { dispatchCommand } from '../dispatch.ts'; +import { withWebProvider, type WebProvider } from '../../platforms/web/provider.ts'; + +const webDevice = { + id: 'web', + name: 'Web', + platform: 'web', + kind: 'device', + booted: true, +} as const; + +test('legacy dispatch no longer reaches an interactor screenshot operation', async () => { + const screenshot = vi.fn(async () => undefined); + + await expect( + withWebProvider( + makeWebProvider({ screenshot }), + async () => await dispatchCommand(webDevice, 'screenshot', ['/tmp/out.png']), + ), + ).rejects.toMatchObject({ + code: 'INVALID_ARGS', + message: 'Unknown command: screenshot', + }); + + expect(screenshot).not.toHaveBeenCalled(); +}); + +function makeWebProvider(overrides: Partial = {}): WebProvider { + return { + open: async () => {}, + close: async () => {}, + snapshot: async () => ({ nodes: [] }), + screenshot: async () => {}, + setViewport: async () => {}, + click: async () => {}, + fill: async () => {}, + typeText: async () => {}, + scroll: async () => {}, + ...overrides, + }; +} diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 03b88fee6c..438bee70ef 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -51,14 +51,13 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'is', 'longpress', 'press', - 'screenshot', 'scroll', 'settings', 'swipe', 'type', 'wait', ]); -const WEB_QUERY_COMMANDS = ['audio', 'find', 'get', 'is', 'screenshot', 'wait'] as const; +const WEB_QUERY_COMMANDS = ['audio', 'find', 'get', 'is', 'wait'] as const; const WEB_INTERACTION_COMMANDS = [ 'click', 'fill', diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index c3cb7f778c..f634977331 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -68,6 +68,7 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.reinstall, PUBLIC_COMMANDS.replay, PUBLIC_COMMANDS.shutdown, + PUBLIC_COMMANDS.screenshot, PUBLIC_COMMANDS.snapshot, PUBLIC_COMMANDS.test, PUBLIC_COMMANDS.trace, @@ -202,6 +203,7 @@ test('platform dispatch command list is built from descriptor dispatch facets', test('generic route commands that reach platform dispatch declare the dispatch facet', () => { const nonDispatchGenericCommands = new Set([ PUBLIC_COMMANDS.gesture, + PUBLIC_COMMANDS.screenshot, PUBLIC_COMMANDS.viewport, ]); diff --git a/src/core/command-descriptor/__tests__/screenshot-runtime-execution.test.ts b/src/core/command-descriptor/__tests__/screenshot-runtime-execution.test.ts new file mode 100644 index 0000000000..98d5b9f2fa --- /dev/null +++ b/src/core/command-descriptor/__tests__/screenshot-runtime-execution.test.ts @@ -0,0 +1,18 @@ +import { expect, test } from 'vitest'; +import { screenshotRuntimePlanUses } from '@agent-device/contracts/platform'; +import { commandDescriptors } from '../registry.ts'; + +test('screenshot descriptor declares its complete runtime uses with no legacy projection', () => { + const screenshot = commandDescriptors.find(({ name }) => name === 'screenshot'); + + expect(screenshot).not.toHaveProperty('capability'); + expect(screenshot).not.toHaveProperty('dispatch'); + expect(screenshot?.platformExecution).toEqual({ + kind: 'device-runtime', + uses: screenshotRuntimePlanUses, + }); + expect(screenshotRuntimePlanUses).toEqual([ + { required: ['captureScreenshot'], preferred: [] }, + { required: ['captureScreenshot', 'captureSnapshot'], preferred: [] }, + ]); +}); diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index c969796af7..24c618fa3b 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -30,6 +30,7 @@ import { prepareAppleRunnerRuntimeUse, runtimeCommandRuntimePlanUses, screenRecordingRuntimePlanUses, + screenshotRuntimePlanUses, shutdownTargetUse, viewportRuntimeUse, } from '@agent-device/contracts/platform'; @@ -1351,17 +1352,22 @@ export const RAW_COMMAND_DESCRIPTORS = [ { name: 'screenshot', deviceClaimPolicy: 'require-owner', - ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/capture/screenshot.ts'] as const } : {}), + ...(ownerFilesEnabled + ? { + ownerFiles: [ + 'src/commands/capture/screenshot.ts', + 'src/daemon/screenshot-runtime.ts', + ] as const, + } + : {}), catalog: { group: 'public' }, frameworkTier: 'core', recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'generic', refFrameEffect: 'preserve' }, - dispatch: {}, - capability: ALL_DEVICE_COMMAND_CAPABILITY, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: screenshotRuntimePlanUses }, }, { name: 'viewport', @@ -1714,6 +1720,18 @@ export function commandSupportsVerifyEvidence(command: string | undefined): bool return resolveCommandPostActionObservationSupport(command) === 'settle-and-verify'; } +/** + * Whether a command's platform behavior comes from a request-bound device runtime (ADR 0019). + * Admission for those commands is the owner's exact operation facts, so a route must never also + * consult a capability bucket for them — and a migrated command has no bucket to consult. Reading + * the discriminator here rather than naming commands at each route means the next unit's + * descriptor flip is the whole change. + */ +export function commandUsesDeviceRuntimeExecution(command: string | undefined): boolean { + if (command === undefined) return false; + return COMMAND_DESCRIPTOR_BY_NAME.get(command)?.platformExecution.kind === 'device-runtime'; +} + /** * The declared timeout policy for a command (ADR 0008). Command names outside * the registry (internal probes, unknown commands) fall back to diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index b3748cb6b9..4d4345e4e1 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -1,12 +1,9 @@ -import { screenshotOptionsFromFlags } from '@agent-device/contracts/capture'; import { parseDeviceRotation } from '@agent-device/contracts/device'; import type { GesturePlan, Interactor, RunnerContext } from '@agent-device/contracts/interaction'; import { parseTvRemoteButton } from '@agent-device/contracts/interaction'; import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { Rect } from '@agent-device/kernel/snapshot'; -import { promises as fs } from 'node:fs'; -import pathModule from 'node:path'; import { emitDiagnostic, withDiagnosticTimer } from '../utils/diagnostics.ts'; import { isKeyboardAction, type KeyboardAction } from '../utils/keyboard-actions.ts'; import { readLocationCoordinate } from '../utils/location-coordinates.ts'; @@ -158,8 +155,6 @@ const DISPATCH_HANDLERS: Record = { handleScrollCommand(interactor, positionals, context), 'trigger-app-event': ({ device, interactor, positionals, context }) => handleTriggerAppEventCommand(device, interactor, positionals, context), - screenshot: ({ interactor, positionals, outPath, context }) => - handleScreenshotCommand(interactor, positionals, outPath, context), back: async ({ interactor, context }) => { await interactor.back(context?.backMode); return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; @@ -237,29 +232,6 @@ async function handleTriggerAppEventCommand( }; } -async function handleScreenshotCommand( - interactor: Interactor, - positionals: string[], - outPath: string | undefined, - context: DispatchContext | undefined, -): Promise> { - const positionalPath = positionals[0]; - const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; - await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); - const screenshotOptions = screenshotOptionsFromFlags(context); - await interactor.screenshot(screenshotPath, { - appBundleId: context?.appBundleId, - pixelDensity: screenshotOptions.pixelDensity, - fullscreen: screenshotOptions.fullscreen, - normalizeStatusBar: screenshotOptions.normalizeStatusBar, - stabilize: screenshotOptions.stabilize, - surface: context?.surface, - skipIosSimulatorBootCheck: context?.skipIosSimulatorBootCheck, - captureBackend: context?.screenshotCaptureBackend, - }); - return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; -} - async function handleClipboardCommand( interactor: Interactor, positionals: string[], diff --git a/src/daemon/__tests__/request-router-screenshot.test.ts b/src/daemon/__tests__/request-router-screenshot.test.ts index 75abe25315..26699bde40 100644 --- a/src/daemon/__tests__/request-router-screenshot.test.ts +++ b/src/daemon/__tests__/request-router-screenshot.test.ts @@ -5,16 +5,13 @@ import os from 'node:os'; import path from 'node:path'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; +// `click` and `scroll` still execute through legacy platform dispatch; `screenshot` does not, and +// binds its fake at the facts/bind seam below instead (ADR 0019). vi.mock('../../core/dispatch.ts', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, dispatchCommand: vi.fn(async () => ({})) }; }); -vi.mock('../handlers/snapshot-interactor-capture.ts', async () => { - const fixture = await import('./legacy-snapshot-capture-fixture.ts'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); - vi.mock('../../platforms/android/app-lifecycle.ts', async (importOriginal) => { const actual = await importOriginal(); return { @@ -25,7 +22,12 @@ vi.mock('../../platforms/android/app-lifecycle.ts', async (importOriginal) => { import { dispatchCommand } from '../../core/dispatch.ts'; import { createRequestHandler } from './test-device-runtime-gateway.ts'; -import { dispatchScreenshotViaRuntime } from '../screenshot-runtime.ts'; +import { + screenshotRuntimeFixture, + writeSolidPng, + type ScreenshotRuntimeFixture, + type ScreenshotRuntimeFixtureOptions, +} from './screenshot-runtime-fixture.ts'; import type { DaemonRequest, SessionState } from '../types.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { attachRefs } from '@agent-device/kernel/snapshot'; @@ -68,41 +70,38 @@ beforeEach(() => { mockDispatch.mockResolvedValue({}); }); -function writeSolidPng(filePath: string, width = 100, height = 50): void { - const png = new PNG({ width, height }); - for (let index = 0; index < png.data.length; index += 4) { - png.data[index] = 255; - png.data[index + 1] = 255; - png.data[index + 2] = 255; - png.data[index + 3] = 255; - } - fs.writeFileSync(filePath, PNG.sync.write(png)); -} +type ScreenshotRouter = Readonly<{ + handler: ReturnType; + sessionStore: ReturnType; + runtime: ScreenshotRuntimeFixture; +}>; -test('screenshot resolves relative positional path against request cwd', async () => { - const callerCwd = mkdtempForTestSync('agent-device-screenshot-cwd-caller-'); +function screenshotRouter( + session: SessionState, + options: ScreenshotRuntimeFixtureOptions = {}, +): ScreenshotRouter { const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeSession('default')); - - let capturedPath: string | undefined; - mockDispatch.mockImplementation(async (_device, command, positionals) => { - if (command === 'screenshot') { - capturedPath = positionals[0]; - if (capturedPath) { - writeSolidPng(capturedPath); - } - } - return {}; - }); - + sessionStore.set(session.name, session); + const runtime = screenshotRuntimeFixture(options); const handler = createRequestHandler({ logPath: path.join(os.tmpdir(), 'daemon.log'), token: 'test-token', sessionStore, leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), + deviceRuntimeGateway: runtime.gateway, trackDownloadableArtifact: () => 'artifact-id', }); + return { handler, sessionStore, runtime }; +} + +function capturedPath(runtime: ScreenshotRuntimeFixture): string | undefined { + return runtime.captureScreenshot.mock.calls[0]?.[0].outPath; +} + +test('screenshot resolves relative positional path against request cwd', async () => { + const callerCwd = mkdtempForTestSync('agent-device-screenshot-cwd-caller-'); + const { handler, sessionStore, runtime } = screenshotRouter(makeSession('default')); await handler({ token: 'test-token', @@ -112,54 +111,100 @@ test('screenshot resolves relative positional path against request cwd', async ( meta: { cwd: callerCwd, requestId: 'req-1', sessionExplicit: true }, }); - expect(capturedPath).toBeTruthy(); - expect(capturedPath).toBe(path.join(callerCwd, 'evidence/test.png')); - expect(path.isAbsolute(capturedPath!)).toBe(true); + expect(capturedPath(runtime)).toBe(path.join(callerCwd, 'evidence/test.png')); + expect(path.isAbsolute(capturedPath(runtime)!)).toBe(true); const recordedAction = sessionStore.get('default')?.actions.at(-1); expect(recordedAction?.positionals).toEqual([path.join(callerCwd, 'evidence/test.png')]); }); -test('default screenshot temp directory is cleaned when capture fails', async () => { - const session = makeSession('default'); - let capturedPath: string | undefined; - mockDispatch.mockImplementation(async (_device, command, positionals) => { - if (command === 'screenshot') capturedPath = positionals[0]; - throw new Error('capture failed'); - }); - - await expect( - dispatchScreenshotViaRuntime({ - session, - sessionName: session.name, - outputPlacement: 'default', - dispatchContext: {}, - }), - ).rejects.toThrow(/capture failed/); +test('screenshot keeps absolute positional path unchanged', async () => { + const absolutePath = path.join(os.tmpdir(), 'evidence/test.png'); + const { handler, sessionStore, runtime } = screenshotRouter(makeSession('default')); + + await handler({ + token: 'test-token', + session: 'default', + command: 'screenshot', + positionals: [absolutePath], + meta: { cwd: '/some/other/dir', requestId: 'req-2', sessionExplicit: true }, + }); - expect(capturedPath).toBeTruthy(); - expect(path.basename(capturedPath!)).toBe('screenshot.png'); - expect(fs.existsSync(path.dirname(capturedPath!))).toBe(false); + expect(capturedPath(runtime)).toBe(absolutePath); + const recordedAction = sessionStore.get('default')?.actions.at(-1); + expect(recordedAction?.positionals).toEqual([absolutePath]); }); -test('session-backed iOS simulator screenshots skip redundant boot probe', async () => { - const session = makeIosSession('ios'); - const outPath = path.join(os.tmpdir(), 'agent-device-ios-session-screenshot.png'); - let capturedContext: Parameters[4]; +test('screenshot resolves --out flag path against request cwd', async () => { + const callerCwd = mkdtempForTestSync('agent-device-screenshot-out-cwd-'); + const { handler, sessionStore, runtime } = screenshotRouter(makeSession('default')); - mockDispatch.mockImplementation(async (_device, _command, _positionals, _outPath, context) => { - capturedContext = context; - return { path: outPath }; + await handler({ + token: 'test-token', + session: 'default', + command: 'screenshot', + positionals: [], + flags: { out: 'evidence/test.png' }, + meta: { cwd: callerCwd, requestId: 'req-3', sessionExplicit: true }, }); - await dispatchScreenshotViaRuntime({ - session, - sessionName: session.name, - outPath, - outputPlacement: 'positional', - dispatchContext: {}, + expect(capturedPath(runtime)).toBe(path.join(callerCwd, 'evidence/test.png')); + expect(path.isAbsolute(capturedPath(runtime)!)).toBe(true); + const recordedAction = sessionStore.get('default')?.actions.at(-1); + expect(recordedAction?.flags.out).toBe(path.join(callerCwd, 'evidence/test.png')); +}); + +test('screenshot runtime supplies default output path when none is requested', async () => { + const { handler, runtime } = screenshotRouter(makeSession('default')); + + const response = await handler({ + token: 'test-token', + session: 'default', + command: 'screenshot', + positionals: [], + meta: { requestId: 'req-default-screenshot' }, }); - expect(capturedContext?.skipIosSimulatorBootCheck).toBe(true); + expect(response.ok).toBe(true); + expect(capturedPath(runtime)).toContain('agent-device-screenshot-'); + expect(path.basename(capturedPath(runtime) ?? '')).toBe('screenshot.png'); + if (response.ok) { + expect(response.data?.path).toBe(capturedPath(runtime)); + } +}); + +test('screenshot forwards macOS session surface to the bound capture', async () => { + const { handler, runtime } = screenshotRouter(makeMacOsMenubarSession('default')); + + await handler({ + token: 'test-token', + session: 'default', + command: 'screenshot', + positionals: ['/tmp/menubar.png'], + meta: { requestId: 'req-surface-screenshot' }, + }); + + expect(runtime.captureScreenshot.mock.calls[0]?.[0].options).toMatchObject({ + surface: 'menubar', + appBundleId: 'com.example.menubarapp', + }); +}); + +test('click forwards macOS menubar session surface to dispatch', async () => { + const { handler } = screenshotRouter(makeMacOsMenubarSession('default')); + + await handler({ + token: 'test-token', + session: 'default', + command: 'click', + positionals: ['100', '200'], + meta: { requestId: 'req-surface-click' }, + }); + + expect(mockDispatch.mock.calls[0]?.[1]).toBe('press'); + expect(mockDispatch.mock.calls[0]?.[4]).toMatchObject({ + surface: 'menubar', + appBundleId: 'com.example.menubarapp', + }); }); test('router serializes concurrent commands for the same device across sessions', async () => { @@ -171,21 +216,27 @@ test('router serializes concurrent commands for the same device across sessions' let active = 0; let maxActive = 0; const gates: Array<() => void> = []; - - mockDispatch.mockImplementation(async (_device, command) => { - order.push(`start-${command}`); + const gate = async (label: string) => { + order.push(`start-${label}`); active += 1; maxActive = Math.max(maxActive, active); - if (command === 'screenshot') { - writeSolidPng('/tmp/first.png'); - } await new Promise((resolve) => { gates.push(() => { active -= 1; - order.push(`end-${command}`); + order.push(`end-${label}`); resolve(); }); }); + }; + + const runtime = screenshotRuntimeFixture({ + onCapture: async (input) => { + writeSolidPng(input.outPath); + await gate('screenshot'); + }, + }); + mockDispatch.mockImplementation(async (_device, command) => { + await gate(command); return {}; }); @@ -195,6 +246,7 @@ test('router serializes concurrent commands for the same device across sessions' sessionStore, leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), + deviceRuntimeGateway: runtime.gateway, trackDownloadableArtifact: () => 'artifact-id', }); @@ -240,164 +292,10 @@ test('router serializes concurrent commands for the same device across sessions' expect(order).toEqual(['start-screenshot', 'end-screenshot', 'start-scroll', 'end-scroll']); }); -test('screenshot forwards macOS session surface to dispatch', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeMacOsMenubarSession('default')); - - mockDispatch.mockImplementation(async () => ({})); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', - }); - - await handler({ - token: 'test-token', - session: 'default', - command: 'screenshot', - positionals: ['/tmp/menubar.png'], - meta: { requestId: 'req-surface-screenshot' }, - }); - - expect(mockDispatch.mock.calls[0]?.[4]).toMatchObject({ - surface: 'menubar', - appBundleId: 'com.example.menubarapp', - }); -}); - -test('click forwards macOS menubar session surface to dispatch', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeMacOsMenubarSession('default')); - - mockDispatch.mockImplementation(async () => ({})); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', - }); - - await handler({ - token: 'test-token', - session: 'default', - command: 'click', - positionals: ['100', '200'], - meta: { requestId: 'req-surface-click' }, - }); - - expect(mockDispatch.mock.calls[0]?.[1]).toBe('press'); - expect(mockDispatch.mock.calls[0]?.[4]).toMatchObject({ - surface: 'menubar', - appBundleId: 'com.example.menubarapp', - }); -}); - -test('screenshot keeps absolute positional path unchanged', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeSession('default')); - - const absolutePath = path.join(os.tmpdir(), 'evidence/test.png'); - let capturedPath: string | undefined; - - mockDispatch.mockImplementation(async (_device, command, positionals) => { - if (command === 'screenshot') { - capturedPath = positionals[0]; - if (capturedPath) { - writeSolidPng(capturedPath); - } - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', - }); - - await handler({ - token: 'test-token', - session: 'default', - command: 'screenshot', - positionals: [absolutePath], - meta: { cwd: '/some/other/dir', requestId: 'req-2', sessionExplicit: true }, - }); - - expect(capturedPath).toBe(absolutePath); - const recordedAction = sessionStore.get('default')?.actions.at(-1); - expect(recordedAction?.positionals).toEqual([absolutePath]); -}); - -test('screenshot runtime supplies default output path when none is requested', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeSession('default')); - - let capturedPath: string | undefined; - mockDispatch.mockImplementation(async (_device, command, positionals) => { - if (command === 'screenshot') { - capturedPath = positionals[0]; - if (capturedPath) { - writeSolidPng(capturedPath); - } - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', - }); - - const response = await handler({ - token: 'test-token', - session: 'default', - command: 'screenshot', - positionals: [], - meta: { requestId: 'req-default-screenshot' }, - }); - - expect(response.ok).toBe(true); - expect(capturedPath).toContain('agent-device-screenshot-'); - expect(path.basename(capturedPath ?? '')).toBe('screenshot.png'); - if (response.ok) { - expect(response.data?.path).toBe(capturedPath); - } -}); - test('iOS simulator screenshot response includes output dimensions and logical density metadata', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeIosSession('default')); const screenshotPath = path.join(os.tmpdir(), `agent-device-ios-meta-${Date.now()}.png`); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command === 'screenshot') { - writeSolidPng(screenshotPath, 402, 874); - return { path: screenshotPath }; - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', + const { handler } = screenshotRouter(makeIosSession('default'), { + onCapture: (input) => writeSolidPng(input.outPath, 402, 874), }); const response = await handler({ @@ -422,25 +320,9 @@ test('iOS simulator screenshot response includes output dimensions and logical d }); test('non-iOS screenshot response tolerates malformed PNG metadata', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeSession('default')); const screenshotPath = path.join(os.tmpdir(), `agent-device-android-truncated-${Date.now()}.png`); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command === 'screenshot') { - fs.writeFileSync(screenshotPath, Buffer.alloc(0)); - return { path: screenshotPath }; - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', + const { handler } = screenshotRouter(makeSession('default'), { + onCapture: (input) => fs.writeFileSync(input.outPath, Buffer.alloc(0)), }); const response = await handler({ @@ -460,25 +342,9 @@ test('non-iOS screenshot response tolerates malformed PNG metadata', async () => }); test('iOS simulator screenshot omits logical density metadata after --scale downscale', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeIosSession('default')); const screenshotPath = path.join(os.tmpdir(), `agent-device-ios-scale-${Date.now()}.png`); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command === 'screenshot') { - writeSolidPng(screenshotPath, 804, 1748); - return { path: screenshotPath }; - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', + const { handler } = screenshotRouter(makeIosSession('default'), { + onCapture: (input) => writeSolidPng(input.outPath, 804, 1748), }); const response = await handler({ @@ -492,11 +358,7 @@ test('iOS simulator screenshot omits logical density metadata after --scale down expect(response.ok).toBe(true); if (response.ok) { - expect(response.data).toMatchObject({ - path: screenshotPath, - width: 402, - height: 874, - }); + expect(response.data).toMatchObject({ path: screenshotPath, width: 402, height: 874 }); expect(response.data).not.toHaveProperty('logicalWidth'); expect(response.data).not.toHaveProperty('logicalHeight'); expect(response.data).not.toHaveProperty('pixelDensity'); @@ -504,21 +366,7 @@ test('iOS simulator screenshot omits logical density metadata after --scale down }); test('screenshot rejects the removed max-size field from older remote clients', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeIosSession('default')); - - mockDispatch.mockImplementation(async () => { - throw new Error('dispatch should not run for a retired-flag request'); - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', - }); + const { handler, runtime } = screenshotRouter(makeIosSession('default')); const response = await handler({ token: 'test-token', @@ -534,62 +382,40 @@ test('screenshot rejects the removed max-size field from older remote clients', expect(response.error.code).toBe('INVALID_ARGS'); expect(response.error.message).toContain('screenshot --max-size was removed; use --scale'); } + expect(runtime.captureScreenshot).not.toHaveBeenCalled(); }); -test('screenshot resolves --out flag path against request cwd', async () => { - const callerCwd = mkdtempForTestSync('agent-device-screenshot-out-cwd-'); - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeSession('default')); - - let capturedOut: string | undefined; - - mockDispatch.mockImplementation(async (_device, command, _positionals, outPath) => { - if (command === 'screenshot') { - capturedOut = outPath; - if (capturedOut) { - writeSolidPng(capturedOut); - } - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', - }); +test('screenshot --pixel-density is rejected outside iOS-family simulators', async () => { + const { handler, runtime } = screenshotRouter(makeSession('default')); - await handler({ + const response = await handler({ token: 'test-token', session: 'default', command: 'screenshot', - positionals: [], - flags: { out: 'evidence/test.png' }, - meta: { cwd: callerCwd, requestId: 'req-3', sessionExplicit: true }, + positionals: ['/tmp/android.png'], + flags: { screenshotPixelDensity: 2 }, + meta: { requestId: 'req-unsupported-density' }, }); - expect(capturedOut).toBeTruthy(); - expect(capturedOut).toBe(path.join(callerCwd, 'evidence/test.png')); - expect(path.isAbsolute(capturedOut!)).toBe(true); - const recordedAction = sessionStore.get('default')?.actions.at(-1); - expect(recordedAction?.flags.out).toBe(path.join(callerCwd, 'evidence/test.png')); + expect(response.ok).toBe(false); + if (!response.ok) { + expect(response.error.message).toContain('currently supported only on iOS-family simulators'); + } + expect(runtime.captureScreenshot).not.toHaveBeenCalled(); }); test('screenshot --overlay-refs captures a fresh snapshot when the session has none', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeSession('default')); const screenshotPath = path.join(os.tmpdir(), `agent-device-overlay-${Date.now()}.png`); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command === 'screenshot') { - writeSolidPng(screenshotPath); - return { path: screenshotPath }; - } - if (command === 'snapshot') { + const order: string[] = []; + const { handler, runtime } = screenshotRouter(makeSession('default'), { + onCapture: (input) => { + order.push('screenshot'); + writeSolidPng(input.outPath); + }, + snapshotResult: () => { + order.push('snapshot'); return { + backend: 'android', nodes: [ { index: 0, @@ -600,17 +426,7 @@ test('screenshot --overlay-refs captures a fresh snapshot when the session has n }, ], }; - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', + }, }); const response = await handler({ @@ -629,80 +445,64 @@ test('screenshot --overlay-refs captures a fresh snapshot when the session has n ref: 'e1', label: 'Continue', rect: { x: 0, y: 0, width: 40, height: 20 }, - overlayRect: { x: 0, y: 0, width: 100, height: 50 }, - center: { x: 50, y: 25 }, + // The Android backend reports device-pixel rects, so the overlay draws them unprojected. + overlayRect: { x: 0, y: 0, width: 40, height: 20 }, + center: { x: 20, y: 10 }, }, ]); } - expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['screenshot', 'snapshot']); + expect(order).toEqual(['screenshot', 'snapshot']); + expect(runtime.binds).toHaveLength(1); }); test('screenshot --overlay-refs uses interactive iOS presentation for row-like other nodes', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeIosSession('default')); const screenshotPath = path.join(os.tmpdir(), `agent-device-overlay-ios-${Date.now()}.png`); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command === 'screenshot') { - writeSolidPng(screenshotPath, 402, 874); - return { path: screenshotPath }; - } - if (command === 'snapshot') { - return { - backend: 'xctest', - nodes: [ - { - index: 0, - depth: 0, - type: 'Application', - label: 'New Expensify Dev', - rect: { x: 0, y: 0, width: 402, height: 874 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Other', - label: '!, Open debugger to view warnings.', - rect: { x: 0, y: 0, width: 402, height: 874 }, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'ScrollView', - label: 'Recent chats', - rect: { x: 8, y: 212, width: 386, height: 600 }, - }, - { - index: 3, - depth: 2, - parentIndex: 2, - type: 'Other', - label: 'Recent chats', - rect: { x: 0, y: 220, width: 402, height: 16 }, - }, - { - index: 4, - depth: 2, - parentIndex: 2, - type: 'Other', - label: 'Receipt missing details, Receipt scanning failed. Enter details manually.', - rect: { x: 8, y: 367, width: 386, height: 64 }, - }, - ], - }; - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', + const { handler, sessionStore, runtime } = screenshotRouter(makeIosSession('default'), { + onCapture: (input) => writeSolidPng(input.outPath, 402, 874), + snapshotResult: () => ({ + backend: 'xctest', + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'New Expensify Dev', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Other', + label: '!, Open debugger to view warnings.', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'ScrollView', + label: 'Recent chats', + rect: { x: 8, y: 212, width: 386, height: 600 }, + }, + { + index: 3, + depth: 2, + parentIndex: 2, + type: 'Other', + label: 'Recent chats', + rect: { x: 0, y: 220, width: 402, height: 16 }, + }, + { + index: 4, + depth: 2, + parentIndex: 2, + type: 'Other', + label: 'Receipt missing details, Receipt scanning failed. Enter details manually.', + rect: { x: 8, y: 367, width: 386, height: 64 }, + }, + ], + }), }); const response = await handler({ @@ -726,15 +526,13 @@ test('screenshot --overlay-refs uses interactive iOS presentation for row-like o }, ]); } - expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['screenshot', 'snapshot']); - expect(mockDispatch.mock.calls[1]?.[4]).toMatchObject({ - snapshotInteractiveOnly: true, + expect(runtime.captureSnapshot.mock.calls[0]?.[0].options).toMatchObject({ + interactiveOnly: true, }); expect(sessionStore.get('default')?.snapshot?.nodes[4]?.type).toBe('Cell'); }); test('screenshot --overlay-refs uses a fresh snapshot instead of stale session snapshot', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); const session = makeSession('default'); session.snapshot = { nodes: attachRefs([ @@ -748,37 +546,20 @@ test('screenshot --overlay-refs uses a fresh snapshot instead of stale session s ]), createdAt: Date.now(), }; - sessionStore.set('default', session); - const screenshotPath = path.join(os.tmpdir(), `agent-device-overlay-${Date.now()}.png`); - mockDispatch.mockImplementation(async (_device, command) => { - if (command === 'screenshot') { - writeSolidPng(screenshotPath); - return { path: screenshotPath }; - } - if (command === 'snapshot') { - return { - nodes: [ - { - index: 0, - type: 'XCUIElementTypeButton', - label: 'Fresh', - hittable: true, - rect: { x: 0, y: 0, width: 40, height: 20 }, - }, - ], - }; - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', + const { handler, sessionStore } = screenshotRouter(session, { + snapshotResult: () => ({ + backend: 'android', + nodes: [ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Fresh', + hittable: true, + rect: { x: 0, y: 0, width: 40, height: 20 }, + }, + ], + }), }); const response = await handler({ @@ -798,8 +579,9 @@ test('screenshot --overlay-refs uses a fresh snapshot instead of stale session s ref: 'e1', label: 'Fresh', rect: { x: 0, y: 0, width: 40, height: 20 }, - overlayRect: { x: 0, y: 0, width: 100, height: 50 }, - center: { x: 50, y: 25 }, + // The Android backend reports device-pixel rects, so the overlay draws them unprojected. + overlayRect: { x: 0, y: 0, width: 40, height: 20 }, + center: { x: 20, y: 10 }, }, ]); } @@ -809,43 +591,22 @@ test('screenshot --overlay-refs uses a fresh snapshot instead of stale session s }); test('screenshot --pixel-density keeps overlay refs aligned to scaled iOS simulator output', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeIosSession('default')); const screenshotPath = path.join(os.tmpdir(), `agent-device-overlay-2x-${Date.now()}.png`); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command === 'screenshot') { - writeSolidPng(screenshotPath, 804, 1748); - return { path: screenshotPath }; - } - if (command === 'snapshot') { - return { - nodes: [ - { - index: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 402, height: 874 }, - }, - { - index: 1, - type: 'XCUIElementTypeButton', - label: 'Continue', - hittable: true, - rect: { x: 10, y: 20, width: 80, height: 30 }, - }, - ], - }; - } - return {}; - }); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', + const { handler } = screenshotRouter(makeIosSession('default'), { + onCapture: (input) => writeSolidPng(input.outPath, 804, 1748), + snapshotResult: () => ({ + backend: 'xctest', + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { + index: 1, + type: 'XCUIElementTypeButton', + label: 'Continue', + hittable: true, + rect: { x: 10, y: 20, width: 80, height: 30 }, + }, + ], + }), }); const response = await handler({ @@ -876,32 +637,3 @@ test('screenshot --pixel-density keeps overlay refs aligned to scaled iOS simula }); } }); - -test('screenshot --pixel-density is rejected outside iOS-family simulators', async () => { - const sessionStore = makeSessionStore('agent-device-router-screenshot-'); - sessionStore.set('default', makeSession('default')); - - const handler = createRequestHandler({ - logPath: path.join(os.tmpdir(), 'daemon.log'), - token: 'test-token', - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: createTestDeviceInventoryGateways(), - trackDownloadableArtifact: () => 'artifact-id', - }); - - const response = await handler({ - token: 'test-token', - session: 'default', - command: 'screenshot', - positionals: ['/tmp/android.png'], - flags: { screenshotPixelDensity: 2 }, - meta: { requestId: 'req-unsupported-density' }, - }); - - expect(response.ok).toBe(false); - if (!response.ok) { - expect(response.error.message).toContain('currently supported only on iOS-family simulators'); - } - expect(mockDispatch).not.toHaveBeenCalled(); -}); diff --git a/src/daemon/__tests__/screenshot-runtime-fixture.ts b/src/daemon/__tests__/screenshot-runtime-fixture.ts new file mode 100644 index 0000000000..d41e0eee75 --- /dev/null +++ b/src/daemon/__tests__/screenshot-runtime-fixture.ts @@ -0,0 +1,119 @@ +import { + localRuntimeOwner, + narrowDeviceBinding, + screenshotRuntimeOperationFacts, + snapshotRuntimeOperationFacts, + type CaptureScreenshotInput, + type CaptureSnapshotInput, + type DeviceRuntimeGateway, + type PlatformRuntimeOperations, + type RuntimeFacts, + type RuntimeOperationFact, + type SnapshotResult, +} from '@agent-device/contracts/platform'; +import { deviceShape, type DeviceInfo } from '@agent-device/kernel/device'; +import fs from 'node:fs'; +import { vi, type Mock } from 'vitest'; +import { PNG } from '../../utils/png.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import { unavailableDeviceRuntimeGateway } from './test-device-runtime-gateway.ts'; + +const available = Object.freeze({ available: true } as const); + +export type ScreenshotRuntimeFixtureOptions = Readonly<{ + /** The exact-owner `captureScreenshot` fact this fake device reports. */ + capture?: RuntimeOperationFact; + /** The exact-owner `captureSnapshot` fact, which `--overlay-refs` also requires. */ + snapshot?: RuntimeOperationFact; + /** Replaces the default "write a solid PNG at the requested path" capture behavior. */ + onCapture?: (input: CaptureScreenshotInput) => Promise | void; + snapshotResult?: (input: CaptureSnapshotInput) => SnapshotResult; +}>; + +export type ScreenshotRuntimeFixture = Readonly<{ + gateway: DeviceRuntimeGateway; + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; + captureScreenshot: Mock<(input: CaptureScreenshotInput) => Promise>; + captureSnapshot: Mock<(input: CaptureSnapshotInput) => Promise>; + /** Every `(device, use)` the route bound, so a test can prove exactly one bind happened. */ + binds: Array>; +}>; + +/** + * The request-scoped screenshot seam a migrated route consumes: exact owner facts plus one + * binding. Tests assert against the capture input the runtime received, never against a legacy + * dispatch call. + */ +export function screenshotRuntimeFixture( + options: ScreenshotRuntimeFixtureOptions = {}, +): ScreenshotRuntimeFixture { + const binds: Array> = []; + const captureScreenshot = vi.fn(async (input: CaptureScreenshotInput) => { + if (options.onCapture) { + await options.onCapture(input); + return; + } + writeSolidPng(input.outPath); + }); + const captureSnapshot = vi.fn( + async (input: CaptureSnapshotInput): Promise => + options.snapshotResult?.(input) ?? { nodes: [], backend: 'android' }, + ); + + // The unavailable gateway is the exhaustive fact catalog; only the capture cells are overridden. + const facts = async (device: DeviceInfo): Promise> => { + const base = await unavailableDeviceRuntimeGateway.inspectFacts(device); + return { + device: { ...deviceShape(device), providerMode: 'local' }, + operations: { + ...base.operations, + ...screenshotRuntimeOperationFacts({ capture: options.capture ?? available }), + ...snapshotRuntimeOperationFacts({ + capture: options.snapshot ?? available, + customActions: options.snapshot ?? available, + withoutActiveApp: options.snapshot ?? available, + }), + }, + }; + }; + + const binding = async (device: DeviceInfo) => { + binds.push({ device }); + return { + device, + owner: localRuntimeOwner(device.platform), + facts: await facts(device), + operations: { + captureScreenshot, + captureSnapshot, + captureSnapshotWithCustomActions: captureSnapshot, + captureSnapshotWithoutActiveApp: captureSnapshot, + }, + [Symbol.asyncDispose]: async () => {}, + }; + }; + + const inspectFacts: InspectDeviceRuntimeFacts = async (device) => await facts(device); + const bindDevice: BindDeviceRuntime = async (device, use) => + narrowDeviceBinding(await binding(device), use); + + return { + gateway: Object.freeze({ + inspectFacts, + bind: async ({ device }) => await binding(device), + shutdown: async () => {}, + }), + inspectFacts, + bindDevice, + captureScreenshot, + captureSnapshot, + binds, + }; +} + +export function writeSolidPng(filePath: string, width = 100, height = 50): void { + const png = new PNG({ width, height }); + png.data.fill(255); + fs.writeFileSync(filePath, PNG.sync.write(png)); +} diff --git a/src/daemon/__tests__/screenshot-runtime.test.ts b/src/daemon/__tests__/screenshot-runtime.test.ts new file mode 100644 index 0000000000..04b997687c --- /dev/null +++ b/src/daemon/__tests__/screenshot-runtime.test.ts @@ -0,0 +1,198 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import { makeSession } from '../../__tests__/test-utils/session-factories.ts'; +import type { GenericPlatformExecutionParams } from '../request-generic-dispatch.ts'; +import { resolveScreenshotGenericExecution } from '../screenshot-runtime.ts'; +import { screenshotRuntimeFixture } from './screenshot-runtime-fixture.ts'; +import type { DaemonRequest, SessionState } from '../types.ts'; + +const unavailableCapture = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf' as const, + hint: 'screenshot is not supported on Vega OS: the Vega runtime exposes remote navigation only.', +}); + +function screenshotRequest(overrides: Partial = {}): DaemonRequest { + return { + command: 'screenshot', + token: 'test-token', + session: 'default', + positionals: [], + ...overrides, + }; +} + +function executionParams( + session: SessionState, + req: DaemonRequest, +): GenericPlatformExecutionParams { + return { + session, + sessionName: session.name, + logPath: path.join(os.tmpdir(), 'daemon.log'), + command: 'screenshot', + request: req, + positionals: req.positionals ?? [], + out: req.flags?.out, + dispatchContext: {}, + }; +} + +test('admits one capture plan, binds once, and hands the runtime the resolved destination', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const outPath = path.join(os.tmpdir(), `agent-device-bound-capture-${Date.now()}.png`); + const req = screenshotRequest({ positionals: [outPath] }); + + const resolved = await resolveScreenshotGenericExecution({ + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + expect(fixture.binds).toHaveLength(1); + await resolved.execute(executionParams(session, req)); + expect(fixture.captureScreenshot).toHaveBeenCalledTimes(1); + expect(fixture.captureScreenshot.mock.calls[0]?.[0]).toMatchObject({ outPath }); + expect(fs.existsSync(outPath)).toBe(true); +}); + +test('an iOS simulator session capture skips the redundant boot probe', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('ios', { device: IOS_SIMULATOR }); + const outPath = path.join(os.tmpdir(), `agent-device-ios-boot-probe-${Date.now()}.png`); + const req = screenshotRequest({ positionals: [outPath] }); + + const resolved = await resolveScreenshotGenericExecution({ + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!resolved.ok) throw new Error('screenshot admission must succeed'); + await resolved.execute(executionParams(session, req)); + + expect(fixture.captureScreenshot.mock.calls[0]?.[0].options).toMatchObject({ + skipIosSimulatorBootCheck: true, + }); +}); + +test('refuses an unavailable exact-owner capture fact before binding', async () => { + const fixture = screenshotRuntimeFixture({ capture: unavailableCapture }); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + + const resolved = await resolveScreenshotGenericExecution({ + req: screenshotRequest({ positionals: ['/tmp/unsupported.png'] }), + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(resolved).toEqual({ + ok: false, + response: { + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + message: 'screenshot is not supported on this device', + hint: unavailableCapture.hint, + details: { reason: 'unsupported-platform-leaf' }, + }, + }, + }); + expect(fixture.binds).toHaveLength(0); + expect(fixture.captureScreenshot).not.toHaveBeenCalled(); +}); + +test('--overlay-refs is refused up front when the target cannot capture a tree', async () => { + const fixture = screenshotRuntimeFixture({ + snapshot: { available: false, reason: 'unsupported-platform-leaf' }, + }); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + + const resolved = await resolveScreenshotGenericExecution({ + req: screenshotRequest({ positionals: ['/tmp/overlay.png'], flags: { overlayRefs: true } }), + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(resolved.ok).toBe(false); + if (resolved.ok) return; + expect(resolved.response).toMatchObject({ + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + hint: 'Re-run screenshot without --overlay-refs.', + }, + }); + // Nothing was captured: a plan that cannot annotate never writes a PNG the caller must discard. + expect(fixture.binds).toHaveLength(0); + expect(fixture.captureScreenshot).not.toHaveBeenCalled(); +}); + +test('the default destination is a reserved temp file, removed when the capture fails', async () => { + let requestedPath: string | undefined; + const fixture = screenshotRuntimeFixture({ + onCapture: (input) => { + requestedPath = input.outPath; + throw new Error('capture failed'); + }, + }); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const req = screenshotRequest(); + + const resolved = await resolveScreenshotGenericExecution({ + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!resolved.ok) throw new Error('screenshot admission must succeed'); + + await expect(resolved.execute(executionParams(session, req))).rejects.toThrow(/capture failed/); + expect(path.basename(requestedPath ?? '')).toBe('screenshot.png'); + expect(fs.existsSync(path.dirname(requestedPath ?? ''))).toBe(false); +}); + +test('rejects the retired max-size field before inspecting owner facts', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: IOS_SIMULATOR }); + + await expect( + resolveScreenshotGenericExecution({ + req: screenshotRequest({ + positionals: ['/tmp/legacy.png'], + flags: { screenshotMaxSize: 720 } as unknown as DaemonRequest['flags'], + }), + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ).rejects.toMatchObject({ code: 'INVALID_ARGS' }); + expect(fixture.binds).toHaveLength(0); +}); + +test('rejects --pixel-density outside iOS-family simulators before inspecting owner facts', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + + await expect( + resolveScreenshotGenericExecution({ + req: screenshotRequest({ + positionals: ['/tmp/android.png'], + flags: { screenshotPixelDensity: 2 }, + }), + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION' }); + expect(fixture.binds).toHaveLength(0); +}); diff --git a/src/daemon/__tests__/snapshot-runtime-fixture.ts b/src/daemon/__tests__/snapshot-runtime-fixture.ts index b9401ee59e..29da54ce75 100644 --- a/src/daemon/__tests__/snapshot-runtime-fixture.ts +++ b/src/daemon/__tests__/snapshot-runtime-fixture.ts @@ -2,7 +2,9 @@ import { localRuntimeOwner, narrowDeviceBinding, providerRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, + type CaptureScreenshotInput, type CaptureSnapshotInput, type PlatformRuntimeOperations, type RuntimeFacts, @@ -14,6 +16,14 @@ import { getRequestSignal } from '../../request/cancel.ts'; import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; import { unavailableDeviceRuntimeGateway } from './test-device-runtime-gateway.ts'; +import { writeSolidPng } from './screenshot-runtime-fixture.ts'; + +/** + * Every capture the fixture's bound screenshot operation received, newest last. Handler tests that + * cannot reach the fixture instance (the snapshot route builds it internally) assert the capture + * intent here instead of on a legacy dispatch call. + */ +export const fixtureScreenshotCaptures: CaptureScreenshotInput[] = []; /** Request-scoped snapshot seam for handler tests that mock the legacy leaf dispatch. */ export function snapshotRuntimeFixture(requestId?: string): Readonly<{ @@ -28,6 +38,10 @@ export function snapshotRuntimeFixture(requestId?: string): Readonly<{ const providerOwned = facts.device.providerMode === 'provider-runtime'; const captureSnapshot = async (input: CaptureSnapshotInput) => await dispatchFixtureSnapshot(device, input, requestSignal); + const captureScreenshot = async (input: CaptureScreenshotInput) => { + fixtureScreenshotCaptures.push(input); + writeSolidPng(input.outPath); + }; return narrowDeviceBinding( { device, @@ -39,6 +53,7 @@ export function snapshotRuntimeFixture(requestId?: string): Readonly<{ captureSnapshot, captureSnapshotWithCustomActions: captureSnapshot, captureSnapshotWithoutActiveApp: captureSnapshot, + captureScreenshot, }, [Symbol.asyncDispose]: async () => {}, }, @@ -70,6 +85,7 @@ async function snapshotFacts(device: DeviceInfo): Promise vi.fn()); - -vi.mock('../../core/dispatch.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - dispatchCommand: dispatchCommandMock, - }; -}); +import { + screenshotRuntimeFixture, + writeSolidPng, + type ScreenshotRuntimeFixture, + type ScreenshotRuntimeFixtureOptions, +} from './screenshot-runtime-fixture.ts'; const SPARSE: SnapshotQualityVerdict = { state: 'sparse', @@ -31,26 +26,29 @@ async function scenario() { return { sessionStore, sessionName, logPath: path.join(root, 'daemon.log') }; } -/** Snapshot captures answer with the seeded verdict; screenshot captures answer per `screenshot`. */ +/** Both captures answer at the request-bound runtime seam: the snapshot carries the seeded + * verdict, and the screenshot leg is the fallback under test. */ function seed( verdict: SnapshotQualityVerdict, - screenshot: () => Promise> = async () => ({ width: 390, height: 844 }), -) { - dispatchCommandMock.mockReset(); - dispatchCommandMock.mockImplementation(async (_device: unknown, command: string) => - command === 'screenshot' - ? await screenshot() - : { - backend: 'xctest', - truncated: false, - quality: verdict, - nodes: [{ index: 0, depth: 0, type: 'Application', label: 'Demo' }], - }, - ); + onCapture: ScreenshotRuntimeFixtureOptions['onCapture'] = (input) => writeSolidPng(input.outPath), +): ScreenshotRuntimeFixture { + return screenshotRuntimeFixture({ + onCapture, + snapshotResult: () => + ({ + backend: 'xctest', + truncated: false, + quality: verdict, + nodes: [{ index: 0, depth: 0, type: 'Application', label: 'Demo' }], + }) as never, + }); } -async function dispatch(input: Awaited>, internalObservation = false) { - const runtime = snapshotRuntimeFixture(); +async function dispatch( + input: Awaited>, + runtime: ScreenshotRuntimeFixture, + internalObservation = false, +) { const response = await dispatchSnapshotViaRuntime({ req: { command: 'snapshot', @@ -62,24 +60,21 @@ async function dispatch(input: Awaited>, internalObs sessionName: input.sessionName, logPath: input.logPath, sessionStore: input.sessionStore, - ...runtime, + inspectFacts: runtime.inspectFacts, + bindDevice: runtime.bindDevice, }); if (!response.ok) throw new Error('expected ok response'); return response.data ?? {}; } -function screenshotCalls() { - return dispatchCommandMock.mock.calls.filter((call) => call[1] === 'screenshot'); -} - test('a sparse snapshot captures the screenshot its own remedy asks for and links it', async () => { const input = await scenario(); - seed(SPARSE); + const runtime = seed(SPARSE); - const data = await dispatch(input); + const data = await dispatch(input, runtime); const warnings = (data.warnings ?? []) as string[]; - expect(screenshotCalls()).toHaveLength(1); + expect(runtime.captureScreenshot).toHaveBeenCalledTimes(1); expect(data.fallbackScreenshotPath).toMatch(/\.png$/); expect(data.artifacts).toEqual([ { @@ -97,36 +92,36 @@ test('a sparse snapshot captures the screenshot its own remedy asks for and link test('a readable snapshot never pays for a screenshot', async () => { const input = await scenario(); - seed({ state: 'healthy', backend: 'tree' }); + const runtime = seed({ state: 'healthy', backend: 'tree' }); - const data = await dispatch(input); + const data = await dispatch(input, runtime); - expect(screenshotCalls()).toHaveLength(0); + expect(runtime.captureScreenshot).not.toHaveBeenCalled(); expect(data.fallbackScreenshotPath).toBeUndefined(); expect(data.artifacts).toBeUndefined(); }); test('internal observations stay silent so a polling wait cannot shoot once per poll', async () => { const input = await scenario(); - seed(SPARSE); + const runtime = seed(SPARSE); - const data = await dispatch(input, true); + const data = await dispatch(input, runtime, true); - expect(screenshotCalls()).toHaveLength(0); + expect(runtime.captureScreenshot).not.toHaveBeenCalled(); expect(data.fallbackScreenshotPath).toBeUndefined(); expect(data.artifacts).toBeUndefined(); }); test('a failed fallback screenshot does not fail the snapshot that was asked for', async () => { const input = await scenario(); - seed(SPARSE, async () => { - throw new Error('screenshot dispatch exploded'); + const runtime = seed(SPARSE, () => { + throw new Error('screenshot capture exploded'); }); - const data = await dispatch(input); + const data = await dispatch(input, runtime); const warnings = (data.warnings ?? []) as string[]; - expect(screenshotCalls()).toHaveLength(1); + expect(runtime.captureScreenshot).toHaveBeenCalledTimes(1); expect(data.fallbackScreenshotPath).toBeUndefined(); expect(data.artifacts).toBeUndefined(); // The manual remedy is still on the response, so the caller is not left without one. diff --git a/src/daemon/__tests__/viewport-runtime.test.ts b/src/daemon/__tests__/viewport-runtime.test.ts index 81bd554a12..0fd1c502f8 100644 --- a/src/daemon/__tests__/viewport-runtime.test.ts +++ b/src/daemon/__tests__/viewport-runtime.test.ts @@ -17,6 +17,7 @@ import { createTestDeviceInventoryGateways } from '../../__tests__/test-utils/de import { LeaseRegistry } from '../lease-registry.ts'; import { activateCompleteRefFrame } from '../ref-frame.ts'; import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import type { GenericPlatformExecutionParams } from '../request-generic-dispatch.ts'; import { resolveBoundViewportRuntime } from '../viewport-runtime.ts'; import { createRequestHandler } from './test-device-runtime-gateway.ts'; @@ -43,6 +44,26 @@ const unavailable = Object.freeze({ hint: 'viewport is not supported by the exact runtime owner', }); +/** Viewport's bound execution ignores the dispatcher's params; it closes over its own input. */ +function viewportExecutionParams(): GenericPlatformExecutionParams { + const session = makeSession('viewport-runtime', { device: webDevice }); + return { + session, + sessionName: session.name, + logPath: '/tmp/daemon.log', + command: 'viewport', + request: { + command: 'viewport', + positionals: ['1280', '900'], + token: 't', + session: session.name, + }, + positionals: ['1280', '900'], + out: undefined, + dispatchContext: {}, + }; +} + function runtimeHarness( fact: RuntimeOperationFact = available, device: typeof webDevice | typeof appleDevice = webDevice, @@ -86,13 +107,13 @@ test('resolves one admitted binding and exposes one normalized viewport operatio bindDevice: harness.bindDevice, }); - expect(resolved).toBeTypeOf('function'); - if (typeof resolved !== 'function') return; + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; expect(harness.inspectFacts).toHaveBeenCalledTimes(1); expect(harness.inspectFacts).toHaveBeenCalledWith(webDevice); expect(harness.bindDevice).toHaveBeenCalledTimes(1); expect(harness.bindDevice).toHaveBeenCalledWith(webDevice, viewportRuntimeUse); - expect(await resolved()).toEqual({ + expect(await resolved.execute(viewportExecutionParams())).toEqual({ width: 1280, height: 900, message: 'Viewport set: 1280x900', @@ -128,10 +149,13 @@ test('rejects an unavailable exact-owner fact before binding', async () => { expect(resolved).toEqual({ ok: false, - error: { - code: 'UNSUPPORTED_OPERATION', - message: 'viewport is not supported on this device', - hint: unavailable.hint, + response: { + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + message: 'viewport is not supported on this device', + hint: unavailable.hint, + }, }, }); expect(harness.inspectFacts).toHaveBeenCalledTimes(1); @@ -155,10 +179,13 @@ test('preserves the Apple viewport recovery hint through admission', async () => expect(resolved).toEqual({ ok: false, - error: { - code: 'UNSUPPORTED_OPERATION', - message: 'viewport is not supported on this device', - hint, + response: { + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + message: 'viewport is not supported on this device', + hint, + }, }, }); expect(harness.inspectFacts).toHaveBeenCalledOnce(); diff --git a/src/daemon/android-snapshot-timeout-evidence.ts b/src/daemon/android-snapshot-timeout-evidence.ts index 10d9fd35a1..f7f889e371 100644 --- a/src/daemon/android-snapshot-timeout-evidence.ts +++ b/src/daemon/android-snapshot-timeout-evidence.ts @@ -2,12 +2,16 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import type { DaemonResponse, SessionState } from './types.ts'; -import { dispatchCommand } from '../core/dispatch.ts'; import { emitDiagnostic } from '../utils/diagnostics.ts'; -import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; +import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; import { contextFromFlags } from './context.ts'; import { annotateScreenshotWithRefs } from './screenshot-overlay.ts'; +import { screenshotExecutionFromContext } from './screenshot-runtime.ts'; +import { + resolveBoundScreenshotRuntime, + type ScreenshotRuntimeBindings, +} from './screenshot-runtime-binding.ts'; type CapturedAndroidSnapshotTimeoutEvidenceBase = { path: string; @@ -37,13 +41,15 @@ type AndroidSnapshotTimeoutEvidence = overlayAnnotationError: string; }); -export async function maybeBuildAndroidSnapshotTimeoutFailure(params: { - error: unknown; - command: 'snapshot' | 'diff'; - logPath: string; - session: SessionState | undefined; - device: SessionState['device']; -}): Promise | undefined> { +export async function maybeBuildAndroidSnapshotTimeoutFailure( + params: { + error: unknown; + command: 'snapshot' | 'diff'; + logPath: string; + session: SessionState | undefined; + device: SessionState['device']; + } & ScreenshotRuntimeBindings, +): Promise | undefined> { if (params.command !== 'snapshot') return undefined; if (params.device.platform !== 'android') return undefined; @@ -62,36 +68,56 @@ export async function maybeBuildAndroidSnapshotTimeoutFailure(params: { }; } -async function captureAndroidSnapshotTimeoutEvidence(params: { - logPath: string; - session: SessionState | undefined; - device: SessionState['device']; -}): Promise { +async function captureAndroidSnapshotTimeoutEvidence( + params: { + logPath: string; + session: SessionState | undefined; + device: SessionState['device']; + } & ScreenshotRuntimeBindings, +): Promise { try { + const capture = await resolveBoundScreenshotRuntime({ + device: params.device, + overlayRefs: false, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!capture.ok) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'The target does not support screenshot capture, so no timeout evidence was taken.', + ); + } const tempDir = await fs.mkdtemp( path.join(os.tmpdir(), 'agent-device-android-snapshot-timeout-'), ); const screenshotPath = path.join(tempDir, 'snapshot-timeout-overlay-refs.png'); - const data = await dispatchCommand(params.device, 'screenshot', [screenshotPath], undefined, { - ...contextFromFlags( - params.logPath, - // Use a fresh unstabilized screenshot context; inheriting snapshot flags could repeat the - // accessibility stabilization timeout that this fallback is trying to avoid. - { screenshotNoStabilize: true }, - params.session?.appBundleId, - params.session?.trace?.outPath, + await capture.runtime.captureScreenshot({ + outPath: screenshotPath, + options: { + appBundleId: params.session?.appBundleId, + // Capture unstabilized: inheriting the snapshot's stabilization could repeat the + // accessibility timeout that this evidence path exists to escape. + stabilize: false, + surface: params.session?.surface, + }, + execution: screenshotExecutionFromContext( + contextFromFlags( + params.logPath, + { screenshotNoStabilize: true }, + params.session?.appBundleId, + params.session?.trace?.outPath, + ), ), - surface: params.session?.surface, }); - const resolvedPath = resolveCapturedScreenshotPath(data, screenshotPath); - await fs.access(resolvedPath); - const evidence = await annotateAndroidSnapshotTimeoutEvidence(resolvedPath, params.session); + await fs.access(screenshotPath); + const evidence = await annotateAndroidSnapshotTimeoutEvidence(screenshotPath, params.session); emitDiagnostic({ level: 'warn', phase: 'android_snapshot_timeout_screenshot_captured', data: { - path: resolvedPath, + path: screenshotPath, overlayRefCount: 'overlayRefCount' in evidence ? evidence.overlayRefCount : undefined, overlayRefsAnnotated: 'overlayRefsAnnotated' in evidence ? evidence.overlayRefsAnnotated : undefined, @@ -157,16 +183,6 @@ async function annotateAndroidSnapshotTimeoutEvidence( } } -function resolveCapturedScreenshotPath(data: unknown, fallbackPath: string): string { - return hasStringPath(data) ? data.path : fallbackPath; -} - -function hasStringPath(value: unknown): value is { path: string } { - return ( - typeof value === 'object' && value !== null && 'path' in value && typeof value.path === 'string' - ); -} - function isAndroidSnapshotTimeoutError(error: NormalizedError): boolean { if (error.code !== 'COMMAND_FAILED') return false; return ( diff --git a/src/daemon/generic-runtime-execution.ts b/src/daemon/generic-runtime-execution.ts new file mode 100644 index 0000000000..bfca5e1635 --- /dev/null +++ b/src/daemon/generic-runtime-execution.ts @@ -0,0 +1,28 @@ +import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; +import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts'; +import type { ScreenshotRuntimeBindings } from './screenshot-runtime-binding.ts'; +import type { DaemonRequest, SessionState } from './types.ts'; +import { resolveBoundViewportRuntime } from './viewport-runtime.ts'; + +/** + * The generic route's runtime-owned leaves (ADR 0019). Each one admits its own exact owner facts + * and binds once here, before the dispatcher runs, so the dispatcher itself never learns a command + * name. `undefined` means the leaf still executes through legacy platform dispatch. + */ +export async function resolveGenericRuntimeExecution( + params: Readonly<{ req: DaemonRequest; session: SessionState }> & ScreenshotRuntimeBindings, +): Promise { + switch (params.req.command) { + case 'screenshot': + return await resolveScreenshotGenericExecution(params); + case 'viewport': + return await resolveBoundViewportRuntime({ + device: params.session.device, + positionals: params.req.positionals ?? [], + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + default: + return undefined; + } +} diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index 65118fc228..6795cb782b 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -5,6 +5,7 @@ import { localRuntimeOwner, narrowDeviceBinding, providerRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, type AppDeploymentResult, type DeviceBinding, @@ -356,6 +357,7 @@ function sourceRuntimeFacts( customActions: unavailable, withoutActiveApp: unavailable, }), + ...screenshotRuntimeOperationFacts({ capture: unavailable }), setViewport: unavailable, deployApp: unavailable, materializeAppSource: materializationAvailable ? { available: true } : unavailable, diff --git a/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts b/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts index 1a85d45f92..423965078e 100644 --- a/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts +++ b/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts @@ -3,6 +3,7 @@ import { localRuntimeOwner, narrowDeviceBinding, providerRuntimeOwner, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, type DeviceBinding, type PlatformRuntimeOperations, @@ -24,6 +25,11 @@ export type CapabilitiesAdmissionRuntimeOptions = Readonly<{ sourceAvailable?: boolean; pushAvailable?: boolean; readinessAvailable?: boolean; + /** + * `screenshot` is fact-owned since R39, so the capabilities projection reads this cell instead + * of a capability bucket. Defaults to available, matching every device the bucket used to admit. + */ + screenshotAvailable?: boolean; }>; export const legacyCapabilityUses = [ @@ -52,12 +58,15 @@ function createAdmissionFacts( ): RuntimeFacts { const unavailable = unavailableOperationFact(options.providerMode); const available = { available: true } as const; - const appsFact = options.appsAvailable ? available : unavailable; + /** Every option-driven cell reads the same way: opted in means available for this fake owner. */ + const cell = (enabled: boolean | undefined) => (enabled ? available : unavailable); + const appsFact = cell(options.appsAvailable); + const screenshotFact = cell(options.screenshotAvailable !== false); return { device: { ...deviceShape(device), providerMode: options.providerMode }, operations: { ...unavailableApplicationLifecycleOperationFacts, - appLogInspect: options.appLogAvailable ? available : unavailable, + appLogInspect: cell(options.appLogAvailable), appLogDoctor: unavailable, appLogStart: unavailable, appLogReattach: unavailable, @@ -68,12 +77,13 @@ function createAdmissionFacts( customActions: unavailable, withoutActiveApp: unavailable, }), + ...screenshotRuntimeOperationFacts({ capture: screenshotFact }), setViewport: unavailable, - deployApp: options.deployAvailable ? available : unavailable, - materializeAppSource: options.sourceAvailable ? available : unavailable, - deployMaterializedApp: options.sourceAvailable ? available : unavailable, - sendPushNotification: options.pushAvailable ? available : unavailable, - networkDump: options.networkAvailable ? available : unavailable, + deployApp: cell(options.deployAvailable), + materializeAppSource: cell(options.sourceAvailable), + deployMaterializedApp: cell(options.sourceAvailable), + sendPushNotification: cell(options.pushAvailable), + networkDump: cell(options.networkAvailable), screenRecordingStart: unavailable, screenRecordingReattach: unavailable, screenRecordingCleanup: unavailable, diff --git a/src/daemon/handlers/__tests__/session-capabilities.test.ts b/src/daemon/handlers/__tests__/session-capabilities.test.ts index 6af61f14e6..a6189ce0da 100644 --- a/src/daemon/handlers/__tests__/session-capabilities.test.ts +++ b/src/daemon/handlers/__tests__/session-capabilities.test.ts @@ -17,6 +17,7 @@ import { localRuntimeOwner, narrowDeviceBinding, providerRuntimeOwner, + screenshotRuntimeOperationFacts, type DeviceBinding, type PlatformRuntimeOperations, type RuntimeOperationFact, @@ -500,6 +501,7 @@ function createAdmissionRuntime(options: { ensureReadyAvailable?: boolean; networkAvailable: boolean; appsAvailable?: boolean; + screenshotAvailable?: boolean; providerMode: RuntimeProviderMode; }) { const uses: Array<{ required: readonly string[]; preferred: readonly string[] }> = []; @@ -521,6 +523,8 @@ type AdmissionRuntimeOptions = Readonly<{ ensureReadyAvailable?: boolean; networkAvailable: boolean; appsAvailable?: boolean; + /** `screenshot` is fact-owned since R39; the projection reads this cell, not a bucket. */ + screenshotAvailable?: boolean; providerMode: RuntimeProviderMode; }>; @@ -583,6 +587,9 @@ function createAdmissionOperationFacts( ) { return { ...unavailableDeploymentSnapshotAndShutdownOperationFacts, + ...screenshotRuntimeOperationFacts({ + capture: options.screenshotAvailable === false ? unavailable : { available: true as const }, + }), appLogInspect: options.appLogAvailable ? { available: true as const } : unavailable, appLogDoctor: unavailable, appLogStart: unavailable, diff --git a/src/daemon/handlers/__tests__/session-command-harness.ts b/src/daemon/handlers/__tests__/session-command-harness.ts index 62dc20c82b..b762f1d357 100644 --- a/src/daemon/handlers/__tests__/session-command-harness.ts +++ b/src/daemon/handlers/__tests__/session-command-harness.ts @@ -6,6 +6,7 @@ import { applicationLifecycleOperationFacts, localRuntimeOwner, narrowDeviceBinding, + screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, type AppDeploymentInput, type AppDeploymentResult, @@ -138,6 +139,11 @@ function readinessFacts(device: DeviceInfo): RuntimeFacts { createUnavailablePlatformRuntimeFacts(device, localRuntimeOwner('apple'), { appLog: { available: false, reason: 'owner-capability-missing' }, network: { available: false, reason: 'owner-capability-missing' }, + screenshot: { available: false, reason: 'owner-capability-missing' }, viewport: { available: false, reason: 'owner-capability-missing' }, readiness: { available: false, reason: 'unsupported-device-kind' }, lifecycle: applicationLifecycleOperationFacts({ @@ -128,6 +129,7 @@ test('appstate rejects web before Android app-state backend dispatch', async () appLog: { available: false, reason: 'unsupported-platform-leaf' }, appState: { available: false, reason: 'unsupported-platform-leaf' }, network: { available: false, reason: 'unsupported-platform-leaf' }, + screenshot: { available: false, reason: 'unsupported-platform-leaf' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, readiness: { available: false, reason: 'unsupported-platform-leaf' }, lifecycle: applicationLifecycleOperationFacts({ diff --git a/src/daemon/handlers/__tests__/snapshot-handler.test.ts b/src/daemon/handlers/__tests__/snapshot-handler.test.ts index 7caa692d06..aeb37edf27 100644 --- a/src/daemon/handlers/__tests__/snapshot-handler.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-handler.test.ts @@ -1,7 +1,6 @@ import { test, expect, vi, afterEach, beforeEach } from 'vitest'; import fs from 'node:fs'; import path from 'node:path'; -import { PNG } from '../../../utils/png.ts'; import { handleSnapshotCommands as handleProductionSnapshotCommands } from '../snapshot.ts'; import { withSessionlessRunnerCleanup } from '../snapshot-session.ts'; import { captureSnapshot } from '../snapshot-capture.ts'; @@ -16,7 +15,10 @@ import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; import { snapshotCliOutput } from '../../../commands/capture/output.ts'; import type { CaptureSnapshotResult } from '@agent-device/contracts/client'; import { mkdtempForTestSync } from '../../../__tests__/test-utils/tmp-dir.ts'; -import { snapshotRuntimeFixture } from '../../__tests__/snapshot-runtime-fixture.ts'; +import { + fixtureScreenshotCaptures, + snapshotRuntimeFixture, +} from '../../__tests__/snapshot-runtime-fixture.ts'; import type { BindDeviceRuntime } from '../../request-runtime-binding.ts'; const dispatchCommandMock = vi.hoisted(() => vi.fn(async (..._args: unknown[]) => ({}))); @@ -178,17 +180,6 @@ beforeEach(() => { mockBuildIosOpenCommandHint.mockResolvedValue(undefined); }); -function writeSolidPng(filePath: string, width = 390, height = 844): void { - const png = new PNG({ width, height }); - for (let index = 0; index < png.data.length; index += 4) { - png.data[index] = 255; - png.data[index + 1] = 255; - png.data[index + 2] = 255; - png.data[index + 3] = 255; - } - fs.writeFileSync(filePath, PNG.sync.write(png)); -} - function makeAndroidTimeoutEvidenceSession(sessionName: string): SessionStore { const sessionStore = makeSessionStore(); const session = makeSession(sessionName, androidDevice); @@ -212,14 +203,8 @@ function makeAndroidTimeoutEvidenceSession(sessionName: string): SessionStore { } function mockAndroidTimeoutEvidenceDispatch(): void { - mockDispatch.mockImplementation(async (_device, command, positionals, _out, context) => { + mockDispatch.mockImplementation(async (_device, command) => { if (command === 'snapshot') throw androidSnapshotTimeoutError(); - if (command === 'screenshot') { - const screenshotPath = positionals[0]!; - expect(context?.screenshotNoStabilize).toBe(true); - writeSolidPng(screenshotPath); - return { path: screenshotPath }; - } return {}; }); } @@ -884,7 +869,8 @@ test('snapshot timeout captures Android screenshot evidence with overlay refs', sessionStore, }); expectAndroidTimeoutEvidence(response); - expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['snapshot', 'screenshot']); + expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['snapshot']); + expect(fixtureScreenshotCaptures.at(-1)?.options).toMatchObject({ stabilize: false }); }); test('snapshot warns when recent snapshot node count collapses sharply', async () => { diff --git a/src/daemon/handlers/session-inventory.ts b/src/daemon/handlers/session-inventory.ts index 90fee37a7e..5abc002e78 100644 --- a/src/daemon/handlers/session-inventory.ts +++ b/src/daemon/handlers/session-inventory.ts @@ -281,6 +281,7 @@ const factOwnedCapabilityOperations: Readonly< close: ['closeApplication', 'finalizeApplicationClose'], prepare: ['prepareAppleRunner'], runtime: ['clearRuntimeHints'], + screenshot: ['captureScreenshot'], viewport: ['setViewport'], }); diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts index 5a9abcf905..d37ec6bd2e 100644 --- a/src/daemon/request-generic-dispatch.ts +++ b/src/daemon/request-generic-dispatch.ts @@ -1,41 +1,67 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { SettleObservation } from '@agent-device/contracts/interaction'; -import { commandSupportsSettleObservation } from '../core/command-descriptor/registry.ts'; +import { + commandSupportsSettleObservation, + commandUsesDeviceRuntimeExecution, +} from '../core/command-descriptor/registry.ts'; import { dispatchCommand } from '../core/dispatch.ts'; import { requireCommandSupported } from './handlers/response.ts'; -import { SessionStore } from './session-store.ts'; +import type { SessionStore } from './session-store.ts'; import type { DaemonCommandContext } from './context.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; -import { buildSnapshotState, captureSnapshotData } from './handlers/snapshot-capture.ts'; -import { setSessionSnapshot } from './session-snapshot.ts'; -import { - dispatchScreenshotViaRuntime, - type ScreenshotOutputPlacement, -} from './screenshot-runtime.ts'; import { ensureAndroidBlockingSystemDialogReady, recoverAndroidBlockingSystemDialog, } from './android-system-dialog.ts'; -import { annotateScreenshotWithRefs } from './screenshot-overlay.ts'; import { markDeferredInteractionOutcome } from './deferred-interaction-outcome.ts'; import { augmentScrollVisualizationResult, recordTouchVisualizationEvent, } from './recording-gestures.ts'; -import { AppError, normalizeError } from '@agent-device/kernel/errors'; -import { retiredScreenshotMaxSizeFlagError } from '@agent-device/contracts/capture'; +import { normalizeError } from '@agent-device/kernel/errors'; import { expireRefFrame } from './ref-frame.ts'; import { resolveRefFrameEffect, shouldGuardAndroidBlockingDialog, } from './daemon-command-registry.ts'; import { isActiveProviderDevice } from '../provider-device-runtime.ts'; -import { - assertSupportedScreenshotPixelDensity, - readScreenshotResultMetadata, -} from '../utils/screenshot-density.ts'; import { buildActionEventResult } from './session-event-action-presentation.ts'; +export type GenericPlatformExecutionParams = { + session: SessionState; + sessionName: string; + logPath: string; + command: string; + request: DaemonRequest; + positionals: string[]; + out: string | undefined; + dispatchContext: DaemonCommandContext; +}; + +/** + * What actually performs a generic leaf's platform work. Legacy leaves get + * {@link executeGenericPlatformCommand}; a command migrated onto a request-bound device runtime + * supplies its own already-admitted, already-bound closure instead (ADR 0019). + */ +export type GenericPlatformExecution = ( + params: GenericPlatformExecutionParams, +) => Promise | void>; + +/** + * What the session action records for this request. A runtime-owned leaf may normalize its + * arguments (a screenshot destination is a user-typed path) and records the normalized form. + */ +export type RecordedGenericRequest = Readonly<{ + positionals: string[]; + flags: Record; +}>; + +/** What a runtime-owned generic leaf resolves to before the dispatcher runs: a refusal, or the + * bound execution plus whatever the session action should record for it. */ +export type ResolvedGenericExecution = + | Readonly<{ ok: false; response: DaemonResponse }> + | Readonly<{ ok: true; execute: GenericPlatformExecution; recorded?: RecordedGenericRequest }>; + export async function dispatchGenericCommand(params: { req: DaemonRequest; session: SessionState; @@ -47,7 +73,8 @@ export async function dispatchGenericCommand(params: { appBundleId?: string, traceLogPath?: string, ) => DaemonCommandContext; - executePlatformCommand: typeof executeGenericPlatformCommand; + executePlatformCommand: GenericPlatformExecution; + recordedRequest?: RecordedGenericRequest; }): Promise { const { req, session, logPath, sessionStore, contextFromFlags } = params; const platformCommand = req.command; @@ -70,8 +97,11 @@ export async function dispatchGenericCommand(params: { const preflightReadiness = await ensureNoAndroidBlockingDialogReady(session, platformCommand); if ('response' in preflightReadiness) return preflightReadiness.response; - const { resolvedPositionals, resolvedOut, recordedPositionals, recordedFlags } = - resolveCommandPositionals(req); + const resolvedPositionals = req.positionals ?? []; + const recorded = params.recordedRequest ?? { + positionals: resolvedPositionals, + flags: req.flags ?? {}, + }; const actionStartedAt = Date.now(); const dispatchContext = { @@ -86,50 +116,73 @@ export async function dispatchGenericCommand(params: { if (resolveRefFrameEffect(req) === 'may-invalidate') { expireRefFrame(session); } - let data = await params.executePlatformCommand({ + const data = await params.executePlatformCommand({ session, sessionName: params.sessionName, logPath, command: platformCommand, request: req, positionals: resolvedPositionals, - out: resolvedOut, + out: req.flags?.out, dispatchContext, }); + return await finalizeGenericCommand({ + req, + session, + sessionStore, + command: platformCommand, + resolvedPositionals, + recorded, + data, + actionStartedAt, + preflightReadiness, + observeSettle: settlePlan.observe, + }); +} + +/** + * Everything a generic leaf owes after its platform work returns: the post-command dialog check, + * the recovered-dialog warning, the recorded action, deferred interaction markers, and the opt-in + * settle observation — in that order, because each one depends on the previous having happened. + */ +async function finalizeGenericCommand(params: { + req: DaemonRequest; + session: SessionState; + sessionStore: SessionStore; + command: string; + resolvedPositionals: string[]; + recorded: RecordedGenericRequest; + data: Record | void; + actionStartedAt: number; + preflightReadiness: AndroidDialogReadiness; + observeSettle?: () => Promise; +}): Promise { + const { req, session, sessionStore, command } = params; const postflightReadiness = await ensureNoAndroidBlockingDialogReady( session, - platformCommand, + command, 'after-command', ); if ('response' in postflightReadiness) return postflightReadiness.response; - if ( - 'status' in preflightReadiness && - preflightReadiness.status === 'recovered' && - (!data || typeof data === 'object') - ) { - data ??= {}; - data.warning = preflightReadiness.warning; - } - const actionFinishedAt = Date.now(); + let data = withRecoveredDialogWarning(params.data, params.preflightReadiness); recordVisualizationAndAction({ session, sessionStore, - command: platformCommand, - resolvedPositionals, - recordedPositionals, - recordedFlags, + command, + resolvedPositionals: params.resolvedPositionals, + recorded: params.recorded, data, - actionStartedAt, - actionFinishedAt, + actionStartedAt: params.actionStartedAt, + actionFinishedAt: Date.now(), flags: req.flags ?? {}, clientArtifactPaths: req.meta?.clientArtifactPaths, }); markDeferredInteractionOutcome({ session, - command: platformCommand, - positionals: resolvedPositionals, + command, + positionals: params.resolvedPositionals, flags: req.flags, }); @@ -137,14 +190,24 @@ export async function dispatchGenericCommand(params: { // in the #1542 post-gesture stabilization, and after the recorded action so // the session history keeps the ACTION's own timing rather than the // observation wait that followed it. - if (settlePlan.observe) { - const settle = await settlePlan.observe(); + if (params.observeSettle) { + const settle = await params.observeSettle(); if (settle) data = { ...(data ?? {}), settle }; } return { ok: true, data: data ?? {} }; } +/** A dialog the preflight had to dismiss is disclosed on the result it made possible. */ +function withRecoveredDialogWarning( + data: Record | void, + preflight: AndroidDialogReadiness, +): Record | void { + if (!('status' in preflight) || preflight.status !== 'recovered') return data; + if (data && typeof data !== 'object') return data; + return { ...(data ?? {}), warning: preflight.warning }; +} + /** * `--settle` (#1638) is opt-in and its observation runs the interaction * runtime — a subgraph this dispatcher otherwise never touches, and one that a @@ -181,13 +244,16 @@ function usesSettleFlags(flags: CommandFlags | undefined): boolean { return flags?.settle === true || flags?.settleQuietMs !== undefined; } +type AndroidDialogReadiness = + | { status: 'clear' } + | { status: 'recovered'; warning: string } + | { response: DaemonResponse }; + async function ensureNoAndroidBlockingDialogReady( session: SessionState, platformCommand: string, phase: 'before-command' | 'after-command' = 'before-command', -): Promise< - { status: 'clear' } | { status: 'recovered'; warning: string } | { response: DaemonResponse } -> { +): Promise { if (session.device.platform !== 'android' || !shouldGuardAndroidBlockingDialog(platformCommand)) { return { status: 'clear' }; } @@ -209,10 +275,11 @@ async function ensureGenericCommandReady( session: SessionState, platformCommand: string, ): Promise { - const unsupported = - platformCommand === 'viewport' - ? null - : requireCommandSupported(platformCommand, session.device, { hint: true }); + // A device-runtime command has no capability bucket: its exact owner facts already admitted it + // (or refused it) before this route was reached. + const unsupported = commandUsesDeviceRuntimeExecution(platformCommand) + ? null + : requireCommandSupported(platformCommand, session.device, { hint: true }); if (unsupported) return unsupported; if ( session.device.platform !== 'android' || @@ -232,156 +299,19 @@ async function ensureGenericCommandReady( }; } -export async function executeGenericPlatformCommand(params: { - session: SessionState; - sessionName: string; - logPath: string; - command: string; - request: DaemonRequest; - positionals: string[]; - out: string | undefined; - dispatchContext: DaemonCommandContext; -}): Promise | void> { +export const executeGenericPlatformCommand: GenericPlatformExecution = async (params) => { const { session, command, positionals, out, dispatchContext } = params; - if (command === 'screenshot') { - return await executeScreenshotPlatformCommand(params); - } return await dispatchCommand(session.device, command, positionals, out, { ...dispatchContext, }); -} - -async function executeScreenshotPlatformCommand(params: { - session: SessionState; - sessionName: string; - logPath: string; - request: DaemonRequest; - positionals: string[]; - out: string | undefined; - dispatchContext: DaemonCommandContext; -}): Promise> { - const { session, request, positionals, out, dispatchContext } = params; - const retiredMaxSize = retiredScreenshotMaxSizeFlagError('screenshot', request.flags); - if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); - assertSupportedScreenshotPixelDensity(session.device, request.flags?.screenshotPixelDensity); - const data = await dispatchScreenshotViaRuntime({ - session, - sessionName: params.sessionName, - outPath: positionals[0] ?? out, - outputPlacement: resolveScreenshotOutputPlacement(request), - dispatchContext, - }); - if (typeof data.path !== 'string') { - return data; - } - if (request.flags?.overlayRefs) { - await applyScreenshotOverlay(session, data, params.logPath); - } - Object.assign( - data, - await readScreenshotResultMetadata({ - device: session.device, - path: data.path, - requestedPixelDensity: request.flags?.screenshotPixelDensity, - scale: request.flags?.screenshotScale, - }), - ); - return data; -} - -function resolveScreenshotOutputPlacement(req: DaemonRequest): ScreenshotOutputPlacement { - if (req.command !== 'screenshot') return 'default'; - if ((req.positionals ?? [])[0]) return 'positional'; - if (req.flags?.out) return 'out'; - return 'default'; -} - -function resolveCommandPositionals(req: DaemonRequest): { - resolvedPositionals: string[]; - resolvedOut: string | undefined; - recordedPositionals: string[]; - recordedFlags: Record; -} { - return req.command === 'screenshot' - ? resolveScreenshotCommandPositionals(req) - : resolveDefaultCommandPositionals(req); -} - -function resolveDefaultCommandPositionals(req: DaemonRequest): { - resolvedPositionals: string[]; - resolvedOut: string | undefined; - recordedPositionals: string[]; - recordedFlags: Record; -} { - const positionals = req.positionals ?? []; - return { - resolvedPositionals: positionals, - resolvedOut: req.flags?.out, - recordedPositionals: positionals, - recordedFlags: req.flags ?? {}, - }; -} - -function resolveScreenshotCommandPositionals(req: DaemonRequest): { - resolvedPositionals: string[]; - resolvedOut: string | undefined; - recordedPositionals: string[]; - recordedFlags: Record; -} { - const positionals = req.positionals ?? []; - const resolvedPositionals = resolveScreenshotPositionals(positionals, req.meta?.cwd); - const resolvedOut = resolveScreenshotOut(req.flags?.out, req.meta?.cwd); - const recordedPositionals = resolvedPositionals; - const recordedFlags = resolvedOut - ? { ...(req.flags ?? {}), out: resolvedOut } - : (req.flags ?? {}); - return { resolvedPositionals, resolvedOut, recordedPositionals, recordedFlags }; -} - -function resolveScreenshotPositionals(positionals: string[], cwd: string | undefined): string[] { - const outPath = positionals[0]; - if (!outPath) return positionals; - return [SessionStore.expandHome(outPath, cwd), ...positionals.slice(1)]; -} - -function resolveScreenshotOut( - out: string | undefined, - cwd: string | undefined, -): string | undefined { - return out ? SessionStore.expandHome(out, cwd) : out; -} - -async function applyScreenshotOverlay( - session: SessionState, - data: Record, - logPath: string, -): Promise { - const overlaySnapshotFlags = { - snapshotInteractiveOnly: true, - } satisfies CommandFlags; - const overlaySnapshotData = await captureSnapshotData({ - device: session.device, - session, - flags: overlaySnapshotFlags, - logPath, - snapshotScope: undefined, - }); - const overlaySnapshot = buildSnapshotState(overlaySnapshotData, overlaySnapshotFlags); - setSessionSnapshot(session, overlaySnapshot); - const overlayRefs = await annotateScreenshotWithRefs({ - screenshotPath: data.path as string, - snapshot: overlaySnapshot, - }); - data.overlayRefs = overlayRefs; -} +}; function recordVisualizationAndAction(params: { session: SessionState; sessionStore: SessionStore; command: string; resolvedPositionals: string[]; - recordedPositionals: string[]; - recordedFlags: Record; + recorded: RecordedGenericRequest; data: Record | void; actionStartedAt: number; actionFinishedAt: number; @@ -393,8 +323,7 @@ function recordVisualizationAndAction(params: { sessionStore, command, resolvedPositionals, - recordedPositionals, - recordedFlags, + recorded, data, actionStartedAt, actionFinishedAt, @@ -418,8 +347,8 @@ function recordVisualizationAndAction(params: { ); sessionStore.recordAction(session, { command, - positionals: recordedPositionals, - flags: recordedFlags, + positionals: recorded.positionals, + flags: recorded.flags, result: buildActionEventResult({ command, meta: { clientArtifactPaths } }, data ?? {}), }); } diff --git a/src/daemon/request-router.ts b/src/daemon/request-router.ts index bad91a41ec..96a2818752 100644 --- a/src/daemon/request-router.ts +++ b/src/daemon/request-router.ts @@ -67,7 +67,7 @@ import { createScreenRecordingAdmissionLedger, type ScreenRecordingAdmissionLedger, } from './screen-recording-admission-ledger.ts'; -import { resolveBoundViewportRuntime } from './viewport-runtime.ts'; +import { resolveGenericRuntimeExecution } from './generic-runtime-execution.ts'; // --------------------------------------------------------------------------- // Request handler API @@ -426,16 +426,13 @@ async function dispatchGenericForLockedScope(params: { return noActiveSessionError(); } - const viewportRuntime = - lockedScope.req.command === 'viewport' - ? await resolveBoundViewportRuntime({ - device: session.device, - positionals: lockedScope.req.positionals ?? [], - inspectFacts: lockedScope.inspectFacts, - bindDevice: lockedScope.bindDevice, - }) - : undefined; - if (viewportRuntime && typeof viewportRuntime !== 'function') return viewportRuntime; + const runtimeExecution = await resolveGenericRuntimeExecution({ + req: lockedScope.req, + session, + inspectFacts: lockedScope.inspectFacts, + bindDevice: lockedScope.bindDevice, + }); + if (runtimeExecution && !runtimeExecution.ok) return runtimeExecution.response; const { dispatchGenericCommand, executeGenericPlatformCommand } = await loadGenericRequestHandlerModule(); @@ -446,7 +443,8 @@ async function dispatchGenericForLockedScope(params: { logPath, sessionStore, contextFromFlags: lockedScope.contextFromFlags, - executePlatformCommand: viewportRuntime ?? executeGenericPlatformCommand, + executePlatformCommand: runtimeExecution?.execute ?? executeGenericPlatformCommand, + ...(runtimeExecution?.recorded ? { recordedRequest: runtimeExecution.recorded } : {}), }); return dispatchResponse; } diff --git a/src/daemon/screenshot-overlay-draw.ts b/src/daemon/screenshot-overlay-draw.ts new file mode 100644 index 0000000000..e228a5d7e2 --- /dev/null +++ b/src/daemon/screenshot-overlay-draw.ts @@ -0,0 +1,153 @@ +import type { Rect, ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; +import type { PNG } from '../utils/png.ts'; +import { clamp } from './screenshot-overlay-rects.ts'; + +/** + * Rasterizing one overlay ref onto a decoded PNG: border, badge, and the bitmap glyphs the badge + * needs. Which node earns a ref, and where its rect lands, is `screenshot-overlay.ts`'s question — + * this module only paints what it is handed. + */ +const BORDER_COLOR = [255, 59, 48, 255] as const; +const BADGE_COLOR = [255, 214, 10, 255] as const; +const TEXT_COLOR = [0, 0, 0, 255] as const; +const FONT_WIDTH = 5; +const FONT_HEIGHT = 7; +const FONT_SPACING = 1; +const BADGE_PADDING_X = 3; +const BADGE_PADDING_Y = 2; +const BADGE_MARGIN = 2; +const BORDER_THICKNESS = 2; +const FONT: Record = { + e: ['01110', '10000', '11110', '10000', '10000', '10001', '01110'], + '0': ['01110', '10001', '10011', '10101', '11001', '10001', '01110'], + '1': ['00100', '01100', '00100', '00100', '00100', '00100', '01110'], + '2': ['01110', '10001', '00001', '00010', '00100', '01000', '11111'], + '3': ['11110', '00001', '00001', '01110', '00001', '00001', '11110'], + '4': ['00010', '00110', '01010', '10010', '11111', '00010', '00010'], + '5': ['11111', '10000', '10000', '11110', '00001', '00001', '11110'], + '6': ['01110', '10000', '10000', '11110', '10001', '10001', '01110'], + '7': ['11111', '00001', '00010', '00100', '01000', '01000', '01000'], + '8': ['01110', '10001', '10001', '01110', '10001', '10001', '01110'], + '9': ['01110', '10001', '10001', '01111', '00001', '00001', '01110'], +} as const; +// Badges currently render only `eN` refs, so the bitmap font intentionally covers `e` and digits. + +export function drawOverlayRef(png: PNG, overlayRef: ScreenshotOverlayRef): void { + drawRectBorder(png, overlayRef.overlayRect, BORDER_COLOR, BORDER_THICKNESS); + drawBadge(png, overlayRef.overlayRect, overlayRef.ref); +} + +function drawRectBorder( + png: PNG, + rect: Rect, + color: readonly [number, number, number, number], + thickness: number, +): void { + for (let offset = 0; offset < thickness; offset += 1) { + drawHorizontalLine(png, rect.x, rect.x + rect.width - 1, rect.y + offset, color); + drawHorizontalLine( + png, + rect.x, + rect.x + rect.width - 1, + rect.y + rect.height - 1 - offset, + color, + ); + drawVerticalLine(png, rect.x + offset, rect.y, rect.y + rect.height - 1, color); + drawVerticalLine( + png, + rect.x + rect.width - 1 - offset, + rect.y, + rect.y + rect.height - 1, + color, + ); + } +} + +function drawBadge(png: PNG, rect: Rect, text: string): void { + const badgeWidth = + BADGE_PADDING_X * 2 + text.length * FONT_WIDTH + Math.max(0, text.length - 1) * FONT_SPACING; + const badgeHeight = BADGE_PADDING_Y * 2 + FONT_HEIGHT; + const x = clamp(rect.x, 0, Math.max(0, png.width - badgeWidth)); + const preferredY = rect.y - badgeHeight - BADGE_MARGIN; + const y = + preferredY >= 0 + ? preferredY + : clamp(rect.y + BADGE_MARGIN, 0, Math.max(0, png.height - badgeHeight)); + fillRect(png, x, y, badgeWidth, badgeHeight, BADGE_COLOR); + drawText(png, x + BADGE_PADDING_X, y + BADGE_PADDING_Y, text, TEXT_COLOR); +} + +function drawText( + png: PNG, + x: number, + y: number, + text: string, + color: readonly [number, number, number, number], +): void { + let cursorX = x; + for (const character of text.toLowerCase()) { + const glyph = FONT[character]; + if (glyph) { + for (let row = 0; row < glyph.length; row += 1) { + for (let column = 0; column < glyph[row]!.length; column += 1) { + if (glyph[row]![column] !== '1') continue; + setPixel(png, cursorX + column, y + row, color); + } + } + } + cursorX += FONT_WIDTH + FONT_SPACING; + } +} + +function fillRect( + png: PNG, + x: number, + y: number, + width: number, + height: number, + color: readonly [number, number, number, number], +): void { + for (let row = 0; row < height; row += 1) { + for (let column = 0; column < width; column += 1) { + setPixel(png, x + column, y + row, color); + } + } +} + +function drawHorizontalLine( + png: PNG, + startX: number, + endX: number, + y: number, + color: readonly [number, number, number, number], +): void { + for (let x = startX; x <= endX; x += 1) { + setPixel(png, x, y, color); + } +} + +function drawVerticalLine( + png: PNG, + x: number, + startY: number, + endY: number, + color: readonly [number, number, number, number], +): void { + for (let y = startY; y <= endY; y += 1) { + setPixel(png, x, y, color); + } +} + +function setPixel( + png: PNG, + x: number, + y: number, + color: readonly [number, number, number, number], +): void { + if (x < 0 || y < 0 || x >= png.width || y >= png.height) return; + const index = (png.width * y + x) * 4; + png.data[index] = color[0]; + png.data[index + 1] = color[1]; + png.data[index + 2] = color[2]; + png.data[index + 3] = color[3]; +} diff --git a/src/daemon/screenshot-overlay-rects.ts b/src/daemon/screenshot-overlay-rects.ts index 24a9dea598..a5590689ee 100644 --- a/src/daemon/screenshot-overlay-rects.ts +++ b/src/daemon/screenshot-overlay-rects.ts @@ -7,6 +7,11 @@ export function hasPositiveRect(rect: Rect | undefined): rect is Rect { return Boolean(rect && rect.width > 0 && rect.height > 0); } +export function clamp(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.max(min, Math.min(max, value)); +} + export function rectArea(rect: Rect): number { return rect.width * rect.height; } diff --git a/src/daemon/screenshot-overlay.ts b/src/daemon/screenshot-overlay.ts index 185177f935..feca236320 100644 --- a/src/daemon/screenshot-overlay.ts +++ b/src/daemon/screenshot-overlay.ts @@ -6,7 +6,6 @@ import { type SnapshotNode, type SnapshotState, } from '@agent-device/kernel/snapshot'; -import type { PNG } from '../utils/png.ts'; import { decodePngAsync, encodePngAsync } from '../utils/png-worker-client.ts'; import { analyzeReactNativeOverlay } from '../core/react-native-overlay.ts'; import { @@ -15,19 +14,10 @@ import { normalizeType, } from '@agent-device/contracts/snapshot'; import { resolveAndroidOverlaySourceRect } from './screenshot-overlay-android.ts'; -import { hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts'; +import { drawOverlayRef } from './screenshot-overlay-draw.ts'; +import { clamp, hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts'; const MAX_OVERLAY_REFS = 24; -const BORDER_COLOR = [255, 59, 48, 255] as const; -const BADGE_COLOR = [255, 214, 10, 255] as const; -const TEXT_COLOR = [0, 0, 0, 255] as const; -const FONT_WIDTH = 5; -const FONT_HEIGHT = 7; -const FONT_SPACING = 1; -const BADGE_PADDING_X = 3; -const BADGE_PADDING_Y = 2; -const BADGE_MARGIN = 2; -const BORDER_THICKNESS = 2; const ANDROID_UNLABELED_CLICKABLE_EXCLUDED_TYPES = [ 'scroll', 'list', @@ -49,21 +39,6 @@ const ACTIONABLE_ROLE_TYPES = [ 'cell', ] as const; -const FONT: Record = { - e: ['01110', '10000', '11110', '10000', '10000', '10001', '01110'], - '0': ['01110', '10001', '10011', '10101', '11001', '10001', '01110'], - '1': ['00100', '01100', '00100', '00100', '00100', '00100', '01110'], - '2': ['01110', '10001', '00001', '00010', '00100', '01000', '11111'], - '3': ['11110', '00001', '00001', '01110', '00001', '00001', '11110'], - '4': ['00010', '00110', '01010', '10010', '11111', '00010', '00010'], - '5': ['11111', '10000', '10000', '11110', '00001', '00001', '11110'], - '6': ['01110', '10000', '10000', '11110', '10001', '10001', '01110'], - '7': ['11111', '00001', '00010', '00100', '01000', '01000', '01000'], - '8': ['01110', '10001', '10001', '01110', '10001', '10001', '01110'], - '9': ['01110', '10001', '10001', '01111', '00001', '00001', '01110'], -} as const; -// Badges currently render only `eN` refs, so the bitmap font intentionally covers `e` and digits. - type OverlayCandidate = Omit & { score: number; }; @@ -494,130 +469,6 @@ function clampRect(rect: Rect, width: number, height: number): Rect { }; } -function clamp(value: number, min: number, max: number): number { - if (!Number.isFinite(value)) return min; - return Math.max(min, Math.min(max, value)); -} - -function drawOverlayRef(png: PNG, overlayRef: ScreenshotOverlayRef): void { - drawRectBorder(png, overlayRef.overlayRect, BORDER_COLOR, BORDER_THICKNESS); - drawBadge(png, overlayRef.overlayRect, overlayRef.ref); -} - -function drawRectBorder( - png: PNG, - rect: Rect, - color: readonly [number, number, number, number], - thickness: number, -): void { - for (let offset = 0; offset < thickness; offset += 1) { - drawHorizontalLine(png, rect.x, rect.x + rect.width - 1, rect.y + offset, color); - drawHorizontalLine( - png, - rect.x, - rect.x + rect.width - 1, - rect.y + rect.height - 1 - offset, - color, - ); - drawVerticalLine(png, rect.x + offset, rect.y, rect.y + rect.height - 1, color); - drawVerticalLine( - png, - rect.x + rect.width - 1 - offset, - rect.y, - rect.y + rect.height - 1, - color, - ); - } -} - -function drawBadge(png: PNG, rect: Rect, text: string): void { - const badgeWidth = - BADGE_PADDING_X * 2 + text.length * FONT_WIDTH + Math.max(0, text.length - 1) * FONT_SPACING; - const badgeHeight = BADGE_PADDING_Y * 2 + FONT_HEIGHT; - const x = clamp(rect.x, 0, Math.max(0, png.width - badgeWidth)); - const preferredY = rect.y - badgeHeight - BADGE_MARGIN; - const y = - preferredY >= 0 - ? preferredY - : clamp(rect.y + BADGE_MARGIN, 0, Math.max(0, png.height - badgeHeight)); - fillRect(png, x, y, badgeWidth, badgeHeight, BADGE_COLOR); - drawText(png, x + BADGE_PADDING_X, y + BADGE_PADDING_Y, text, TEXT_COLOR); -} - -function drawText( - png: PNG, - x: number, - y: number, - text: string, - color: readonly [number, number, number, number], -): void { - let cursorX = x; - for (const character of text.toLowerCase()) { - const glyph = FONT[character]; - if (glyph) { - for (let row = 0; row < glyph.length; row += 1) { - for (let column = 0; column < glyph[row]!.length; column += 1) { - if (glyph[row]![column] !== '1') continue; - setPixel(png, cursorX + column, y + row, color); - } - } - } - cursorX += FONT_WIDTH + FONT_SPACING; - } -} - -function fillRect( - png: PNG, - x: number, - y: number, - width: number, - height: number, - color: readonly [number, number, number, number], -): void { - for (let row = 0; row < height; row += 1) { - for (let column = 0; column < width; column += 1) { - setPixel(png, x + column, y + row, color); - } - } -} - -function drawHorizontalLine( - png: PNG, - startX: number, - endX: number, - y: number, - color: readonly [number, number, number, number], -): void { - for (let x = startX; x <= endX; x += 1) { - setPixel(png, x, y, color); - } -} - -function drawVerticalLine( - png: PNG, - x: number, - startY: number, - endY: number, - color: readonly [number, number, number, number], -): void { - for (let y = startY; y <= endY; y += 1) { - setPixel(png, x, y, color); - } -} - -function setPixel( - png: PNG, - x: number, - y: number, - color: readonly [number, number, number, number], -): void { - if (x < 0 || y < 0 || x >= png.width || y >= png.height) return; - const index = (png.width * y + x) * 4; - png.data[index] = color[0]; - png.data[index + 1] = color[1]; - png.data[index + 2] = color[2]; - png.data[index + 3] = color[3]; -} function compareOverlayCandidatesByPosition( left: OverlayCandidate, right: OverlayCandidate, diff --git a/src/daemon/screenshot-runtime-binding.ts b/src/daemon/screenshot-runtime-binding.ts new file mode 100644 index 0000000000..ce69a66ef3 --- /dev/null +++ b/src/daemon/screenshot-runtime-binding.ts @@ -0,0 +1,118 @@ +import { + resolveScreenshotRuntimePlan, + type CaptureScreenshotInput, + type CaptureSnapshotInput, + type RuntimeOperationFact, + type ScreenshotRuntimeOperations, + type ScreenshotRuntimePlan, + type SnapshotResult, + type SnapshotRuntimeOperations, +} from '@agent-device/contracts/platform'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { errorResponse } from './handlers/response.ts'; +import { + admitRuntimePlan, + requireRuntimeBinding, + unavailableRuntimeOperationResponse, + unwrapAdmittedRuntimePlan, + type AdmittedRuntimePlan, +} from './handlers/session-runtime-admission.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; +import type { DaemonResponse } from './types.ts'; + +export type ScreenshotRuntimeBindings = Readonly<{ + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; +}>; + +/** + * One request's admitted capture authority. `captureSnapshot` is present exactly when the + * overlay-refs plan was admitted, so a caller cannot annotate a capture it never declared — and + * both operations come from the same single binding. + */ +export type BoundScreenshotRuntime = Readonly<{ + captureScreenshot(input: CaptureScreenshotInput): Promise; + captureSnapshot?: (input: CaptureSnapshotInput) => Promise; +}>; + +export type ResolvedScreenshotRuntime = + | Readonly<{ ok: true; runtime: BoundScreenshotRuntime }> + | Readonly<{ ok: false; response: DaemonResponse }>; + +/** Resolves one plan, inspects its owner facts once, then binds once on the admitted device. */ +export async function resolveBoundScreenshotRuntime( + params: Readonly<{ device: DeviceInfo; overlayRefs: boolean }> & ScreenshotRuntimeBindings, +): Promise { + const plan = resolveScreenshotRuntimePlan({ overlayRefs: params.overlayRefs }); + const admission = await admitRuntimePlan({ + device: params.device, + plan, + inspectFacts: params.inspectFacts, + }); + if (!admission.admitted) { + return { + ok: false, + response: screenshotPlanUnavailableResponse(admission.operation, admission.fact), + }; + } + return { ok: true, runtime: await bindScreenshotRuntime(params.bindDevice, admission) }; +} + +/** + * Binds only an admitted plan, on the device it was admitted for: the token is minted by + * `admitRuntimePlan` alone and unwrapped by exact identity, so nothing that was not admitted can + * reach the capture operations. + */ +async function bindScreenshotRuntime( + bindDevice: BindDeviceRuntime | undefined, + admission: AdmittedRuntimePlan, +): Promise { + const bind = requireRuntimeBinding(bindDevice); + const { device, plan } = unwrapAdmittedRuntimePlan(admission); + switch (plan.kind) { + case 'capture': { + const runtime = await bind(device, plan.use); + return Object.freeze({ captureScreenshot: selectScreenshotCapture(runtime) }); + } + case 'capture-with-overlay-refs': { + const runtime = await bind(device, plan.use); + return Object.freeze({ + captureScreenshot: selectScreenshotCapture(runtime), + captureSnapshot: selectOverlaySnapshot(runtime), + }); + } + } +} + +type BoundScreenshotOperation = Readonly<{ + operations: Readonly>; +}>; + +/** The one narrowed capture call every screenshot plan funnels through. */ +function selectScreenshotCapture(runtime: BoundScreenshotOperation<'captureScreenshot'>) { + return async (input: CaptureScreenshotInput) => await runtime.operations.captureScreenshot(input); +} + +/** The overlay plan's tree read, from the same binding as its capture. */ +function selectOverlaySnapshot( + runtime: Readonly<{ operations: Readonly> }>, +) { + return async (input: CaptureSnapshotInput) => await runtime.operations.captureSnapshot(input); +} + +function screenshotPlanUnavailableResponse( + operation: ScreenshotRuntimePlan['use']['required'][number], + fact: RuntimeOperationFact, +): DaemonResponse { + if (operation === 'captureScreenshot') { + return unavailableRuntimeOperationResponse('screenshot', fact)!; + } + return errorResponse( + 'UNSUPPORTED_OPERATION', + '--overlay-refs annotates a capture with the refs of a snapshot taken on the same screen, which this target cannot capture.', + { reason: fact.available ? undefined : fact.reason }, + { + hint: (fact.available ? undefined : fact.hint) ?? 'Re-run screenshot without --overlay-refs.', + }, + ); +} diff --git a/src/daemon/screenshot-runtime.ts b/src/daemon/screenshot-runtime.ts index 4795113bcf..a5d6ef399f 100644 --- a/src/daemon/screenshot-runtime.ts +++ b/src/daemon/screenshot-runtime.ts @@ -1,33 +1,100 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import { + retiredScreenshotMaxSizeFlagError, + screenshotFlagsFromOptions, + screenshotOptionsFromFlags, +} from '@agent-device/contracts/capture'; +import type { ScreenshotRuntimeExecution } from '@agent-device/contracts/platform'; import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import type { AgentDeviceBackend } from '../backend.ts'; import type { ArtifactAdapter } from '../io.ts'; import { createAgentDevice, localCommandPolicy } from '../runtime.ts'; -import { dispatchCommand } from '../core/dispatch.ts'; import { - screenshotFlagsFromOptions, - screenshotOptionsFromFlags, -} from '@agent-device/contracts/capture'; -import { AppError } from '@agent-device/kernel/errors'; -import { readScreenshotResultData } from '../utils/screenshot-result.ts'; + assertSupportedScreenshotPixelDensity, + readScreenshotResultMetadata, +} from '../utils/screenshot-density.ts'; import type { DaemonCommandContext } from './context.ts'; -import type { SessionState } from './types.ts'; +import { buildSnapshotState, captureSnapshotData } from './handlers/snapshot-capture.ts'; +import type { + RecordedGenericRequest, + ResolvedGenericExecution, +} from './request-generic-dispatch.ts'; import { createDaemonRuntimeSessionStore } from './runtime-session.ts'; +import { annotateScreenshotWithRefs } from './screenshot-overlay.ts'; +import { + resolveBoundScreenshotRuntime, + type BoundScreenshotRuntime, + type ScreenshotRuntimeBindings, +} from './screenshot-runtime-binding.ts'; +import { setSessionSnapshot } from './session-snapshot.ts'; +import { SessionStore } from './session-store.ts'; +import type { DaemonRequest, SessionState } from './types.ts'; -export type ScreenshotOutputPlacement = 'positional' | 'out' | 'default'; +/** + * The `screenshot` leaf of the generic route. Argument policy is answered before any device work, + * then one plan is admitted and bound once; the returned closure is what the generic dispatcher + * runs in place of legacy platform dispatch. + */ +export async function resolveScreenshotGenericExecution( + params: Readonly<{ + req: DaemonRequest; + session: SessionState; + }> & + ScreenshotRuntimeBindings, +): Promise { + const { req, session } = params; + const retiredMaxSize = retiredScreenshotMaxSizeFlagError('screenshot', req.flags); + if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); + assertSupportedScreenshotPixelDensity(session.device, req.flags?.screenshotPixelDensity); -export async function dispatchScreenshotViaRuntime(params: { - session: SessionState; - sessionName: string; - outPath?: string; - outputPlacement: ScreenshotOutputPlacement; - dispatchContext: DaemonCommandContext; -}): Promise> { - const { session, sessionName, outPath, outputPlacement, dispatchContext } = params; + const request = readScreenshotRequest(req); + const resolved = await resolveBoundScreenshotRuntime({ + device: session.device, + overlayRefs: req.flags?.overlayRefs === true, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!resolved.ok) return resolved; + + const runtime = resolved.runtime; + return { + ok: true, + recorded: request.recorded, + execute: async (execution) => + await executeScreenshot({ + session: execution.session, + sessionName: execution.sessionName, + logPath: execution.logPath, + dispatchContext: execution.dispatchContext, + flags: execution.request.flags, + outPath: request.outPath, + runtime, + }), + }; +} + +/** + * Reserves the artifact, runs the admitted capture, and applies the shared scale/publish policy. + * Also serves the two observation paths that capture a screenshot as evidence rather than as the + * command the caller asked for. + */ +export async function captureScreenshotArtifact( + params: Readonly<{ + session: SessionState; + sessionName: string; + outPath?: string; + dispatchContext: DaemonCommandContext; + captureScreenshot: BoundScreenshotRuntime['captureScreenshot']; + }>, +): Promise { + const { session, sessionName, outPath, dispatchContext } = params; const runtime = createAgentDevice({ - backend: createDispatchScreenshotBackend({ session, outputPlacement, dispatchContext }), + backend: createBoundScreenshotBackend(params), artifacts: createDaemonScreenshotArtifactAdapter(), sessions: createDaemonRuntimeSessionStore({ sessionName, @@ -48,31 +115,167 @@ export async function dispatchScreenshotViaRuntime(params: { }); } -function createDispatchScreenshotBackend(params: { - session: SessionState; - outputPlacement: ScreenshotOutputPlacement; - dispatchContext: DaemonCommandContext; -}): AgentDeviceBackend { - const { session, outputPlacement, dispatchContext } = params; +/** + * What the shared capture command hands back. Restated here rather than imported from + * `commands/`: the daemon sits below the command surface (R2), and this adapter's artifact + * publisher emits no descriptors, so the destination and its message are the whole result. + */ +type CapturedScreenshot = Readonly<{ path: string; message?: string }>; + +/** Runner metadata the capture needs; cancellation comes from the request binding, not from here. */ +export function screenshotExecutionFromContext( + context: DaemonCommandContext, +): ScreenshotRuntimeExecution { + return { + requestId: context.requestId, + verbose: context.verbose, + logPath: context.logPath, + traceLogPath: context.traceLogPath, + iosXctestrunFile: context.iosXctestrunFile, + iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, + iosXctestEnvDir: context.iosXctestEnvDir, + runnerLeaseContext: context.runnerLeaseContext, + }; +} + +async function executeScreenshot( + params: Readonly<{ + session: SessionState; + sessionName: string; + logPath: string; + dispatchContext: DaemonCommandContext; + flags: CommandFlags | undefined; + outPath: string | undefined; + runtime: BoundScreenshotRuntime; + }>, +): Promise> { + const { session, runtime, flags } = params; + const captured = await captureScreenshotArtifact({ + session, + sessionName: params.sessionName, + outPath: params.outPath, + dispatchContext: params.dispatchContext, + captureScreenshot: runtime.captureScreenshot, + }); + const captureSnapshot = runtime.captureSnapshot; + return { + ...captured, + ...(captureSnapshot + ? { + overlayRefs: await annotateScreenshotWithSessionRefs({ + session, + logPath: params.logPath, + screenshotPath: captured.path, + dispatchContext: params.dispatchContext, + captureSnapshot, + }), + } + : {}), + ...(await readScreenshotResultMetadata({ + device: session.device, + path: captured.path, + requestedPixelDensity: flags?.screenshotPixelDensity, + scale: flags?.screenshotScale, + })), + }; +} + +/** + * `--overlay-refs` republishes the annotated tree as the session's snapshot, so the refs it drew + * are the refs a following interaction resolves. The tree comes from the same admitted binding as + * the capture — the plan required both operations before either ran. + */ +async function annotateScreenshotWithSessionRefs( + params: Readonly<{ + session: SessionState; + logPath: string; + screenshotPath: string; + dispatchContext: DaemonCommandContext; + captureSnapshot: NonNullable; + }>, +): Promise { + const { session, dispatchContext } = params; + const overlaySnapshotFlags = { snapshotInteractiveOnly: true } satisfies CommandFlags; + const overlaySnapshotData = await captureSnapshotData({ + device: session.device, + session, + flags: overlaySnapshotFlags, + logPath: params.logPath, + snapshotScope: undefined, + captureData: async () => + await params.captureSnapshot({ + options: { + appBundleId: session.appBundleId, + interactiveOnly: true, + surface: session.surface, + }, + execution: screenshotExecutionFromContext(dispatchContext), + }), + }); + const overlaySnapshot = buildSnapshotState(overlaySnapshotData, overlaySnapshotFlags); + setSessionSnapshot(session, overlaySnapshot); + return await annotateScreenshotWithRefs({ + screenshotPath: params.screenshotPath, + snapshot: overlaySnapshot, + }); +} + +/** + * `screenshot [path]` and `--out ` are the same destination expressed two ways, and both are + * user-typed paths resolved against the client's cwd. The normalized form is also what the session + * action records, so a replay resolves the same file instead of re-expanding `~` somewhere else. + */ +function readScreenshotRequest( + req: DaemonRequest, +): Readonly<{ outPath: string | undefined; recorded: RecordedGenericRequest }> { + const positionals = req.positionals ?? []; + const flags = req.flags ?? {}; + const expand = (value: string | undefined) => + value === undefined ? undefined : SessionStore.expandHome(value, req.meta?.cwd); + const positionalPath = expand(positionals[0]); + const outFlag = expand(flags.out); + return { + outPath: positionalPath ?? outFlag, + recorded: { + positionals: positionalPath ? [positionalPath, ...positionals.slice(1)] : positionals, + flags: outFlag ? { ...flags, out: outFlag } : flags, + }, + }; +} + +function createBoundScreenshotBackend( + params: Readonly<{ + session: SessionState; + dispatchContext: DaemonCommandContext; + captureScreenshot: BoundScreenshotRuntime['captureScreenshot']; + }>, +): AgentDeviceBackend { + const { session, dispatchContext, captureScreenshot } = params; return { platform: publicPlatformString(session.device), captureScreenshot: async (_context, outPath, options) => { - const context = { + const resolved = screenshotOptionsFromFlags({ ...dispatchContext, ...screenshotFlagsFromOptions(options), - surface: options?.surface, - skipIosSimulatorBootCheck: - dispatchContext.skipIosSimulatorBootCheck ?? - (isIosFamily(session.device) && session.device.kind === 'simulator'), - }; - if (outputPlacement === 'out') { - return readScreenshotResultData( - await dispatchCommand(session.device, 'screenshot', [], outPath, context), - ); - } - return readScreenshotResultData( - await dispatchCommand(session.device, 'screenshot', [outPath], undefined, context), - ); + }); + await captureScreenshot({ + outPath, + options: { + appBundleId: dispatchContext.appBundleId, + pixelDensity: resolved.pixelDensity, + fullscreen: resolved.fullscreen, + normalizeStatusBar: resolved.normalizeStatusBar, + stabilize: resolved.stabilize, + surface: options?.surface, + // An open session already proved the simulator is booted, so the per-capture recheck is + // the daemon's own knowledge rather than something the caller has to assert. + skipIosSimulatorBootCheck: + dispatchContext.skipIosSimulatorBootCheck ?? + (isIosFamily(session.device) && session.device.kind === 'simulator'), + captureBackend: dispatchContext.screenshotCaptureBackend, + }, + execution: screenshotExecutionFromContext(dispatchContext), + }); }, }; } diff --git a/src/daemon/snapshot-command-runtime.ts b/src/daemon/snapshot-command-runtime.ts index df0d0e5ab0..b404998483 100644 --- a/src/daemon/snapshot-command-runtime.ts +++ b/src/daemon/snapshot-command-runtime.ts @@ -79,6 +79,8 @@ export async function dispatchSnapshotRuntimeCommand( logPath, session, device, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }); if (!timeoutResponse) throw error; return timeoutResponse; diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index ff795ce9f2..8b650d8f80 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -39,6 +39,8 @@ export async function dispatchSnapshotViaRuntime( sessionName: resolvedSessionName, logPath: params.logPath, verdict: result.snapshotQuality, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }); const published = fallbackScreenshot ? { diff --git a/src/daemon/sparse-fallback-screenshot.ts b/src/daemon/sparse-fallback-screenshot.ts index 97ddef13df..e1f4effd45 100644 --- a/src/daemon/sparse-fallback-screenshot.ts +++ b/src/daemon/sparse-fallback-screenshot.ts @@ -1,7 +1,11 @@ import type { SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; import { isSparseSnapshotQualityVerdict } from '../snapshot-quality/verdict.ts'; import { contextFromFlags } from './context.ts'; -import { dispatchScreenshotViaRuntime } from './screenshot-runtime.ts'; +import { captureScreenshotArtifact } from './screenshot-runtime.ts'; +import { + resolveBoundScreenshotRuntime, + type ScreenshotRuntimeBindings, +} from './screenshot-runtime-binding.ts'; import type { DaemonRequest, SessionState } from './types.ts'; export type SparseFallbackScreenshot = { @@ -26,13 +30,15 @@ export type SparseFallbackScreenshot = { * `captureSnapshot` directly rather than through this runtime command, so a wait polling * an unreadable screen cannot turn into a screenshot per poll. */ -export async function captureSparseFallbackScreenshot(params: { - req: DaemonRequest; - session: SessionState | undefined; - sessionName: string; - logPath: string; - verdict: SnapshotQualityVerdict | undefined; -}): Promise { +export async function captureSparseFallbackScreenshot( + params: { + req: DaemonRequest; + session: SessionState | undefined; + sessionName: string; + logPath: string; + verdict: SnapshotQualityVerdict | undefined; + } & ScreenshotRuntimeBindings, +): Promise { const session = params.session; if (!session) return undefined; if (!isSparseSnapshotQualityVerdict(params.verdict)) return undefined; @@ -51,20 +57,31 @@ export async function captureSparseFallbackScreenshot(params: { }; } -async function captureFallbackScreenshotPath(params: { - req: DaemonRequest; - session: SessionState; - sessionName: string; - logPath: string; -}): Promise { +async function captureFallbackScreenshotPath( + params: { + req: DaemonRequest; + session: SessionState; + sessionName: string; + logPath: string; + } & ScreenshotRuntimeBindings, +): Promise { const { req, session } = params; try { - const data = await dispatchScreenshotViaRuntime({ + const capture = await resolveBoundScreenshotRuntime({ + device: session.device, + overlayRefs: false, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + // A target that cannot capture pixels owes the caller nothing here: the sparse verdict's own + // warning still carries the manual remedy. + if (!capture.ok) return undefined; + const data = await captureScreenshotArtifact({ session, sessionName: params.sessionName, // No caller-supplied destination: the screenshot artifact adapter mints a temp // path, so the fallback never writes where an explicit `--out` would have. - outputPlacement: 'default', + captureScreenshot: capture.runtime.captureScreenshot, dispatchContext: contextFromFlags( params.logPath, // The request's own flags carry the toolchain selection (xctestrun file, @@ -77,7 +94,7 @@ async function captureFallbackScreenshotPath(params: { req.meta, ), }); - return typeof data.path === 'string' ? data.path : undefined; + return data.path; } catch { // A convenience on an already-degraded path. The sparse verdict's own warning still // carries the manual remedy, so a failed fallback must not fail the snapshot the diff --git a/src/daemon/viewport-runtime.ts b/src/daemon/viewport-runtime.ts index 916a2c6d35..838d548db9 100644 --- a/src/daemon/viewport-runtime.ts +++ b/src/daemon/viewport-runtime.ts @@ -1,6 +1,7 @@ import { readViewportDimensions } from '@agent-device/contracts/capture'; import { viewportRuntimeUse } from '@agent-device/contracts/platform'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; import { admitRuntimeUse, type RuntimeAdmissionBindings } from './runtime-admission.ts'; export async function resolveBoundViewportRuntime( @@ -8,7 +9,7 @@ export async function resolveBoundViewportRuntime( device: DeviceInfo; positionals: string[]; } & RuntimeAdmissionBindings, -) { +): Promise { const input = readViewportDimensions(params.positionals); const admission = await admitRuntimeUse({ command: 'viewport', @@ -17,9 +18,12 @@ export async function resolveBoundViewportRuntime( inspectFacts: params.inspectFacts, bindDevice: params.bindDevice, }); - if (admission.type === 'response') return admission.response; - return async () => { - await admission.runtime.operations.setViewport(input); - return { ...input, message: `Viewport set: ${input.width}x${input.height}` }; + if (admission.type === 'response') return { ok: false, response: admission.response }; + return { + ok: true, + execute: async () => { + await admission.runtime.operations.setViewport(input); + return { ...input, message: `Viewport set: ${input.width}x${input.height}` }; + }, }; } diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index 3b9299f5e2..a5c37b86f5 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -9,6 +9,7 @@ import { createUnavailablePlatformRuntimeFacts, localRuntimeOwner, providerRuntimeOwner, + screenshotRuntimeOperationFacts, viewportRuntimeOperationFacts, } from '@agent-device/contracts/platform'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -31,6 +32,82 @@ import { } from './platform-runtime-gateway.fixtures.ts'; describe('composed platform runtime gateway', () => { + test('loads only the selected Apple owner for screenshot facts and binding', async () => { + const appleDevice: DeviceInfo = { + platform: 'apple', + appleOs: 'ios', + id: 'ios-simulator', + name: 'iPhone', + kind: 'simulator', + target: 'mobile', + booted: true, + }; + const unavailable = { available: false, reason: 'unsupported-platform-leaf' } as const; + const owner = localRuntimeOwner('apple'); + const baseFacts = createUnavailablePlatformRuntimeFacts(appleDevice, owner, { + appLog: unavailable, + network: unavailable, + screenshot: unavailable, + viewport: unavailable, + lifecycle: applicationLifecycleOperationFacts({ + resolveOpenTarget: unavailable, + prepareApplicationOpen: unavailable, + openApplication: unavailable, + applyRuntimeHints: unavailable, + clearRuntimeHints: unavailable, + closeApplication: unavailable, + finalizeApplicationClose: unavailable, + prepareAppleRunner: unavailable, + configureProviderPortReverse: unavailable, + }), + }); + const facts = { + ...baseFacts, + operations: { + ...baseFacts.operations, + ...screenshotRuntimeOperationFacts({ capture: { available: true } }), + }, + }; + const captureScreenshot = vi.fn(async () => undefined); + const binding: DeviceBinding = { + device: appleDevice, + owner, + facts, + operations: { captureScreenshot }, + [Symbol.asyncDispose]: async () => {}, + }; + const appleLoad = vi.fn(async () => ({ + owner, + ownsDevice: () => true, + inspectFacts: async () => facts, + bind: async () => binding, + shutdown: async () => {}, + })); + const webLoad = vi.fn(async () => { + throw new Error('unselected web runtime must stay lazy'); + }); + const runtimeGateway = createComposedPlatformRuntimeGateway({ + modules: new Map([ + ['apple', { family: 'apple', loadRuntime: appleLoad }], + ['web', { family: 'web', loadRuntime: webLoad }], + ]), + loadHost: async () => ({}) as PlatformRuntimeHost, + }); + + await expect(runtimeGateway.inspectFacts(appleDevice)).resolves.toMatchObject({ + operations: { captureScreenshot: { available: true } }, + }); + const selected = await runtimeGateway.bind({ + device: appleDevice, + intent: { kind: 'ordinary' }, + scope, + }); + await selected.operations.captureScreenshot?.({ outPath: '/tmp/gateway.png' }); + expect(appleLoad).toHaveBeenCalledTimes(1); + expect(webLoad).not.toHaveBeenCalled(); + expect(captureScreenshot).toHaveBeenCalledWith({ outPath: '/tmp/gateway.png' }); + }); + test('loads only the selected web owner for viewport facts and binding', async () => { const webDevice: DeviceInfo = { platform: 'web', @@ -46,6 +123,7 @@ describe('composed platform runtime gateway', () => { const baseFacts = createUnavailablePlatformRuntimeFacts(webDevice, owner, { appLog: unavailable, network: unavailable, + screenshot: unavailable, viewport: unavailable, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: unavailable, diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index 68e2c725b3..990d93f295 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -303,6 +303,7 @@ function unavailableProviderBinding( appLog: unavailable, appState: unavailable, network: unavailable, + screenshot: unavailable, viewport: unavailable, lifecycle: unavailableProviderLifecycleFacts(unavailable), }); @@ -320,6 +321,7 @@ function unavailableProviderFacts(runtime: ProviderDeviceRuntime, device: Device appLog: unavailable, appState: unavailable, network: unavailable, + screenshot: unavailable, viewport: unavailable, readiness: unavailable, lifecycle: unavailableProviderLifecycleFacts(unavailable),