From 35faf2d8db2829a3670c1dba858f775921504755 Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 19 Aug 2026 12:13:12 +0200 Subject: [PATCH 1/9] refactor: migrate get to the request-bound device runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `get` declares `elementReadRuntimeUse` (required `captureSnapshot`, preferred `readTextAtPoint`), admits once from exact owner facts, refuses before binding, and binds exactly once. Its capability bucket, the static HarmonyOS/Web command sets that augmented it, and `requireCommandSupported` admission for `get` are gone; `'get'` leaves the `createSelectorRuntime` capability union. The neutral `readTextAtPoint` operation replaces the branch-per-family legacy `read` dispatch on the `get` path. Every local family and both providers now classify it exhaustively — Web, HarmonyOS, Vega and every provider row report it unavailable, which is behaviour-preserving because the legacy dispatch had no arm for them and threw on every call before falling back. R36 is the new parametrized cutover row. --- .../contracts/src/element-text-runtime.ts | 60 ++++++ packages/contracts/src/facades/platform.ts | 18 +- .../src/platform-runtime-operations.ts | 30 +++ .../src/platform-runtime-unavailable.test.ts | 4 + .../src/platform-runtime-unavailable.ts | 8 + packages/platform-android/src/runtime.test.ts | 4 + packages/platform-android/src/runtime.ts | 13 ++ packages/platform-apple/src/runtime.test.ts | 5 + packages/platform-apple/src/runtime.ts | 26 +++ .../platform-harmonyos/src/runtime.test.ts | 5 + packages/platform-harmonyos/src/runtime.ts | 9 + packages/platform-linux/src/runtime.test.ts | 3 + packages/platform-linux/src/runtime.ts | 10 + packages/platform-vega/src/runtime.test.ts | 2 + packages/platform-vega/src/runtime.ts | 1 + packages/platform-web/src/runtime.test.ts | 4 + packages/platform-web/src/runtime.ts | 9 + .../src/app-log-runtime.test.ts | 7 + .../provider-limrun/src/app-log-runtime.ts | 13 ++ .../src/platform-runtime.test.ts | 8 + .../src/platform-runtime.ts | 14 ++ .../layering/runtime-command-cutover-table.ts | 27 ++- .../test-utils/runtime-operation-facts.ts | 2 + .../capability-plugin-routing-parity.test.ts | 1 - src/core/capabilities.ts | 3 +- .../__tests__/parity.test.ts | 1 + src/core/command-descriptor/registry.ts | 4 +- src/daemon/__tests__/get-runtime.test.ts | 184 ++++++++++++++++++ src/daemon/get-runtime.ts | 129 ++++++++++++ .../handlers/__tests__/install-source.test.ts | 1 + .../interaction-get-runtime-fixture.ts | 102 ++++++++++ .../__tests__/interaction-read.test.ts | 76 +++++--- .../interaction-target-evidence.test.ts | 3 + .../__tests__/interaction-touch-fixtures.ts | 2 + .../handlers/__tests__/interaction.test.ts | 78 +++++++- .../session-capabilities.fixtures.ts | 6 + .../__tests__/session-command-harness.ts | 1 + .../handlers/__tests__/session-state.test.ts | 2 + src/daemon/handlers/interaction-common.ts | 3 + .../interaction-read-legacy-dispatch.ts | 44 +++++ src/daemon/handlers/interaction-read.ts | 44 +++-- src/daemon/handlers/snapshot-capture.ts | 62 +++--- src/daemon/request-handler-chain.ts | 2 + src/daemon/selector-capture-runtime.ts | 8 + src/daemon/selector-runtime-backend.ts | 37 +++- src/daemon/selector-runtime.ts | 19 +- src/platform-runtime-element-text-host.ts | 71 +++++++ src/platform-runtime-gateway.test.ts | 1 + src/platform-runtime-gateway.ts | 2 + src/platform-runtime-operation-host.ts | 2 + 50 files changed, 1081 insertions(+), 89 deletions(-) create mode 100644 packages/contracts/src/element-text-runtime.ts create mode 100644 src/daemon/__tests__/get-runtime.test.ts create mode 100644 src/daemon/get-runtime.ts create mode 100644 src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts create mode 100644 src/daemon/handlers/interaction-read-legacy-dispatch.ts create mode 100644 src/platform-runtime-element-text-host.ts diff --git a/packages/contracts/src/element-text-runtime.ts b/packages/contracts/src/element-text-runtime.ts new file mode 100644 index 0000000000..c4abb8d9e4 --- /dev/null +++ b/packages/contracts/src/element-text-runtime.ts @@ -0,0 +1,60 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Point } from '@agent-device/kernel/snapshot'; +import type { RunnerContext } from './interactor-types.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; +import type { SessionSurface } from './session-surface.ts'; + +/** Runner metadata the selected read implementation needs, without request-owned state. */ +export type ElementTextRuntimeExecution = Readonly>; + +/** + * Neutral intent for one point-addressed element read. The point is already resolved from the + * node the caller matched, so the operation names no command, request, session, or CLI flag. + */ +export type ReadTextAtPointInput = Readonly<{ + point: Point; + options?: Readonly<{ appBundleId?: string; surface?: SessionSurface }>; + execution?: ElementTextRuntimeExecution; +}>; + +export type ElementTextRuntimeOperations = Readonly<{ + /** + * The live text an owner reads at a point, which can exceed the readable text carried by an + * already-captured snapshot node (an editable field whose value is longer than its label). + * Declared `preferred`, never `required`: every consumer's required path answers from the + * snapshot tree, so an owner without this operation still executes the command completely. + */ + readTextAtPoint(input: ReadTextAtPointInput): Promise; +}>; + +export type ElementTextRuntimeOperationFacts = Readonly<{ + readTextAtPoint: RuntimeOperationFact; +}>; + +export function elementTextRuntimeOperationFacts( + input: ElementTextRuntimeOperationFacts, +): ElementTextRuntimeOperationFacts { + return Object.freeze({ readTextAtPoint: input.readTextAtPoint }); +} + +/** + * The existing per-family read mechanics, injected by composition. Families reach their own + * tools through this port rather than importing root modules, matching the snapshot runtime's + * interactor-resolver seam. + */ +export type ElementTextRuntimeHost = Readonly<{ + readTextAtPoint(device: DeviceInfo, input: ReadTextAtPointInput): Promise; +}>; + +/** Captures one selected owner's read authority for the lifetime of a request binding. */ +export function bindElementTextRuntime( + params: Readonly<{ + device: DeviceInfo; + host: ElementTextRuntimeHost; + }>, +): ElementTextRuntimeOperations { + return Object.freeze({ + readTextAtPoint: async (input: ReadTextAtPointInput) => + await params.host.readTextAtPoint(params.device, input), + }); +} diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index 6ac3c411d9..abcb848dd1 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -235,8 +235,13 @@ export { appStateRuntimeUses, appStateUse, shutdownTargetUse, + elementReadRuntimeUse, + elementReadRuntimePlan, +} from '../platform-runtime-operations.ts'; +export type { + DeviceReadinessRuntimePlan, + ElementReadRuntimePlan, } from '../platform-runtime-operations.ts'; -export type { DeviceReadinessRuntimePlan } from '../platform-runtime-operations.ts'; export { bindLocalScreenshotInteractor, bindProviderScreenshotInteractor, @@ -270,6 +275,17 @@ export type { ViewportRuntimeOperationFacts, ViewportRuntimeOperations, } from '../viewport-runtime.ts'; +export { + bindElementTextRuntime, + elementTextRuntimeOperationFacts, +} from '../element-text-runtime.ts'; +export type { + ElementTextRuntimeExecution, + ElementTextRuntimeHost, + ElementTextRuntimeOperationFacts, + ElementTextRuntimeOperations, + ReadTextAtPointInput, +} from '../element-text-runtime.ts'; export type { AppStateRuntimeCommand, AppStateRuntimeCommandResult, diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 53f4e0d047..845a45caa0 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -15,6 +15,10 @@ import type { ScreenRecordingRuntimeOperations } from './screen-recording-runtim import type { ScreenshotRuntimeOperations } from './screenshot-runtime.ts'; import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; +import type { + ElementTextRuntimeHost, + ElementTextRuntimeOperations, +} from './element-text-runtime.ts'; import type { DeviceReadinessRuntimeHost, DeviceReadinessRuntimeOperations, @@ -47,6 +51,7 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & ScreenshotRuntimeOperations & SnapshotRuntimeOperations & ViewportRuntimeOperations & + ElementTextRuntimeOperations & DeviceReadinessRuntimeOperations & DeviceShutdownRuntimeOperations & ApplicationLifecycleRuntimeOperations; @@ -66,6 +71,30 @@ export const bootTargetHeadlessUse = defineUse({ export const appsRuntimeUse = defineUse({ required: ['ensureReady', 'listApps'] }); export const captureSnapshotUse = defineUse({ required: ['captureSnapshot'] }); export const viewportRuntimeUse = defineUse({ required: ['setViewport'] }); +/** + * `get` reads one element's text or attributes. The required path answers from the captured + * tree on every supported cell; `readTextAtPoint` is the owner-provided live read that recovers + * fuller text for editable/expandable elements, so it is preferred rather than required (ADR + * 0019 §2). An owner without it still executes `get` completely. + */ +export const elementReadRuntimeUse = defineUse({ + required: ['captureSnapshot'], + preferred: ['readTextAtPoint'], +}); + +/** + * `get`'s use does not vary with its input, so there is nothing to resolve: the one plan is a + * frozen constant rather than a `resolve…RuntimePlan` over a single row. + */ +export type ElementReadRuntimePlan = Readonly<{ + kind: 'element-read'; + use: typeof elementReadRuntimeUse; +}>; + +export const elementReadRuntimePlan: ElementReadRuntimePlan = Object.freeze({ + kind: 'element-read', + use: elementReadRuntimeUse, +}); const captureSnapshotWithCustomActionsUse = defineUse({ required: ['captureSnapshot', 'captureSnapshotWithCustomActions'], }); @@ -224,6 +253,7 @@ export type PlatformRuntimeHost = AppLogRuntimeHost & }>; screenRecording: ScreenRecordingRuntimeHost; snapshot: SnapshotRuntimeHost; + elementText: ElementTextRuntimeHost; deviceReadiness: DeviceReadinessRuntimeHost; deviceShutdown: DeviceShutdownRuntimeHost; localInteractors: LocalApplicationInteractorHost; diff --git a/packages/contracts/src/platform-runtime-unavailable.test.ts b/packages/contracts/src/platform-runtime-unavailable.test.ts index e9f15472a5..749a034248 100644 --- a/packages/contracts/src/platform-runtime-unavailable.test.ts +++ b/packages/contracts/src/platform-runtime-unavailable.test.ts @@ -31,6 +31,7 @@ test('generic unavailable binding preserves exact provider ownership and mode', network: { available: false, reason: 'owner-capability-missing' }, screenshot: { available: false, reason: 'unsupported-device-kind' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, + elementText: { available: false, reason: 'unsupported-provider-mode' }, lifecycle, }); @@ -44,6 +45,9 @@ test('generic unavailable binding preserves exact provider ownership and mode', assert.deepEqual(binding.facts.operations.captureScreenshot, { available: false, reason: 'unsupported-device-kind', + assert.deepEqual(binding.facts.operations.readTextAtPoint, { + available: false, + reason: 'unsupported-provider-mode', }); assert.deepEqual(binding.operations, {}); await binding[Symbol.asyncDispose](); diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 4e34870f6c..cc5138dab4 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -13,6 +13,7 @@ import type { import { screenshotRuntimeOperationFacts } from './screenshot-runtime.ts'; import { snapshotRuntimeOperationFacts } from './snapshot-runtime.ts'; import { viewportRuntimeOperationFacts } from './viewport-runtime.ts'; +import { elementTextRuntimeOperationFacts } from './element-text-runtime.ts'; /** * A runtime-contract helper for provider ownership gaps. It never assigns lifecycle semantics: @@ -28,6 +29,7 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{ screenshot: RuntimeOperationUnavailability; snapshot?: RuntimeOperationUnavailability; viewport: RuntimeOperationUnavailability; + elementText: RuntimeOperationUnavailability; readiness?: RuntimeOperationUnavailability; shutdown?: RuntimeOperationUnavailability; lifecycle: ApplicationLifecycleOperationFacts; @@ -43,6 +45,7 @@ type FrozenUnavailablePlatformRuntimeFacts = Readonly<{ screenshot: RuntimeOperationUnavailability; snapshot: RuntimeOperationUnavailability; viewport: RuntimeOperationUnavailability; + elementText: RuntimeOperationUnavailability; readiness: RuntimeOperationUnavailability; shutdown: RuntimeOperationUnavailability; lifecycle: ApplicationLifecycleOperationFacts; @@ -77,6 +80,7 @@ export function createUnavailablePlatformRuntimeFacts( screenshot, snapshot, viewport, + elementText, readiness, shutdown, lifecycle, @@ -109,6 +113,7 @@ export function createUnavailablePlatformRuntimeFacts( withoutActiveApp: snapshot, }), ...viewportRuntimeOperationFacts({ setViewport: viewport }), + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementText }), ensureReady: readiness, bootTarget: readiness, bootTargetHeadless: readiness, @@ -138,6 +143,9 @@ function freezeUnavailableFacts( viewport: Object.freeze({ ...unavailable.viewport }), readiness: orNetwork(unavailable.readiness), shutdown: orNetwork(unavailable.shutdown), + elementText: Object.freeze({ ...unavailable.elementText }), + readiness: Object.freeze({ ...(unavailable.readiness ?? unavailable.network) }), + shutdown: Object.freeze({ ...(unavailable.shutdown ?? unavailable.network) }), lifecycle: applicationLifecycleOperationFacts(unavailable.lifecycle), }); } diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index f34ac21c7e..440babae35 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -94,6 +94,8 @@ test.each([ expect(facts.operations.bootTarget).toEqual({ available: true }); expect(facts.operations.bootTargetHeadless.available).toBe(runtimeDevice.kind === 'emulator'); expect(facts.operations.captureSnapshot).toEqual({ available: true }); + // uiautomator reads text at a point over the same adb transport the capture uses. + expect(facts.operations.readTextAtPoint).toEqual({ available: true }); expect(facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(facts.operations.setViewport).toMatchObject({ available: false }); @@ -101,6 +103,7 @@ test.each([ expect(facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); + expect(binding.operations.readTextAtPoint).toBeTypeOf('function'); await expect(binding.operations.ensureReady?.({})).resolves.toMatchObject({ id: runtimeDevice.id, @@ -268,6 +271,7 @@ test.each([ expect(facts.operations.bootTarget).toEqual({ available: true }); expect(facts.operations.bootTargetHeadless.available).toBe(runtimeDevice.kind === 'emulator'); expect(facts.operations.captureSnapshot.available).toBe(runtimeDevice.kind !== 'simulator'); + expect(facts.operations.readTextAtPoint.available).toBe(runtimeDevice.kind !== 'simulator'); expect(binding.operations.captureSnapshot).toBeTypeOf( runtimeDevice.kind === 'simulator' ? 'undefined' : 'function', ); diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 585c2e571c..976648e98c 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -10,7 +10,9 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, + bindElementTextRuntime, bindLocalSnapshotInteractor, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -30,6 +32,10 @@ import { const owner = localRuntimeOwner('android'); const available = Object.freeze({ available: true } as const); +const elementTextKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', +} as const); const headlessUnavailable = Object.freeze({ available: false, reason: 'unsupported-device-kind', @@ -148,6 +154,11 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor capture: device.kind === 'simulator' ? screenshotKindUnavailable : available, }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + // uiautomator reads text at a point through the same adb path the snapshot uses, so the + // synthetic `simulator` row is the only Android kind without a live read. + ...elementTextRuntimeOperationFacts({ + readTextAtPoint: device.kind === 'simulator' ? elementTextKindUnavailable : available, + }), ensureReady: available, bootTarget: available, bootTargetHeadless: device.kind === 'emulator' ? available : headlessUnavailable, @@ -207,6 +218,8 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, }) + ...(facts.operations.readTextAtPoint.available + ? bindElementTextRuntime({ device: request.device, host: host.elementText }) : {}), ensureReady: async (input: EnsureReadyInput) => await ensureAndroidReady( diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 84431a2838..b51e45fad4 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -134,6 +134,10 @@ function expectAppleSnapshotAvailability( device.appleOs === 'macos', ); expect(binding.operations.captureSnapshot).toBeTypeOf(available ? 'function' : 'undefined'); + // The live point read needs a driveable Apple UI, so it follows the same watchOS sentinel the + // capture does; every other supported leaf advertises and binds it. + expect(binding.facts.operations.readTextAtPoint.available).toBe(available); + expect(binding.operations.readTextAtPoint).toBeTypeOf(available ? 'function' : 'undefined'); } test.each(['frontmost-app', 'desktop', 'menubar'] as const)( @@ -426,4 +430,5 @@ function expectLegacyLifecycleFactCell( facts.device.appleOs !== 'watchos' && (facts.device.kind === 'simulator' || facts.device.kind === 'device'); expect(facts.operations.captureSnapshot.available).toBe(snapshotAvailable); + expect(facts.operations.readTextAtPoint.available).toBe(snapshotAvailable); } diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 8903e1664c..a8721188c7 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -9,6 +9,8 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, + bindElementTextRuntime, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -55,6 +57,15 @@ const headlessUnavailable = Object.freeze({ reason: 'unsupported-provider-mode', hint: 'Headless boot is supported only for local Android emulators.', } as const); +const elementTextLeafUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'watchOS has no XCUITest-driveable UI, so element text comes from the captured tree only.', +} as const); +const elementTextKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', +} as const); const watchOpenTargetUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', @@ -224,6 +235,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR ...appleSnapshotFacts(device), ...screenshotRuntimeOperationFacts({ capture: appleScreenshotFact(device) }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + ...elementTextRuntimeOperationFacts({ readTextAtPoint: appleElementTextFact(device) }), ensureReady: readiness, bootTarget: boot, bootTargetHeadless: headlessUnavailable, @@ -274,6 +286,8 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, }) + ...(facts.operations.readTextAtPoint.available + ? bindElementTextRuntime({ device: request.device, host: host.elementText }) : {}), ...(facts.operations.ensureReady.available ? { @@ -322,6 +336,18 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR }); } +/** + * The live point read is the XCUITest runner's `readText` for app sessions and the macOS helper + * for desktop/menubar surfaces. Both need a driveable Apple UI, so watchOS and the non + * simulator/device kinds have no read at all. + */ +function appleElementTextFact(device: DeviceInfo) { + if (resolveDeviceAppleOs(device) === 'watchos') return elementTextLeafUnavailable; + return device.kind === 'simulator' || device.kind === 'device' + ? available + : elementTextKindUnavailable; +} + function appleSnapshotFact(device: DeviceInfo) { if (resolveDeviceAppleOs(device) === 'watchos') return snapshotKindUnavailable; return device.kind === 'simulator' || device.kind === 'device' diff --git a/packages/platform-harmonyos/src/runtime.test.ts b/packages/platform-harmonyos/src/runtime.test.ts index 12d40fd980..48519138a9 100644 --- a/packages/platform-harmonyos/src/runtime.test.ts +++ b/packages/platform-harmonyos/src/runtime.test.ts @@ -72,6 +72,10 @@ test.each([ expect(binding.operations.setViewport).toBeUndefined(); expect(facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + // HarmonyOS has no point-read tool: `get` answers from the captured tree, which is what the + // legacy dispatch already did once its Apple-runner fall-through failed. + expect(facts.operations.readTextAtPoint).toMatchObject({ available: false }); + expect(binding.operations.readTextAtPoint).toBeUndefined(); await expect(binding.operations.ensureReady?.({})).resolves.toMatchObject({ booted: true }); await expect( binding.operations.listApps?.({ device: runtimeDevice, filter: 'all' }), @@ -188,6 +192,7 @@ test.each([ expect(facts.operations.ensureReady).toMatchObject({ available: true }); expect(facts.operations.bootTarget).toMatchObject({ available: false }); expect(facts.operations.bootTargetHeadless).toMatchObject({ available: false }); + expect(facts.operations.readTextAtPoint.available).toBe(false); expect(facts.operations.captureSnapshot.available).toBe( runtimeDevice.kind === 'emulator' || runtimeDevice.kind === 'device', ); diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index dafbdabafb..f81aebb053 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -9,6 +9,7 @@ import { availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, bindLocalSnapshotInteractor, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -28,6 +29,11 @@ import { } from './deployment/runtime.ts'; const owner = localRuntimeOwner('harmonyos'); +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'HarmonyOS reads element text from the captured tree only.', +} as const); const available = Object.freeze({ available: true } as const); const unavailable = Object.freeze({ available: false, @@ -143,6 +149,9 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor : screenshotKindUnavailable, }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + // HarmonyOS has no point-read tool: `get` answers from the captured tree, which is what + // the legacy dispatch already did after its Apple-runner attempt failed. + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), ensureReady: available, bootTarget: unavailable, bootTargetHeadless: unavailable, diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index 56700ea5bf..56ce748126 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -100,6 +100,9 @@ test.each([ expect(binding.facts.operations.appState).toMatchObject({ available: false }); expect(binding.facts.operations.listApps).toMatchObject({ available: false }); expect(binding.facts.operations.captureSnapshot.available).toBe(device.kind === 'device'); + // The Linux read is value-first where the captured tree is label-first, so the desktop row + // genuinely reads differently from its snapshot text and advertises the live read. + expect(binding.facts.operations.readTextAtPoint.available).toBe(device.kind === 'device'); expect(binding.facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(binding.facts.operations.captureSnapshotWithoutActiveApp.available).toBe( device.kind === 'device', diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 10c95f1c9d..2ddeaf0b5b 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -11,7 +11,9 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, + bindElementTextRuntime, createUnavailablePlatformRuntimeFacts, + elementTextRuntimeOperationFacts, localRuntimeOwner, sameRuntimeOwner, screenshotRuntimeOperationFacts, @@ -24,6 +26,7 @@ import { bindLinuxApplicationLifecycle } from './lifecycle.ts'; const supported = Object.freeze({ available: true } as const); const linuxOwner = localRuntimeOwner('linux'); const unsupportedPlatformLeaf = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); +const elementTextKindUnavailable = unavailableLinuxRuntimeFact('unsupported-device-kind'); const runtimeHintsUnavailable = unavailableLinuxRuntimeFact( 'unsupported-platform-leaf', 'Runtime hints are supported only for local iOS-family simulators and Android devices.', @@ -92,6 +95,8 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, }) + ...(facts.operations.readTextAtPoint.available + ? bindElementTextRuntime({ device: request.device, host: host.elementText }) : {}), }), [Symbol.asyncDispose]: async () => undefined, @@ -110,6 +115,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts screenshot: screenshotKindUnavailable, snapshot: snapshotKindUnavailable, viewport: unsupportedPlatformLeaf, + elementText: elementTextKindUnavailable, readiness: unsupportedPlatformLeaf, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: openTarget, @@ -134,6 +140,10 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts }), ...screenshotRuntimeOperationFacts({ capture: device.kind === 'device' ? supported : screenshotKindUnavailable, + // The Linux read is value-first (AXValue/title/description) where the captured tree is + // label-first, so the desktop row genuinely reads differently from its snapshot text. + ...elementTextRuntimeOperationFacts({ + readTextAtPoint: device.kind === 'device' ? supported : elementTextKindUnavailable, }), }, }); diff --git a/packages/platform-vega/src/runtime.test.ts b/packages/platform-vega/src/runtime.test.ts index 8859b3a7e6..6ab34f8523 100644 --- a/packages/platform-vega/src/runtime.test.ts +++ b/packages/platform-vega/src/runtime.test.ts @@ -135,6 +135,8 @@ test.each([ reason: 'unsupported-platform-leaf', }); expect(binding.operations.captureSnapshot).toBeUndefined(); + expect(binding.facts.operations.readTextAtPoint.available).toBe(false); + expect(binding.operations.readTextAtPoint).toBeUndefined(); expect(binding.facts.operations.setViewport).toMatchObject({ available: false }); expect(binding.facts.operations.captureScreenshot).toMatchObject({ available: false, diff --git a/packages/platform-vega/src/runtime.ts b/packages/platform-vega/src/runtime.ts index ebd37d6ceb..5c96a1598c 100644 --- a/packages/platform-vega/src/runtime.ts +++ b/packages/platform-vega/src/runtime.ts @@ -90,6 +90,7 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts screenshot: screenshotUnavailable, snapshot: unsupportedPlatformLeaf, viewport: unsupportedPlatformLeaf, + elementText: unsupportedPlatformLeaf, readiness: unsupportedPlatformLeaf, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: openTarget, diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index 9b9c4f6014..fd226aba86 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -54,6 +54,10 @@ test('preserves a narrow web provider dump including empty successful entries', expect(binding.operations.setViewport).toBeTypeOf('function'); expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + // No point-addressed read on the web backend: `get` answers from the captured DOM tree. The + // legacy `read` dispatch had no web arm at all and threw on every call before falling back. + expect(binding.facts.operations.readTextAtPoint.available).toBe(false); + expect(binding.operations.readTextAtPoint).toBeUndefined(); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); expectLifecycleFacts(binding); }); diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index 5e301ffebb..82390e1c70 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -10,6 +10,7 @@ import { availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, bindLocalSnapshotInteractor, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -23,6 +24,11 @@ import { bindWebApplicationLifecycle } from './lifecycle.ts'; const owner = localRuntimeOwner('web'); const available = Object.freeze({ available: true } as const); +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'Web targets read element text from the captured tree only.', +} as const); const appLogUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', @@ -244,6 +250,9 @@ function webRuntimeFacts( }), ...screenshotRuntimeOperationFacts({ capture: browserDevice }), ...viewportRuntimeOperationFacts({ setViewport: browserDevice }), + // The web backend has no point-addressed read: `get` answers from the captured DOM tree, + // which is what the legacy dispatch already did once its Apple-runner attempt failed. + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), ensureReady: readinessUnavailable, bootTarget: readinessUnavailable, bootTargetHeadless: readinessUnavailable, diff --git a/packages/provider-limrun/src/app-log-runtime.test.ts b/packages/provider-limrun/src/app-log-runtime.test.ts index c372c1c91d..c928b7e0c3 100644 --- a/packages/provider-limrun/src/app-log-runtime.test.ts +++ b/packages/provider-limrun/src/app-log-runtime.test.ts @@ -284,6 +284,13 @@ test.each([ expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); + // Limrun owns the device remotely and exposes no local point-read tool, so the live read is + // unavailable and `get` answers from the captured tree — never by borrowing the local runtime. + expect(binding.facts.operations.readTextAtPoint).toMatchObject({ + available: false, + reason: 'unsupported-provider-mode', + }); + expect(binding.operations.readTextAtPoint).toBeUndefined(); await expect( binding.operations.listApps?.({ device: runtimeDevice, filter: 'all' }), ).resolves.toEqual([]); diff --git a/packages/provider-limrun/src/app-log-runtime.ts b/packages/provider-limrun/src/app-log-runtime.ts index 76d8cb01be..68e4b78416 100644 --- a/packages/provider-limrun/src/app-log-runtime.ts +++ b/packages/provider-limrun/src/app-log-runtime.ts @@ -27,6 +27,7 @@ import { providerRuntimeOwner, sameRuntimeOwner, screenshotRuntimeOperationFacts, + elementTextRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, } from '@agent-device/contracts/platform'; @@ -92,6 +93,16 @@ const viewportUnavailable = Object.freeze({ reason: 'unsupported-provider-mode', hint: 'Limrun does not expose viewport resizing.', } as const); +/** + * A point read needs a local tool (adb uiautomator, the XCUITest runner). Limrun's transport + * carries none of them, so the owner reports no live read and `get` answers from the captured + * tree; provider ownership never borrows the local family read. + */ +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'Limrun-owned devices read element text from the captured tree only.', +} as const); const recordingUnavailable = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -191,6 +202,7 @@ export function createLimrunPlatformRuntimeOwner( network: liveSessionUnavailable, screenshot: liveSessionUnavailable, viewport: liveSessionUnavailable, + elementText: liveSessionUnavailable, readiness: liveSessionUnavailable, shutdown: liveSessionUnavailable, lifecycle: limrunLifecycleFacts(device, false), @@ -442,6 +454,7 @@ function facts( }), ...screenshotRuntimeOperationFacts({ capture: available }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), ensureReady: available, bootTarget: available, bootTargetHeadless: headlessUnavailable, diff --git a/packages/provider-webdriver/src/platform-runtime.test.ts b/packages/provider-webdriver/src/platform-runtime.test.ts index c14d401e33..d1be56e097 100644 --- a/packages/provider-webdriver/src/platform-runtime.test.ts +++ b/packages/provider-webdriver/src/platform-runtime.test.ts @@ -239,6 +239,13 @@ test('captures through only the active exact WebDriver interactor', async () => expect(binding.operations.setViewport).toBeUndefined(); expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + // Provider ownership is authoritative and fails closed: a WebDriver owner's transport carries + // no local point-read tool, so it advertises none and never borrows the local family read. + expect(binding.facts.operations.readTextAtPoint).toMatchObject({ + available: false, + reason: 'unsupported-provider-mode', + }); + expect(binding.operations.readTextAtPoint).toBeUndefined(); await expect( binding.operations.captureSnapshot?.({ options: { interactiveOnly: true } }), ).resolves.toEqual({ backend: 'android', nodes: [] }); @@ -273,6 +280,7 @@ test.each([ expect(facts.operations.captureSnapshotWithoutActiveApp.available).toBe(false); expect(facts.operations.setViewport.available).toBe(false); expect(facts.operations.captureScreenshot.available).toBe(false); + expect(facts.operations.readTextAtPoint.available).toBe(false); if (state.isSessionActive()) { const binding = await owner.bind({ device, diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index 813b04ff18..19f36d5ad0 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -87,6 +87,18 @@ const viewportUnavailable = Object.freeze({ hint: 'WebDriver provider runtimes do not expose viewport resizing.', } as const); +/** + * A point read is a local-tool operation (adb uiautomator, the XCUITest runner, the macOS + * helper). A WebDriver owner's transport carries none of them, so the owner reports no live + * read and `get` answers from the captured tree; provider ownership never borrows the local + * family read. + */ +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'WebDriver provider runtimes read element text from the captured tree only.', +} as const); + const appStateUnavailable = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -268,6 +280,7 @@ function webDriverFacts( screenRecording: inactiveSession, screenshot: inactiveSession, viewport: inactiveSession, + elementText: inactiveSession, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: inactiveSession, prepareApplicationOpen: inactiveSession, @@ -289,6 +302,7 @@ function webDriverFacts( screenRecording: recordingUnavailable, screenshot: screenshotUnavailable, viewport: viewportUnavailable, + elementText: elementTextUnavailable, lifecycle: webDriverLifecycleFacts(device), }); // Both capture cells need the same reachability: an interactor this provider can drive, on a diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index c856a822da..ddf0825fb9 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -25,7 +25,7 @@ import { retiredDispatchProjectionViolations } from './runtime-command-cutover-d * A row id is a report heading, so it must be unique across every stack that adds rows here. * `cutoverTableDefects` rejects a duplicate; lifecycle starts at R28 after the accepted * shutdown, install/deploy, and application-lifecycle allocations. Snapshot starts at R32; - * diff follows at R33. + * diff follows at R33, viewport at R34, and get at R36 (R35 is reserved for find). */ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ { @@ -506,6 +506,31 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ }, extensions: [diffRetiredDispatchProjectionProof], }, + { + rule: 'R36 get-runtime-cutover', + command: 'get', + subject: 'element read', + tier: 'request-scoped', + execution: 'device-runtime', + // `get`'s legacy admission WAS its capability bucket plus the static family command sets the + // matrix augments it with; the row's automatic admission columns reject the bucket and the + // `requireCommandSupported('get', …)` call, and these names are the augmentation entries that + // had to disappear with them (`addWebCommandCapabilities` throws for a web-listed command + // with no matrix row, so the web entry could not be left behind). + legacyRetirement: { + routeNames: ['WEB_QUERY_COMMANDS_WITH_GET', 'HARMONYOS_GET_SUPPORT'], + }, + runtimeTypeNames: ['ElementTextRuntimeOperations', 'SnapshotRuntimeOperations'], + operations: { names: ['captureSnapshot', 'readTextAtPoint'] }, + singularExecution: { + routes: ['dispatchGetViaRuntime'], + operations: ['captureSnapshot', 'readTextAtPoint'], + operationOwners: { + captureSnapshot: ['selectElementReadOperations'], + readTextAtPoint: ['selectElementTextRead'], + }, + }, + }, { rule: 'R34 viewport-runtime-cutover', command: 'viewport', diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index 8f2b6f3f55..d7400257d4 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -1,6 +1,7 @@ import { applicationLifecycleOperationFacts, screenshotRuntimeOperationFacts, + elementTextRuntimeOperationFacts, snapshotRuntimeOperationFacts, type RuntimeOperationFact, } from '@agent-device/contracts/platform'; @@ -33,6 +34,7 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre ...unavailableShutdownOperationFacts, ...screenshotRuntimeOperationFacts({ capture: unavailable }), setViewport: unavailable, + ...elementTextRuntimeOperationFacts({ readTextAtPoint: unavailable }), }); /** Default facts for tests that are unrelated to application lifecycle commands. */ diff --git a/src/core/__tests__/capability-plugin-routing-parity.test.ts b/src/core/__tests__/capability-plugin-routing-parity.test.ts index 1c6983fc01..e2f8375ec3 100644 --- a/src/core/__tests__/capability-plugin-routing-parity.test.ts +++ b/src/core/__tests__/capability-plugin-routing-parity.test.ts @@ -270,7 +270,6 @@ test('HarmonyOS static capabilities omit runtime-backed command admissions', () 'find', 'focus', 'gesture', - 'get', 'home', 'is', 'keyboard', diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 438bee70ef..e63cc1f59c 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -44,7 +44,6 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'fill', 'find', 'focus', - 'get', 'home', 'gesture', 'keyboard', @@ -57,7 +56,7 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'type', 'wait', ]); -const WEB_QUERY_COMMANDS = ['audio', 'find', 'get', 'is', 'wait'] as const; +const WEB_QUERY_COMMANDS = ['audio', 'find', 'is', 'wait'] as const; const WEB_INTERACTION_COMMANDS = [ 'click', 'fill', diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index f634977331..f401543280 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -57,6 +57,7 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.diff, PUBLIC_COMMANDS.doctor, PUBLIC_COMMANDS.events, + PUBLIC_COMMANDS.get, PUBLIC_COMMANDS.install, PUBLIC_COMMANDS.installFromSource, PUBLIC_COMMANDS.logs, diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 24c618fa3b..ec926da2cc 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -26,6 +26,7 @@ import { readySendPushNotificationUse, openApplicationRuntimePlanUses, closeApplicationRuntimePlanUses, + elementReadRuntimeUse, snapshotRuntimePlanUses, prepareAppleRunnerRuntimeUse, runtimeCommandRuntimePlanUses, @@ -1188,10 +1189,9 @@ export const RAW_COMMAND_DESCRIPTORS = [ recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'interaction', refFrameEffect: 'preserve' }, - capability: ALL_DEVICE_COMMAND_CAPABILITY, timeoutPolicy: postActionObservationTimeoutPolicy('get', PRESERVE_DAEMON_TIMEOUT_POLICY), batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: [elementReadRuntimeUse] as const }, }, { name: 'read', diff --git a/src/daemon/__tests__/get-runtime.test.ts b/src/daemon/__tests__/get-runtime.test.ts new file mode 100644 index 0000000000..79707d51ef --- /dev/null +++ b/src/daemon/__tests__/get-runtime.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + applicationLifecycleOperationFacts, + createUnavailablePlatformRuntimeFacts, + localRuntimeOwner, + narrowDeviceBinding, + providerRuntimeOwner, + type DeviceBinding, + type PlatformRuntimeOperations, + type RuntimeFacts, + type RuntimeOperationUnavailability, + type RuntimeOwnerRef, +} from '@agent-device/contracts/platform'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { resolveBoundGetRuntime } from '../get-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import type { SessionState } from '../types.ts'; +import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; + +const available = Object.freeze({ available: true } as const); + +function unavailableFact(reason: RuntimeOperationUnavailability['reason'], hint?: string) { + return Object.freeze({ available: false, reason, ...(hint ? { hint } : {}) } as const); +} + +function facts( + device: DeviceInfo, + owner: RuntimeOwnerRef, + operations: Readonly<{ + captureSnapshot: RuntimeFacts['operations']['captureSnapshot']; + readTextAtPoint: RuntimeFacts['operations']['readTextAtPoint']; + }>, +): RuntimeFacts { + const missing = unavailableFact('owner-capability-missing'); + const base = createUnavailablePlatformRuntimeFacts(device, owner, { + appLog: missing, + network: missing, + viewport: missing, + elementText: missing, + lifecycle: applicationLifecycleOperationFacts({ + resolveOpenTarget: missing, + prepareApplicationOpen: missing, + openApplication: missing, + applyRuntimeHints: missing, + clearRuntimeHints: missing, + closeApplication: missing, + finalizeApplicationClose: missing, + prepareAppleRunner: missing, + configureProviderPortReverse: missing, + }), + }); + return Object.freeze({ + device: base.device, + operations: { ...base.operations, ...operations }, + }); +} + +function harness( + options: Readonly<{ + owner?: RuntimeOwnerRef; + captureSnapshot?: RuntimeFacts['operations']['captureSnapshot']; + readTextAtPoint?: RuntimeFacts['operations']['readTextAtPoint']; + }> = {}, +) { + const owner = options.owner ?? localRuntimeOwner('apple'); + const operations = { + captureSnapshot: options.captureSnapshot ?? available, + readTextAtPoint: options.readTextAtPoint ?? available, + }; + const captureSnapshot = vi.fn(async () => ({ backend: 'xctest', nodes: [] }) as never); + const readTextAtPoint = vi.fn(async () => 'live text'); + const inspectFacts = vi.fn(async (device: DeviceInfo) => + facts(device, owner, operations), + ) as InspectDeviceRuntimeFacts; + const bindDevice = vi.fn(async (device: DeviceInfo, use) => { + const binding = Object.freeze({ + device, + owner, + facts: facts(device, owner, operations), + operations: Object.freeze({ + ...(operations.captureSnapshot.available ? { captureSnapshot } : {}), + ...(operations.readTextAtPoint.available ? { readTextAtPoint } : {}), + }), + [Symbol.asyncDispose]: async () => undefined, + }) as unknown as DeviceBinding; + return narrowDeviceBinding(binding, use); + }) as BindDeviceRuntime; + return { inspectFacts, bindDevice, captureSnapshot, readTextAtPoint }; +} + +function session(): SessionState { + return makeIosSession('get-runtime', { appBundleId: 'com.example.app' }); +} + +describe('resolveBoundGetRuntime', () => { + it('refuses without an active session before touching facts or binding', async () => { + const seams = harness(); + const resolved = await resolveBoundGetRuntime({ session: undefined, ...seams }); + expect(resolved.ok).toBe(false); + expect(seams.inspectFacts).not.toHaveBeenCalled(); + expect(seams.bindDevice).not.toHaveBeenCalled(); + }); + + it('inspects owner facts exactly once and binds exactly once', async () => { + const seams = harness(); + const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); + expect(resolved.ok).toBe(true); + expect(seams.inspectFacts).toHaveBeenCalledTimes(1); + expect(seams.bindDevice).toHaveBeenCalledTimes(1); + }); + + it('binds the admitted device with the element-read use', async () => { + const seams = harness(); + const active = session(); + await resolveBoundGetRuntime({ session: active, ...seams }); + const [boundDevice, use] = vi.mocked(seams.bindDevice).mock.calls[0] ?? []; + expect(boundDevice?.id).toBe(active.device.id); + expect(use).toEqual({ required: ['captureSnapshot'], preferred: ['readTextAtPoint'] }); + }); + + it('refuses before binding when the required capture is unavailable', async () => { + const seams = harness({ + captureSnapshot: unavailableFact('unsupported-platform-leaf', 'no snapshot backend'), + }); + const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); + expect(resolved.ok).toBe(false); + if (!resolved.ok && !resolved.response.ok) { + expect(resolved.response.error.code).toBe('UNSUPPORTED_OPERATION'); + expect(resolved.response.error.hint).toBe('no snapshot backend'); + } + expect(seams.inspectFacts).toHaveBeenCalledTimes(1); + expect(seams.bindDevice).not.toHaveBeenCalled(); + }); + + it('still admits and binds when only the preferred read is unavailable', async () => { + const seams = harness({ readTextAtPoint: unavailableFact('unsupported-platform-leaf') }); + const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); + expect(resolved.ok).toBe(true); + if (resolved.ok) { + expect(resolved.operations.captureSnapshot).toBeTypeOf('function'); + expect(resolved.operations.readTextAtPoint).toBeUndefined(); + } + expect(seams.bindDevice).toHaveBeenCalledTimes(1); + }); + + it('exposes the preferred read when the owner advertises it', async () => { + const seams = harness(); + const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); + expect(resolved.ok).toBe(true); + if (resolved.ok) { + await resolved.operations.readTextAtPoint?.({ point: { x: 1, y: 2 } }); + expect(seams.readTextAtPoint).toHaveBeenCalledTimes(1); + } + }); + + // Provider ownership is authoritative: an unavailable provider fact fails closed rather than + // borrowing the local family runtime. + it('fails closed for a provider owner whose facts refuse the capture', async () => { + const seams = harness({ + owner: providerRuntimeOwner('webdriver', 'tenant-a'), + captureSnapshot: unavailableFact('unsupported-provider-mode'), + }); + const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); + expect(resolved.ok).toBe(false); + expect(seams.bindDevice).not.toHaveBeenCalled(); + }); + + it('binds a provider owner that advertises capture but no live read', async () => { + const seams = harness({ + owner: providerRuntimeOwner('webdriver', 'tenant-a'), + readTextAtPoint: unavailableFact('unsupported-provider-mode'), + }); + const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); + expect(resolved.ok).toBe(true); + if (resolved.ok) expect(resolved.operations.readTextAtPoint).toBeUndefined(); + }); + + it('refuses to bind without a runtime gateway', async () => { + const seams = harness(); + await expect( + resolveBoundGetRuntime({ session: session(), inspectFacts: seams.inspectFacts }), + ).rejects.toThrow(/binding is unavailable/i); + }); +}); diff --git a/src/daemon/get-runtime.ts b/src/daemon/get-runtime.ts new file mode 100644 index 0000000000..4a1edf4d16 --- /dev/null +++ b/src/daemon/get-runtime.ts @@ -0,0 +1,129 @@ +import { + elementReadRuntimePlan, + type CaptureSnapshotInput, + type ElementReadRuntimePlan, + type ElementTextRuntimeOperations, + type ReadTextAtPointInput, + type SnapshotResult, +} from '@agent-device/contracts/platform'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; +import type { DaemonResponse, SessionState } from './types.ts'; +import { noActiveSessionError } from './handlers/response.ts'; +import { + admitRuntimePlan, + requireRuntimeBinding, + unavailableRuntimeOperationResponse, + unwrapAdmittedRuntimePlan, + type AdmittedRuntimePlan, +} from './handlers/session-runtime-admission.ts'; + +/** + * The operations `get` executes with. `captureSnapshot` is required, so it is non-optional here; + * `readTextAtPoint` is the declared preferred operation and is present only on owners whose facts + * advertise it. Its absence changes which text `get` can read, never whether `get` can run. + */ +export type BoundGetRuntimeOperations = Readonly<{ + captureSnapshot(input: CaptureSnapshotInput): Promise; + readTextAtPoint?: ElementTextRuntimeOperations['readTextAtPoint']; +}>; + +export type ResolvedGetRuntime = + | Readonly<{ + ok: true; + session: SessionState; + device: SessionState['device']; + operations: BoundGetRuntimeOperations; + }> + | Readonly<{ ok: false; response: DaemonResponse }>; + +/** + * `get`'s one facts-first admission and one binding. The session owns the device, so there is no + * separate target resolution: admission inspects that device's owner facts once, refuses before + * binding when the required capture is unavailable, then binds exactly once on the admitted + * device through the admitted plan's token. + */ +export async function resolveBoundGetRuntime( + params: Readonly<{ + session: SessionState | undefined; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; + }>, +): Promise { + const { session } = params; + if (!session) return { ok: false, response: noActiveSessionError() }; + const device = session.device; + const admission = await admitRuntimePlan({ + device, + plan: elementReadRuntimePlan, + inspectFacts: params.inspectFacts, + }); + if (!admission.admitted) { + return { ok: false, response: unavailableRuntimeOperationResponse('get', admission.fact)! }; + } + return { + ok: true, + session, + device, + operations: await bindGetRuntime(params.bindDevice, admission), + }; +} + +/** + * Binds only an admitted plan, on the device it was admitted for: the token is minted by + * `admitRuntimePlan` alone and unwrapped by exact identity, so a bare plan, a separate device, or + * a look-alike cannot reach these operations. + */ +async function bindGetRuntime( + bindDevice: BindDeviceRuntime | undefined, + admission: AdmittedRuntimePlan, +): Promise { + const bind = requireRuntimeBinding(bindDevice); + const { device, plan } = unwrapAdmittedRuntimePlan(admission); + return selectElementReadOperations(await bind(device, plan.use)); +} + +/** + * Mirrors the snapshot binder's `BoundSnapshotOperation`: the operation this projection names is + * non-optional, so a value of this type IS the proof that the owner advertised it. + */ +type BoundElementReadOperation = Readonly<{ + operations: Readonly>; +}>; + +type BoundElementReadCatalog = Readonly<{ + captureSnapshot(input: CaptureSnapshotInput): Promise; + readTextAtPoint: ElementTextRuntimeOperations['readTextAtPoint']; +}>; + +type NarrowedElementReadRuntime = Readonly<{ + operations: Readonly<{ + captureSnapshot(input: CaptureSnapshotInput): Promise; + readTextAtPoint?: ElementTextRuntimeOperations['readTextAtPoint']; + }>; +}>; + +/** + * The lexical owner of `get`'s required capture call. The preferred read is handed to its own + * owner below only when the owner facts advertised it: the narrowed projection is *constructed* + * from a non-undefined local, so presence is carried by the type system rather than repaired + * with a non-null assertion or a defensive throw. + */ +function selectElementReadOperations( + runtime: NarrowedElementReadRuntime, +): BoundGetRuntimeOperations { + const readTextAtPoint = runtime.operations.readTextAtPoint; + return Object.freeze({ + captureSnapshot: async (input: CaptureSnapshotInput) => + await runtime.operations.captureSnapshot(input), + ...(readTextAtPoint + ? { readTextAtPoint: selectElementTextRead({ operations: { readTextAtPoint } }) } + : {}), + }); +} + +/** The lexical owner of `get`'s preferred element-text read. */ +function selectElementTextRead( + runtime: BoundElementReadOperation<'readTextAtPoint'>, +): ElementTextRuntimeOperations['readTextAtPoint'] { + return async (input: ReadTextAtPointInput) => await runtime.operations.readTextAtPoint(input); +} diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index 6795cb782b..9fbe562efb 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -359,6 +359,7 @@ function sourceRuntimeFacts( }), ...screenshotRuntimeOperationFacts({ capture: unavailable }), setViewport: unavailable, + readTextAtPoint: unavailable, deployApp: unavailable, materializeAppSource: materializationAvailable ? { available: true } : unavailable, deployMaterializedApp: materializationAvailable ? { available: true } : unavailable, diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts new file mode 100644 index 0000000000..e7fe731977 --- /dev/null +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -0,0 +1,102 @@ +import { vi } from 'vitest'; +import { + createUnavailablePlatformRuntimeFacts, + localRuntimeOwner, + narrowDeviceBinding, + applicationLifecycleOperationFacts, + type CaptureSnapshotInput, + type DeviceBinding, + type PlatformRuntimeOperations, + type ReadTextAtPointInput, + type RuntimeFacts, +} from '@agent-device/contracts/platform'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { + BindDeviceRuntime, + InspectDeviceRuntimeFacts, +} from '../../request-runtime-binding.ts'; +import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; + +/** + * The request-bound runtime seam `get` consumes, faked at `inspectFacts` / `bindDevice` — never + * at `core/dispatch.ts`. The bound capture still runs the interactor capture the surrounding + * interaction tests already mock, so only the two `get` operations are fixture-owned here. + */ +export const mockReadTextAtPoint = vi.fn(async (_input: ReadTextAtPointInput) => ''); + +/** Flip to model an owner whose facts advertise no live element read (web, HarmonyOS, provider). */ +export const elementReadFixtureState = { readTextAtPointAvailable: true }; + +export function resetGetRuntimeFixture(): void { + mockReadTextAtPoint.mockReset(); + mockReadTextAtPoint.mockResolvedValue(''); + elementReadFixtureState.readTextAtPointAvailable = true; +} + +const available = Object.freeze({ available: true } as const); +const unavailable = Object.freeze({ + available: false, + reason: 'owner-capability-missing', +} as const); + +function elementReadFacts(device: DeviceInfo): RuntimeFacts { + const base = createUnavailablePlatformRuntimeFacts(device, localRuntimeOwner('apple'), { + appLog: unavailable, + network: unavailable, + viewport: unavailable, + elementText: unavailable, + lifecycle: applicationLifecycleOperationFacts({ + resolveOpenTarget: unavailable, + prepareApplicationOpen: unavailable, + openApplication: unavailable, + applyRuntimeHints: unavailable, + clearRuntimeHints: unavailable, + closeApplication: unavailable, + finalizeApplicationClose: unavailable, + prepareAppleRunner: unavailable, + configureProviderPortReverse: unavailable, + }), + }); + return Object.freeze({ + device: base.device, + operations: { + ...base.operations, + captureSnapshot: available, + readTextAtPoint: elementReadFixtureState.readTextAtPointAvailable ? available : unavailable, + }, + }); +} + +const mockInspectElementReadFacts: InspectDeviceRuntimeFacts = vi.fn(async (device: DeviceInfo) => + elementReadFacts(device), +); + +const mockBindElementReadRuntime: BindDeviceRuntime = vi.fn(async (device: DeviceInfo, use) => { + const facts = elementReadFacts(device); + const binding: DeviceBinding = Object.freeze({ + device, + owner: localRuntimeOwner('apple'), + facts, + operations: Object.freeze({ + captureSnapshot: async (input: CaptureSnapshotInput) => + await captureSnapshotWithInteractor({ + device, + runnerContext: { ...input.execution, appBundleId: input.options?.appBundleId }, + options: { ...input.options }, + }), + ...(elementReadFixtureState.readTextAtPointAvailable + ? { readTextAtPoint: mockReadTextAtPoint } + : {}), + }), + [Symbol.asyncDispose]: async () => undefined, + }) as DeviceBinding; + return narrowDeviceBinding(binding, use); +}) as BindDeviceRuntime; + +/** Spread into any interaction-handler params so `get` can admit and bind. */ +export function getRuntimeBindings(): Readonly<{ + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; +}> { + return { inspectFacts: mockInspectElementReadFacts, bindDevice: mockBindElementReadRuntime }; +} diff --git a/src/daemon/handlers/__tests__/interaction-read.test.ts b/src/daemon/handlers/__tests__/interaction-read.test.ts index 6f5414bbdb..26c25789f0 100644 --- a/src/daemon/handlers/__tests__/interaction-read.test.ts +++ b/src/daemon/handlers/__tests__/interaction-read.test.ts @@ -1,18 +1,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; - -vi.mock('../../../core/dispatch.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - dispatchCommand: vi.fn(async () => ({ text: 'backend-text' })), - }; -}); - -import { dispatchCommand } from '../../../core/dispatch.ts'; +import type { ReadTextAtPointInput } from '@agent-device/contracts/platform'; import { readTextForNode } from '../interaction-read.ts'; -const mockDispatch = vi.mocked(dispatchCommand); +/** + * Bound at the seam the handler consumes (the runtime's `readTextAtPoint` operation), never at + * `core/dispatch.ts`: `get` is migrated, so the live read reaches this fake through the request + * binding rather than through the legacy dispatcher. + */ +const readTextAtPoint = vi.fn(async (_input: ReadTextAtPointInput) => 'backend-text'); function node(overrides: Partial): SnapshotNode { return { @@ -27,53 +23,89 @@ const baseParams = { device: { platform: 'ios' } as never, flags: undefined, contextFromFlags: () => ({}) as never, + readTextAtPoint, }; describe('readTextForNode', () => { - beforeEach(() => mockDispatch.mockClear()); + beforeEach(() => readTextAtPoint.mockClear()); - it('returns snapshot text without a backend read for non-editable nodes', async () => { + it('returns snapshot text without a live read for non-editable nodes', async () => { const text = await readTextForNode({ ...baseParams, node: node({ type: 'button', label: 'General' }), }); expect(text).toBe('General'); - expect(mockDispatch).not.toHaveBeenCalled(); + expect(readTextAtPoint).not.toHaveBeenCalled(); }); - it('still re-reads via the backend for editable text inputs (live value may exceed snapshot)', async () => { + it('still re-reads live for editable text inputs (live value may exceed snapshot)', async () => { const text = await readTextForNode({ ...baseParams, node: node({ type: 'textfield', value: 'snap' }), }); - expect(mockDispatch).toHaveBeenCalledOnce(); + expect(readTextAtPoint).toHaveBeenCalledOnce(); expect(text).toBe('backend-text'); }); + it('reads at the resolved node center', async () => { + await readTextForNode({ ...baseParams, node: node({ type: 'textfield', value: 'snap' }) }); + expect(readTextAtPoint.mock.calls[0]?.[0].point).toEqual({ x: 50, y: 20 }); + }); + it('re-reads when the snapshot node has no readable text', async () => { await readTextForNode({ ...baseParams, node: node({ type: 'other' }) }); - expect(mockDispatch).toHaveBeenCalledOnce(); + expect(readTextAtPoint).toHaveBeenCalledOnce(); }); - it('returns snapshot text without a backend read when the node has no resolvable center', async () => { + it('returns snapshot text without a live read when the node has no resolvable center', async () => { const text = await readTextForNode({ ...baseParams, node: node({ type: 'button', label: 'General', rect: undefined }), }); expect(text).toBe('General'); - expect(mockDispatch).not.toHaveBeenCalled(); + expect(readTextAtPoint).not.toHaveBeenCalled(); }); - it('does NOT skip the backend read on non-iOS platforms (value-first read semantics differ)', async () => { + it('does NOT skip the live read on non-iOS platforms (value-first read semantics differ)', async () => { for (const platform of ['android', 'macos', 'linux'] as const) { - mockDispatch.mockClear(); + readTextAtPoint.mockClear(); const text = await readTextForNode({ ...baseParams, device: { platform } as never, node: node({ type: 'button', label: 'General' }), }); - expect(mockDispatch).toHaveBeenCalledOnce(); + expect(readTextAtPoint).toHaveBeenCalledOnce(); expect(text).toBe('backend-text'); } }); + + // The preferred-operation absence row: an owner whose facts advertise no live read answers + // entirely from the captured tree. That is the complete required path, not a fallback. + it('answers from the captured tree when the bound owner exposes no live read', async () => { + const text = await readTextForNode({ + ...baseParams, + readTextAtPoint: undefined, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + expect(readTextAtPoint).not.toHaveBeenCalled(); + }); + + it('falls back to the captured tree through a typed reason when the live read fails', async () => { + readTextAtPoint.mockRejectedValueOnce(new Error('runner transport closed')); + const text = await readTextForNode({ + ...baseParams, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + }); + + it('falls back to the captured tree when the live read returns blank text', async () => { + readTextAtPoint.mockResolvedValueOnce(' '); + const text = await readTextForNode({ + ...baseParams, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + }); }); diff --git a/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts b/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts index c42901dc74..6b9f8c1e06 100644 --- a/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts +++ b/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts @@ -47,6 +47,7 @@ vi.mock('../../../platforms/apple/core/runner/runner-client.ts', async (importOr }; }); +import { getRuntimeBindings, resetGetRuntimeFixture } from './interaction-get-runtime-fixture.ts'; import { dispatchCommand } from '../../../core/dispatch.ts'; const mockDispatch = vi.mocked(dispatchCommand); @@ -57,6 +58,7 @@ beforeEach(() => { mockDispatch.mockResolvedValue({}); mockRunAppleRunnerCommand.mockReset(); mockRunAppleRunnerCommand.mockResolvedValue({}); + resetGetRuntimeFixture(); }); const SAVE_BUTTON_NODES: RawSnapshotNode[] = [ @@ -99,6 +101,7 @@ async function runCommand( sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); } diff --git a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts b/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts index a43d03118a..58eb6418e7 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts @@ -10,6 +10,7 @@ import { activateCompleteRefFrame } from '../../ref-frame.ts'; import type { SessionStore } from '../../session-store.ts'; import type { SessionState } from '../../types.ts'; import { handleInteractionCommands } from '../interaction.ts'; +import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; import { buildSnapshotState } from '../../snapshot-state.ts'; /** @@ -125,6 +126,7 @@ export async function runInteraction( sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); } diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index cf7402d109..7a2f73a2cf 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -58,6 +58,12 @@ vi.mock('../../../platforms/apple/core/runner/runner-client.ts', async (importOr }; }); +import { + elementReadFixtureState, + getRuntimeBindings, + mockReadTextAtPoint, + resetGetRuntimeFixture, +} from './interaction-get-runtime-fixture.ts'; import { dispatchCommand } from '../../../core/dispatch.ts'; const mockDispatch = vi.mocked(dispatchCommand); import { @@ -75,6 +81,7 @@ beforeEach(() => { mockGetAndroidBlockingDialogFocus.mockResolvedValue(null); mockRunAppleRunnerCommand.mockReset(); mockRunAppleRunnerCommand.mockResolvedValue({}); + resetGetRuntimeFixture(); }); test('get text prefers underlying value for text surfaces and avoids recording giant ref labels', async () => { @@ -111,6 +118,7 @@ test('get text prefers underlying value for text surfaces and avoids recording g sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); @@ -145,10 +153,7 @@ test('get text uses backend read expansion when the resolved node has a rect', a }; sessionStore.set(sessionName, session); - mockDispatch.mockResolvedValue({ - action: 'read', - text: 'package com.example.app\nclass MainActivity {}', - }); + mockReadTextAtPoint.mockResolvedValue('package com.example.app\nclass MainActivity {}'); const response = await handleInteractionCommands({ req: { @@ -161,17 +166,64 @@ test('get text uses backend read expansion when the resolved node has a rect', a sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); - expect(mockDispatch).toHaveBeenCalledTimes(1); - expect(mockDispatch.mock.calls[0]?.[1]).toBe('read'); - expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['80', '80']); + // The live read now reaches the bound runtime operation, not the legacy `read` dispatch. + expect(mockDispatch).not.toHaveBeenCalled(); + expect(mockReadTextAtPoint).toHaveBeenCalledTimes(1); + expect(mockReadTextAtPoint.mock.calls[0]?.[0].point).toEqual({ x: 80, y: 80 }); expect(response?.ok).toBe(true); if (response?.ok) { expect(response.data?.text).toBe('package com.example.app\nclass MainActivity {}'); } }); +test('get text answers from the captured tree when the bound owner advertises no live read', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'get-text-no-live-read'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + depth: 0, + type: 'TextView', + label: 'Editor for MainActivity.kt', + value: 'preview only', + rect: { x: 20, y: 40, width: 120, height: 80 }, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + elementReadFixtureState.readTextAtPointAvailable = false; + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'get', + positionals: ['text', '@e1'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + // Preferred-operation absence is not a failure and not a fallback: the required capture path + // answers completely, and nothing reaches the legacy dispatcher. + expect(mockReadTextAtPoint).not.toHaveBeenCalled(); + expect(mockDispatch).not.toHaveBeenCalled(); + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data?.text).toBe('preview only'); + } +}); + test('get text simple iOS id selector uses runner query without snapshot', async () => { const sessionStore = makeSessionStore(); const sessionName = 'get-text-ios-direct-selector'; @@ -205,6 +257,7 @@ test('get text simple iOS id selector uses runner query without snapshot', async sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -292,6 +345,7 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -323,6 +377,7 @@ test('get text simple iOS id selector does not snapshot-fallback on ambiguous ru sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(false); @@ -382,6 +437,7 @@ test('is visible preserves CLI snapshot flags during runtime snapshot capture', sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -413,6 +469,7 @@ test('is visible reuses fresh cached iOS snapshots with rects', async () => { sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -445,6 +502,7 @@ test('is visible recaptures web snapshots when cached nodes may lack rects', asy sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -486,6 +544,7 @@ test('is selected simple iOS id selector uses runner query without snapshot', as sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -544,6 +603,7 @@ test('is simple iOS selector returns false directly when runner predicate fails' sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -599,6 +659,7 @@ test('is simple iOS selector falls back to snapshot while gesture stabilization sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -646,6 +707,7 @@ test('is visible passes for list text that inherits viewport visibility from an sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); @@ -691,6 +753,7 @@ test('is visible fails for nodes outside the current viewport', async () => { sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); @@ -729,6 +792,7 @@ test('is reports Android permission dialog blocker when app content assertion fa sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); diff --git a/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts b/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts index 423965078e..287769ec45 100644 --- a/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts +++ b/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts @@ -84,6 +84,12 @@ function createAdmissionFacts( deployMaterializedApp: cell(options.sourceAvailable), sendPushNotification: cell(options.pushAvailable), networkDump: cell(options.networkAvailable), + readTextAtPoint: unavailable, + deployApp: options.deployAvailable ? available : unavailable, + materializeAppSource: options.sourceAvailable ? available : unavailable, + deployMaterializedApp: options.sourceAvailable ? available : unavailable, + sendPushNotification: options.pushAvailable ? available : unavailable, + networkDump: options.networkAvailable ? available : unavailable, screenRecordingStart: unavailable, screenRecordingReattach: unavailable, screenRecordingCleanup: unavailable, diff --git a/src/daemon/handlers/__tests__/session-command-harness.ts b/src/daemon/handlers/__tests__/session-command-harness.ts index b762f1d357..3b8f90b6a7 100644 --- a/src/daemon/handlers/__tests__/session-command-harness.ts +++ b/src/daemon/handlers/__tests__/session-command-harness.ts @@ -145,6 +145,7 @@ function readinessFacts(device: DeviceInfo): RuntimeFacts { network: { available: false, reason: 'owner-capability-missing' }, screenshot: { available: false, reason: 'owner-capability-missing' }, viewport: { available: false, reason: 'owner-capability-missing' }, + elementText: { available: false, reason: 'owner-capability-missing' }, readiness: { available: false, reason: 'unsupported-device-kind' }, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: { available: false, reason: 'owner-capability-missing' }, @@ -131,6 +132,7 @@ test('appstate rejects web before Android app-state backend dispatch', async () network: { available: false, reason: 'unsupported-platform-leaf' }, screenshot: { available: false, reason: 'unsupported-platform-leaf' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, + elementText: { available: false, reason: 'unsupported-platform-leaf' }, readiness: { available: false, reason: 'unsupported-platform-leaf' }, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: { available: false, reason: 'unsupported-platform-leaf' }, diff --git a/src/daemon/handlers/interaction-common.ts b/src/daemon/handlers/interaction-common.ts index 104c222c77..8edd16f320 100644 --- a/src/daemon/handlers/interaction-common.ts +++ b/src/daemon/handlers/interaction-common.ts @@ -12,6 +12,7 @@ import { inferFillText } from '../action-utils.ts'; import { recordedInputPlaceholder } from '../../replay/recorded-input.ts'; import { parameterizeRecordedFillPayload } from '../parameterized-recorded-fill.ts'; import { isSessionRecording } from '../session-script-publication-capability.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; export type ContextFromFlags = ( flags: CommandFlags | undefined, @@ -25,6 +26,8 @@ export type InteractionHandlerParams = { logPath?: string; sessionStore: SessionStore; contextFromFlags: ContextFromFlags; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; }; export function finalizeTouchInteraction(params: { diff --git a/src/daemon/handlers/interaction-read-legacy-dispatch.ts b/src/daemon/handlers/interaction-read-legacy-dispatch.ts new file mode 100644 index 0000000000..e5bea27edf --- /dev/null +++ b/src/daemon/handlers/interaction-read-legacy-dispatch.ts @@ -0,0 +1,44 @@ +import { dispatchCommand } from '../../core/dispatch.ts'; +import type { SessionState } from '../types.ts'; +import type { ContextFromFlags } from './interaction-common.ts'; +import type { CommandFlags } from '@agent-device/contracts/command'; +import type { ReadElementTextAtPoint } from './interaction-read.ts'; + +/** + * The legacy `read` dispatch, adapted to the neutral point-read shape. + * + * OWNED DEBT, with a named retirement trigger (#1739): `readText` on the shared selector backend + * serves `get text` and read-only `find … get text`. `get` is migrated and passes its bound + * `readTextAtPoint` instead of this adapter; `find` is a separate unit and still reaches the + * platform through the `read` dispatch alias. This module, the `read` registry entry, its + * `dispatch: {}` projection, `DISPATCH_HANDLERS.read`, and `handleReadCommand` all retire + * together in the unit that migrates `find`'s read-only path — the last selector-read consumer. + * + * It is not a fallback: no command chooses between this and a bound operation. Which one the + * shared backend receives is fixed by whether the calling command has cut over. + */ +export function legacyDispatchReadTextAtPoint(params: { + device: SessionState['device']; + flags: CommandFlags | undefined; + surface?: SessionState['surface']; + contextFromFlags: ContextFromFlags; +}): ReadElementTextAtPoint { + return async (input) => { + const rawData = await dispatchCommand( + params.device, + 'read', + [String(input.point.x), String(input.point.y)], + undefined, + { + ...params.contextFromFlags( + params.flags, + input.options?.appBundleId, + input.execution?.traceLogPath, + ), + surface: params.surface, + }, + ); + const data = rawData && typeof rawData === 'object' ? rawData : undefined; + return typeof data?.text === 'string' ? data.text : ''; + }; +} diff --git a/src/daemon/handlers/interaction-read.ts b/src/daemon/handlers/interaction-read.ts index 7f69b6945d..9fdd19ffb1 100644 --- a/src/daemon/handlers/interaction-read.ts +++ b/src/daemon/handlers/interaction-read.ts @@ -1,6 +1,6 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { ElementTextRuntimeOperations } from '@agent-device/contracts/platform'; import { isIosFamily } from '@agent-device/kernel/device'; -import { dispatchCommand } from '../../core/dispatch.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { SessionState } from '../types.ts'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; @@ -8,6 +8,14 @@ import { extractReadableText, prefersValueForReadableText } from '../../utils/te import type { ContextFromFlags } from './interaction-common.ts'; import { resolveRectCenter } from './interaction-targeting.ts'; +export type ReadElementTextAtPoint = ElementTextRuntimeOperations['readTextAtPoint']; + +/** + * When a selector read consults the owner's live point read, and what it does with the answer. + * This module owns that policy only: the read itself is injected, so nothing here names a + * platform. A migrated command passes its bound `readTextAtPoint` — including passing nothing + * when its selected owner's facts report no live read, which is the complete required path. + */ export async function readTextForNode(params: { device: SessionState['device']; node: SnapshotNode; @@ -15,16 +23,20 @@ export async function readTextForNode(params: { appBundleId?: string; traceOutPath?: string; surface?: SessionState['surface']; + readTextAtPoint?: ReadElementTextAtPoint; contextFromFlags: ContextFromFlags; }): Promise { const { device, node, flags, appBundleId, traceOutPath, surface, contextFromFlags } = params; const fallbackText = extractReadableText(node); + const readTextAtPoint = params.readTextAtPoint; + // No live read on this owner: the captured tree is the whole answer, with nothing to disclose. + if (!readTextAtPoint) return fallbackText; const center = resolveRectCenter(node.rect); if (!center) { return fallbackText; } - // iOS only: the XCUITest backend `read` re-resolves the element at a point by enumerating + // iOS only: the XCUITest backend `readText` re-resolves the element at a point by enumerating // the full element tree (allElementsBoundByIndex), which is ~20x slower than the snapshot we // already captured to resolve this node. That re-read only recovers fuller text for // editable/expandable inputs (textField/searchField/textView/…), where the live value can @@ -36,19 +48,22 @@ export async function readTextForNode(params: { return fallbackText; } + const context = contextFromFlags(flags, appBundleId, traceOutPath); try { - const rawData = await dispatchCommand( - device, - 'read', - [String(center.x), String(center.y)], - undefined, - { - ...contextFromFlags(flags, appBundleId, traceOutPath), - surface, + const text = await readTextAtPoint({ + point: center, + options: { appBundleId, surface }, + execution: { + requestId: context.requestId, + verbose: context.verbose, + logPath: context.logPath, + traceLogPath: context.traceLogPath, + iosXctestrunFile: context.iosXctestrunFile, + iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, + iosXctestEnvDir: context.iosXctestEnvDir, + runnerLeaseContext: context.runnerLeaseContext, }, - ); - const data = rawData && typeof rawData === 'object' ? rawData : undefined; - const text = typeof data?.text === 'string' ? data.text : ''; + }); if (text.trim()) { return text; } @@ -64,6 +79,9 @@ export async function readTextForNode(params: { }); return fallbackText; } catch (error) { + // ADR 0019 §2: a preferred operation's failure may fall back to the complete required path + // through a TYPED reason. `interaction_read_fallback` is that reason — structured, never + // sniffed from an error message — and the required path (the captured tree) is what answers. emitDiagnostic({ level: 'warn', phase: 'interaction_read_fallback', diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index fa1d324d93..88f9bb732b 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -1,4 +1,5 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { CaptureSnapshotInput } from '@agent-device/contracts/platform'; import { recordSnapshotTiming, snapshotCaptureAnnotationsFrom, @@ -35,11 +36,13 @@ type CaptureSnapshotParams = { androidFreshnessMode?: AndroidFreshnessMode; signal?: AbortSignal; /** - * Request-bound platform capture. Migrated callers inject the selected - * runtime operation; legacy consumers keep the existing interactor path - * until their own command descriptor cuts over. + * Request-bound platform capture. Migrated callers inject the selected runtime operation; + * legacy consumers keep the existing interactor path until their own command descriptor cuts + * over. The already-resolved neutral capture intent is passed in, so a per-call consumer (the + * selector capture runtime, whose flags differ per call) does not have to rebuild it; a caller + * whose intent is fixed at bind time simply ignores the argument. */ - captureData?: () => Promise; + captureData?: (input: CaptureSnapshotInput) => Promise; }; type SnapshotData = { @@ -81,7 +84,6 @@ export async function captureSnapshot( } export async function captureSnapshotData(params: CaptureSnapshotParams): Promise { - if (params.captureData) return await params.captureData(); const { device, session, logPath } = params; const context = contextFromFlags( logPath, @@ -89,33 +91,33 @@ export async function captureSnapshotData(params: CaptureSnapshotParams): Promis session?.appBundleId, session?.trace?.outPath, ); + const options = { + appBundleId: context.appBundleId, + interactiveOnly: context.snapshotInteractiveOnly, + preferredBackend: context.snapshotPreferredBackend, + depth: context.snapshotDepth, + scope: context.snapshotScope, + raw: context.snapshotRaw, + customActions: context.snapshotCustomActions, + includeRects: params.includeRects, + includeHiddenContentHints: context.snapshotIncludeHiddenContentHints, + surface: session?.surface, + }; + const execution = { + requestId: context.requestId, + verbose: context.verbose, + logPath: context.logPath, + traceLogPath: context.traceLogPath, + iosXctestrunFile: context.iosXctestrunFile, + iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, + iosXctestEnvDir: context.iosXctestEnvDir, + runnerLeaseContext: context.runnerLeaseContext, + }; + if (params.captureData) return await params.captureData({ options, execution }); return await captureSnapshotWithInteractor({ device, - runnerContext: { - requestId: context.requestId, - signal: params.signal, - appBundleId: context.appBundleId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }, - options: { - appBundleId: context.appBundleId, - signal: params.signal, - interactiveOnly: context.snapshotInteractiveOnly, - preferredBackend: context.snapshotPreferredBackend, - depth: context.snapshotDepth, - scope: context.snapshotScope, - raw: context.snapshotRaw, - customActions: context.snapshotCustomActions, - includeRects: params.includeRects, - includeHiddenContentHints: context.snapshotIncludeHiddenContentHints, - surface: session?.surface, - }, + runnerContext: { ...execution, signal: params.signal, appBundleId: context.appBundleId }, + options: { ...options, signal: params.signal }, }); } diff --git a/src/daemon/request-handler-chain.ts b/src/daemon/request-handler-chain.ts index 509696a941..0b1240a863 100644 --- a/src/daemon/request-handler-chain.ts +++ b/src/daemon/request-handler-chain.ts @@ -241,6 +241,8 @@ async function runInteractionHandler( logPath: params.logPath, sessionStore: params.sessionStore, contextFromFlags: params.contextFromFlags, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }), ); } diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index fd71d97fd8..747a936a04 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -1,4 +1,5 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { CaptureSnapshotInput, SnapshotResult } from '@agent-device/contracts/platform'; import type { BackendSnapshotResult } from '../backend.ts'; import { buildSnapshotPresentationKey, @@ -25,6 +26,12 @@ type SelectorCaptureRuntimeParams = { // Sessionless routes have no session record to read the consumed capture back from, so the // capture runtime reports every consumed snapshot here for response-level disclosures. consumedSnapshot?: { state?: SnapshotState }; + /** + * Request-bound platform capture, supplied by a selector command that already bound its device + * runtime. Unmigrated selector commands omit it and keep the legacy interactor capture until + * their own descriptor cuts over. + */ + captureData?: (input: CaptureSnapshotInput) => Promise; }; /** @@ -182,6 +189,7 @@ async function runCapture( snapshotScope, includeRects: request.includeRects, signal: request.signal, + ...(params.captureData ? { captureData: params.captureData } : {}), }); return capture.snapshot; } diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 170ed407d7..e7d4e0d275 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -12,11 +12,14 @@ import { createDaemonRuntimeSessionStore } from './runtime-session.ts'; import { contextFromFlags } from './context.ts'; import { ensureDeviceReady } from './device-ready.ts'; import { readTextForNode } from './handlers/interaction-read.ts'; +import { legacyDispatchReadTextAtPoint } from './handlers/interaction-read-legacy-dispatch.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import type { ContextFromFlags } from './handlers/interaction-common.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; import { createSelectorCaptureRuntime } from './selector-capture-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; +import type { BoundGetRuntimeOperations } from './get-runtime.ts'; import { isActiveProviderDevice } from '../provider-device-runtime.ts'; import { getRequestSignal } from '../request/cancel.ts'; import { snapshotOptionsToFlags } from '../backend-snapshot-options.ts'; @@ -27,6 +30,12 @@ export type SelectorRuntimeParams = { logPath?: string; sessionStore: SessionStore; contextFromFlags?: ContextFromFlags; + /** + * Request-bound device runtime seams. Migrated selector commands resolve their own plan from + * these; unmigrated ones ignore them and keep their legacy admission until their unit lands. + */ + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; // Filled by the capture runtime with the snapshot each selector command actually consumed; // sessionless routes disclose from here because no session record stores the capture. consumedSnapshot?: { state?: SnapshotState }; @@ -36,6 +45,11 @@ export type SelectorRuntimeParams = { type SelectorRuntimeDeviceParams = SelectorRuntimeParams & { session: SessionState | undefined; device: SessionState['device']; + /** + * The operations the calling command already bound. A migrated command passes them so its + * capture and element read execute through its own binding instead of the legacy adapter. + */ + operations?: BoundGetRuntimeOperations; }; type AppleRunnerFindTextTarget = { @@ -64,7 +78,7 @@ export function createSelectorRuntimeForDevice(params: SelectorRuntimeDevicePara export async function createSelectorRuntime( params: SelectorRuntimeParams, - options: { requireSession: boolean; capability: 'find' | 'get' | 'is' }, + options: { requireSession: boolean; capability: 'find' | 'is' }, ): Promise< | { ok: true; runtime: ReturnType } | { ok: false; response: DaemonResponse } @@ -93,6 +107,20 @@ export async function createSelectorRuntime( function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDeviceBackend { const { req, session, device, logPath, sessionName, sessionStore } = params; + const resolveContextFromFlags: ContextFromFlags = + params.contextFromFlags ?? + ((flags, appBundleId, traceLogPath) => + contextFromFlags(logPath ?? '', flags, appBundleId, traceLogPath)); + // A migrated command's binding is authoritative, including when it reports no live read; + // an unmigrated command keeps the legacy dispatch until its own descriptor cuts over. + const readTextAtPoint = params.operations + ? params.operations.readTextAtPoint + : legacyDispatchReadTextAtPoint({ + device, + flags: req.flags, + surface: session?.surface, + contextFromFlags: resolveContextFromFlags, + }); const captureRuntime = createSelectorCaptureRuntime({ device, session, @@ -101,6 +129,7 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice req, consumedSnapshot: params.consumedSnapshot, logPath, + captureData: params.operations?.captureSnapshot, }); return { platform: publicPlatformString(device), @@ -135,10 +164,8 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice appBundleId: session?.appBundleId, traceOutPath: session?.trace?.outPath, surface: session?.surface, - contextFromFlags: - params.contextFromFlags ?? - ((flags, appBundleId, traceLogPath) => - contextFromFlags(logPath ?? '', flags, appBundleId, traceLogPath)), + readTextAtPoint, + contextFromFlags: resolveContextFromFlags, }), }), findText: async (context, text) => ({ diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 9626763e53..26957ec35d 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -52,6 +52,7 @@ import { createSelectorRuntimeForDevice, type SelectorRuntimeParams, } from './selector-runtime-backend.ts'; +import { resolveBoundGetRuntime } from './get-runtime.ts'; export type DirectIosSelectorQueryResult = { found: boolean; @@ -170,11 +171,19 @@ export async function dispatchGetViaRuntime( if (directResponse) return directResponse; } - const resolvedRuntime = await createSelectorRuntime(params, { - requireSession: true, - capability: 'get', + const boundRuntime = await resolveBoundGetRuntime({ + session: params.sessionStore.get(params.sessionName), + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!boundRuntime.ok) return boundRuntime.response; + params.consumedSnapshot ??= {}; + const runtime = createSelectorRuntimeForDevice({ + ...params, + session: boundRuntime.session, + device: boundRuntime.device, + operations: boundRuntime.operations, }); - if (!resolvedRuntime.ok) return resolvedRuntime.response; // #1076 + ADR 0014: a get @ref binds against the retained ref-frame evidence, // so it never silently retargets to a newer positional tree. Its warning is @@ -190,7 +199,7 @@ export async function dispatchGetViaRuntime( }) : undefined; const response = await toDaemonResponse(async () => { - const result = await resolvedRuntime.runtime.selectors.get({ + const result = await runtime.selectors.get({ session: params.sessionName, requestId: req.meta?.requestId, property: sub, diff --git a/src/platform-runtime-element-text-host.ts b/src/platform-runtime-element-text-host.ts new file mode 100644 index 0000000000..158b217392 --- /dev/null +++ b/src/platform-runtime-element-text-host.ts @@ -0,0 +1,71 @@ +import type { + ElementTextRuntimeHost, + ReadTextAtPointInput, +} from '@agent-device/contracts/platform'; +import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; + +/** + * The per-family "read the live text at a point" mechanics, injected into the platform packages + * by composition (ADR 0019 §1). Each family reaches its own tool here instead of importing a + * root module, and every branch is only ever entered for a device whose owner already reported + * `readTextAtPoint` available — so there is no fall-through arm and no failure fallback. + */ +export function createElementTextRuntimeHost(): ElementTextRuntimeHost { + return Object.freeze({ + readTextAtPoint: async (device: DeviceInfo, input: ReadTextAtPointInput) => + await readTextAtPoint(device, input), + }); +} + +async function readTextAtPoint(device: DeviceInfo, input: ReadTextAtPointInput): Promise { + if (device.platform === 'android') return await readAndroidText(device, input); + if (device.platform === 'linux') return await readLinuxText(input); + if (usesMacOsHelperSurface(device, input)) return await readMacOsSurfaceText(input); + // macOS app sessions run through the XCUITest runner; only desktop/menubar surfaces use the + // helper, and every other Apple leaf reaches the runner directly. + return await readAppleRunnerText(device, input); +} + +/** Only non-app macOS surfaces are helper-read; an app session is runner-read like any Apple leaf. */ +function usesMacOsHelperSurface(device: DeviceInfo, input: ReadTextAtPointInput): boolean { + const surface = input.options?.surface; + return isMacOs(device) && surface !== undefined && surface !== 'app'; +} + +async function readAndroidText(device: DeviceInfo, input: ReadTextAtPointInput): Promise { + const { readAndroidTextAtPoint } = await import('./platforms/android/input-actions.ts'); + return (await readAndroidTextAtPoint(device, input.point.x, input.point.y)) ?? ''; +} + +async function readLinuxText(input: ReadTextAtPointInput): Promise { + const { readLinuxTextAtPoint } = await import('./platforms/linux/snapshot.ts'); + return await readLinuxTextAtPoint(input.point.x, input.point.y, input.options?.surface); +} + +async function readMacOsSurfaceText(input: ReadTextAtPointInput): Promise { + const { runMacOsReadTextAction } = await import('./platforms/apple/os/macos/helper.ts'); + const result = await runMacOsReadTextAction(input.point.x, input.point.y, { + bundleId: input.options?.appBundleId, + surface: input.options?.surface, + }); + return result.text; +} + +async function readAppleRunnerText( + device: DeviceInfo, + input: ReadTextAtPointInput, +): Promise { + const { runAppleRunnerCommand } = await import('./platforms/apple/core/runner/runner-client.ts'); + const result = await runAppleRunnerCommand( + device, + { + command: 'readText', + x: input.point.x, + y: input.point.y, + appBundleId: input.options?.appBundleId, + }, + { ...input.execution }, + ); + if (typeof result.text === 'string') return result.text; + return typeof result.message === 'string' ? result.message : ''; +} diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index a5c37b86f5..93366c26c8 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -125,6 +125,7 @@ describe('composed platform runtime gateway', () => { network: unavailable, screenshot: unavailable, viewport: unavailable, + elementText: unavailable, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: unavailable, prepareApplicationOpen: unavailable, diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index 990d93f295..74975f0858 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -305,6 +305,7 @@ function unavailableProviderBinding( network: unavailable, screenshot: unavailable, viewport: unavailable, + elementText: unavailable, lifecycle: unavailableProviderLifecycleFacts(unavailable), }); } @@ -323,6 +324,7 @@ function unavailableProviderFacts(runtime: ProviderDeviceRuntime, device: Device network: unavailable, screenshot: unavailable, viewport: unavailable, + elementText: unavailable, readiness: unavailable, lifecycle: unavailableProviderLifecycleFacts(unavailable), }, diff --git a/src/platform-runtime-operation-host.ts b/src/platform-runtime-operation-host.ts index 5efe999939..1423fc3b20 100644 --- a/src/platform-runtime-operation-host.ts +++ b/src/platform-runtime-operation-host.ts @@ -28,6 +28,7 @@ import { createAndroidApplicationTools } from './platform-runtime-android-applic import { createLocalApplicationInteractorHost } from './platform-runtime-local-application-interactors.ts'; import { createApplicationResourceLifecycle } from './platform-runtime-application-resources.ts'; import { createSnapshotRuntimeHost } from './snapshot/snapshot-desktop-surface.ts'; +import { createElementTextRuntimeHost } from './platform-runtime-element-text-host.ts'; export function createPlatformRuntimeHost(options: { sessionsDir: string; @@ -106,6 +107,7 @@ export function createPlatformRuntimeHost(options: { ), screenRecording: createScreenRecordingRuntimeHost(), snapshot: createSnapshotRuntimeHost(), + elementText: createElementTextRuntimeHost(), localInteractors: createLocalApplicationInteractorHost(), appleApplications, androidApplications, From b86c2f259904a8876c115ae8154ac928abadb18c Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 19 Aug 2026 13:29:51 +0200 Subject: [PATCH 2/9] fix(get): admit before the direct-iOS fast path; close the element-read outcome Review blockers on #1877. 1. `dispatchGetViaRuntime` could complete the direct-iOS selector query before `resolveBoundGetRuntime`. Once `get` declares `device-runtime`, ADR 0019 requires resolve -> admit -> bind before anything in the request path operates, so admission now runs first for every target shape and the fast path is a fast path *within* an admitted request. Regression: an eligible direct selector cannot operate when facts refuse admission. 2. `readTextAtPoint` returned `Promise` and `readTextForNode` caught any throw and fell back, assigning a typed diagnostic after an untyped failure. It now returns a closed `ElementTextReadOutcome`; fallback happens only for the contract's classified reasons; unexpected errors propagate. The reason union is derived from its runtime list so the two cannot drift, and an unhandled reason is a compile error at the consumer. This retires the generic catch the start record promised. --- .../src/element-text-runtime.test.ts | 70 ++++++++++++++ .../contracts/src/element-text-runtime.ts | 38 +++++++- packages/contracts/src/facades/platform.ts | 3 + .../interaction-get-runtime-fixture.ts | 23 ++++- .../__tests__/interaction-read.test.ts | 42 ++++++--- .../handlers/__tests__/interaction.test.ts | 41 +++++++- .../interaction-read-legacy-dispatch.ts | 3 +- src/daemon/handlers/interaction-read.ts | 94 ++++++++++--------- src/daemon/selector-runtime.ts | 16 +++- src/platform-runtime-element-text-host.ts | 44 ++++++--- 10 files changed, 292 insertions(+), 82 deletions(-) create mode 100644 packages/contracts/src/element-text-runtime.test.ts diff --git a/packages/contracts/src/element-text-runtime.test.ts b/packages/contracts/src/element-text-runtime.test.ts new file mode 100644 index 0000000000..143148c2dd --- /dev/null +++ b/packages/contracts/src/element-text-runtime.test.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + elementTextRead, + type ElementTextReadOutcome, + type ElementTextUnreadableReason, +} from './element-text-runtime.ts'; + +/** + * The reasons this suite exercises. Kept local on purpose: exhaustiveness is enforced at the + * CONSUMER by `classifiedFallbackReason`'s `never` arm (a new reason is a compile error there), + * so a second exported runtime list would be an unconsumed parallel source of truth that could + * silently drift. The annotation is what ties this list back to the union. + */ +const UNREADABLE_REASONS: readonly ElementTextUnreadableReason[] = [ + 'no-text-at-point', + 'surface-not-readable', +]; + +/** + * ADR 0019 §2 contract coverage for the preferred element-text read. + * + * A preferred operation may fall its consumer back to the required path only through a TYPED + * reason. These tests pin that the reason set is closed and exhaustively enumerated, so a new + * reason cannot be added without a consumer having to classify it — which is what keeps the + * retired generic `catch` from creeping back as "some other failure, just fall back". + */ + +test('the outcome union is closed: every value is a read or a classified unreadable', () => { + const outcomes: readonly ElementTextReadOutcome[] = [ + elementTextRead('live value'), + ...UNREADABLE_REASONS.map((reason) => ({ status: 'unreadable', reason }) as const), + ]; + for (const outcome of outcomes) { + if (outcome.status === 'read') { + assert.equal(typeof outcome.text, 'string'); + continue; + } + assert.ok( + (UNREADABLE_REASONS as readonly string[]).includes(outcome.reason), + `unreadable outcome carries an unclassified reason: ${outcome.reason}`, + ); + } +}); + +test('a non-blank owner answer is a read that preserves the exact text', () => { + const outcome = elementTextRead(' padded value '); + assert.deepEqual(outcome, { status: 'read', text: ' padded value ' }); +}); + +// Blank is a classification, not a read: an owner answering with whitespace has said there is +// nothing at this point, and saying so by reason keeps consumers off "empty or failed?" guesswork. +for (const [label, value] of [ + ['empty string', ''], + ['whitespace', ' \n\t '], + ['undefined', undefined], + ['null', null], +] as const) { + test(`a ${label} owner answer classifies as no-text-at-point`, () => { + assert.deepEqual(elementTextRead(value), { + status: 'unreadable', + reason: 'no-text-at-point', + }); + }); +} + +test('read outcomes are frozen so a consumer cannot mutate a classification', () => { + assert.ok(Object.isFrozen(elementTextRead('value'))); + assert.ok(Object.isFrozen(elementTextRead(''))); +}); diff --git a/packages/contracts/src/element-text-runtime.ts b/packages/contracts/src/element-text-runtime.ts index c4abb8d9e4..61ca6cb334 100644 --- a/packages/contracts/src/element-text-runtime.ts +++ b/packages/contracts/src/element-text-runtime.ts @@ -17,14 +17,48 @@ export type ReadTextAtPointInput = Readonly<{ execution?: ElementTextRuntimeExecution; }>; +/** + * Why an owner that HAS a live read still produced no text for this point. + * + * Closed on purpose (ADR 0019 §2): a consumer may fall back to the required path only for a + * reason named here. Anything else — a runner transport failure, a helper crash, a bug — is an + * unexpected error and propagates, because silently answering from a stale captured tree after + * an unclassified failure is exactly the "generic catch fallback" the ADR forbids. + */ +export type ElementTextUnreadableReason = + /** The owner queried successfully and there is nothing readable at this point. */ + | 'no-text-at-point' + /** The owner's read surface exists but declined this query (unsupported element/surface). */ + | 'surface-not-readable'; + +/** The closed outcome of one live element-text read. */ +export type ElementTextReadOutcome = + | Readonly<{ status: 'read'; text: string }> + | Readonly<{ status: 'unreadable'; reason: ElementTextUnreadableReason }>; + +/** + * Normalizes a raw owner read into the closed outcome. Blank text is not a read: an owner that + * answers with whitespace has told us there is nothing at this point, and saying so by reason + * keeps every consumer off "did it fail or is it empty?" guesswork. + */ +export function elementTextRead(text: string | undefined | null): ElementTextReadOutcome { + if (typeof text !== 'string' || text.trim().length === 0) { + return Object.freeze({ status: 'unreadable', reason: 'no-text-at-point' } as const); + } + return Object.freeze({ status: 'read', text } as const); +} + export type ElementTextRuntimeOperations = Readonly<{ /** * The live text an owner reads at a point, which can exceed the readable text carried by an * already-captured snapshot node (an editable field whose value is longer than its label). * Declared `preferred`, never `required`: every consumer's required path answers from the * snapshot tree, so an owner without this operation still executes the command completely. + * + * Returns a closed typed outcome rather than a bare string, so a consumer never has to + * distinguish "no text here" from "the read blew up" by catching. */ - readTextAtPoint(input: ReadTextAtPointInput): Promise; + readTextAtPoint(input: ReadTextAtPointInput): Promise; }>; export type ElementTextRuntimeOperationFacts = Readonly<{ @@ -43,7 +77,7 @@ export function elementTextRuntimeOperationFacts( * interactor-resolver seam. */ export type ElementTextRuntimeHost = Readonly<{ - readTextAtPoint(device: DeviceInfo, input: ReadTextAtPointInput): Promise; + readTextAtPoint(device: DeviceInfo, input: ReadTextAtPointInput): Promise; }>; /** Captures one selected owner's read authority for the lifetime of a request binding. */ diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index abcb848dd1..f1166bd646 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -277,13 +277,16 @@ export type { } from '../viewport-runtime.ts'; export { bindElementTextRuntime, + elementTextRead, elementTextRuntimeOperationFacts, } from '../element-text-runtime.ts'; export type { + ElementTextReadOutcome, ElementTextRuntimeExecution, ElementTextRuntimeHost, ElementTextRuntimeOperationFacts, ElementTextRuntimeOperations, + ElementTextUnreadableReason, ReadTextAtPointInput, } from '../element-text-runtime.ts'; export type { diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts index e7fe731977..751ef031cf 100644 --- a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -7,6 +7,7 @@ import { type CaptureSnapshotInput, type DeviceBinding, type PlatformRuntimeOperations, + type ElementTextReadOutcome, type ReadTextAtPointInput, type RuntimeFacts, } from '@agent-device/contracts/platform'; @@ -22,15 +23,27 @@ import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts * at `core/dispatch.ts`. The bound capture still runs the interactor capture the surrounding * interaction tests already mock, so only the two `get` operations are fixture-owned here. */ -export const mockReadTextAtPoint = vi.fn(async (_input: ReadTextAtPointInput) => ''); +export const mockReadTextAtPoint = vi.fn( + async (_input: ReadTextAtPointInput): Promise => + Object.freeze({ status: 'unreadable', reason: 'no-text-at-point' } as const), +); -/** Flip to model an owner whose facts advertise no live element read (web, HarmonyOS, provider). */ -export const elementReadFixtureState = { readTextAtPointAvailable: true }; +/** + * Flip to model an exact owner cell: no live element read (web, HarmonyOS, provider), or no + * capture at all (the watchOS sentinel, an inactive provider), which refuses admission outright. + */ +export const elementReadFixtureState = { + readTextAtPointAvailable: true, + captureSnapshotAvailable: true, +}; export function resetGetRuntimeFixture(): void { mockReadTextAtPoint.mockReset(); - mockReadTextAtPoint.mockResolvedValue(''); + mockReadTextAtPoint.mockResolvedValue( + Object.freeze({ status: 'unreadable', reason: 'no-text-at-point' } as const), + ); elementReadFixtureState.readTextAtPointAvailable = true; + elementReadFixtureState.captureSnapshotAvailable = true; } const available = Object.freeze({ available: true } as const); @@ -61,7 +74,7 @@ function elementReadFacts(device: DeviceInfo): RuntimeFacts 'backend-text'); +const readTextAtPoint = vi.fn( + async (_input: ReadTextAtPointInput): Promise => + elementTextRead('backend-text'), +); function node(overrides: Partial): SnapshotNode { return { @@ -91,8 +98,21 @@ describe('readTextForNode', () => { expect(readTextAtPoint).not.toHaveBeenCalled(); }); - it('falls back to the captured tree through a typed reason when the live read fails', async () => { - readTextAtPoint.mockRejectedValueOnce(new Error('runner transport closed')); + // ADR 0019 §2: the ONLY fallbacks are the contract's classified reasons. + it.each(['no-text-at-point', 'surface-not-readable'] as const)( + 'falls back to the captured tree for the classified reason %s', + async (reason) => { + readTextAtPoint.mockResolvedValueOnce({ status: 'unreadable', reason }); + const text = await readTextForNode({ + ...baseParams, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + }, + ); + + it('classifies a blank live read as no-text-at-point rather than reading blank text', async () => { + readTextAtPoint.mockResolvedValueOnce(elementTextRead(' ')); const text = await readTextForNode({ ...baseParams, node: node({ type: 'textfield', value: 'snap' }), @@ -100,12 +120,12 @@ describe('readTextForNode', () => { expect(text).toBe('snap'); }); - it('falls back to the captured tree when the live read returns blank text', async () => { - readTextAtPoint.mockResolvedValueOnce(' '); - const text = await readTextForNode({ - ...baseParams, - node: node({ type: 'textfield', value: 'snap' }), - }); - expect(text).toBe('snap'); + // The retired generic catch: an unclassified failure must NOT become "this element has no + // text". It propagates, so a runner/helper failure can never be answered from a stale tree. + it('propagates an unexpected live-read failure instead of falling back', async () => { + readTextAtPoint.mockRejectedValueOnce(new Error('runner transport closed')); + await expect( + readTextForNode({ ...baseParams, node: node({ type: 'textfield', value: 'snap' }) }), + ).rejects.toThrow(/runner transport closed/); }); }); diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 7a2f73a2cf..27a73899e6 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -58,6 +58,7 @@ vi.mock('../../../platforms/apple/core/runner/runner-client.ts', async (importOr }; }); +import { elementTextRead } from '@agent-device/contracts/platform'; import { elementReadFixtureState, getRuntimeBindings, @@ -153,7 +154,9 @@ test('get text uses backend read expansion when the resolved node has a rect', a }; sessionStore.set(sessionName, session); - mockReadTextAtPoint.mockResolvedValue('package com.example.app\nclass MainActivity {}'); + mockReadTextAtPoint.mockResolvedValue( + elementTextRead('package com.example.app\nclass MainActivity {}'), + ); const response = await handleInteractionCommands({ req: { @@ -224,6 +227,42 @@ test('get text answers from the captured tree when the bound owner advertises no } }); +// ADR 0019 regression: `get` declares `device-runtime`, so an ELIGIBLE direct-iOS selector — +// one the fast path would otherwise answer without a tree capture — must not reach the device +// until the request has resolved, admitted, and bound. A refused admission means zero runner +// queries, not a fast-path answer that skipped exact-owner facts entirely. +test('an eligible direct iOS selector cannot operate before admission', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'get-text-direct-before-admission'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + elementReadFixtureState.captureSnapshotAvailable = false; + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + text: 'Ada Lovelace', + nodes: [{ index: 0, depth: 0, type: 'StaticText', label: 'Ada Lovelace' }], + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'get', + positionals: ['text', 'id=name'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); + // The whole point: the fast path never ran. + expect(mockRunAppleRunnerCommand).not.toHaveBeenCalled(); + expect(mockDispatch).not.toHaveBeenCalled(); +}); + test('get text simple iOS id selector uses runner query without snapshot', async () => { const sessionStore = makeSessionStore(); const sessionName = 'get-text-ios-direct-selector'; diff --git a/src/daemon/handlers/interaction-read-legacy-dispatch.ts b/src/daemon/handlers/interaction-read-legacy-dispatch.ts index e5bea27edf..a609cdbea7 100644 --- a/src/daemon/handlers/interaction-read-legacy-dispatch.ts +++ b/src/daemon/handlers/interaction-read-legacy-dispatch.ts @@ -2,6 +2,7 @@ import { dispatchCommand } from '../../core/dispatch.ts'; import type { SessionState } from '../types.ts'; import type { ContextFromFlags } from './interaction-common.ts'; import type { CommandFlags } from '@agent-device/contracts/command'; +import { elementTextRead } from '@agent-device/contracts/platform'; import type { ReadElementTextAtPoint } from './interaction-read.ts'; /** @@ -39,6 +40,6 @@ export function legacyDispatchReadTextAtPoint(params: { }, ); const data = rawData && typeof rawData === 'object' ? rawData : undefined; - return typeof data?.text === 'string' ? data.text : ''; + return elementTextRead(typeof data?.text === 'string' ? data.text : undefined); }; } diff --git a/src/daemon/handlers/interaction-read.ts b/src/daemon/handlers/interaction-read.ts index 9fdd19ffb1..a558b035f8 100644 --- a/src/daemon/handlers/interaction-read.ts +++ b/src/daemon/handlers/interaction-read.ts @@ -1,5 +1,8 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import type { ElementTextRuntimeOperations } from '@agent-device/contracts/platform'; +import type { + ElementTextRuntimeOperations, + ElementTextUnreadableReason, +} from '@agent-device/contracts/platform'; import { isIosFamily } from '@agent-device/kernel/device'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { SessionState } from '../types.ts'; @@ -49,50 +52,51 @@ export async function readTextForNode(params: { } const context = contextFromFlags(flags, appBundleId, traceOutPath); - try { - const text = await readTextAtPoint({ - point: center, - options: { appBundleId, surface }, - execution: { - requestId: context.requestId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }, - }); - if (text.trim()) { - return text; + // No try/catch: an unexpected read failure propagates. Only the outcomes the contract + // classifies fall back to the captured tree (ADR 0019 §2 typed reason), so a runner or + // helper failure can never masquerade as "this element has no text". + const outcome = await readTextAtPoint({ + point: center, + options: { appBundleId, surface }, + execution: { + requestId: context.requestId, + verbose: context.verbose, + logPath: context.logPath, + traceLogPath: context.traceLogPath, + iosXctestrunFile: context.iosXctestrunFile, + iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, + iosXctestEnvDir: context.iosXctestEnvDir, + runnerLeaseContext: context.runnerLeaseContext, + }, + }); + if (outcome.status === 'read') return outcome.text; + emitDiagnostic({ + level: 'warn', + phase: 'interaction_read_fallback', + data: { + reason: classifiedFallbackReason(outcome.reason), + nodeRef: node.ref, + surface, + platform: device.platform, + }, + }); + return fallbackText; +} + +/** + * The typed reason a fallback to the captured tree is allowed, one diagnostic reason per + * classified outcome. The `satisfies never` arm makes a new `ElementTextUnreadableReason` + * a COMPILE error here rather than a silent untyped fallback. + */ +function classifiedFallbackReason(reason: ElementTextUnreadableReason): string { + switch (reason) { + case 'no-text-at-point': + return 'no_text_at_point'; + case 'surface-not-readable': + return 'surface_not_readable'; + default: { + const unhandled: never = reason; + return unhandled; } - emitDiagnostic({ - level: 'warn', - phase: 'interaction_read_fallback', - data: { - reason: 'empty_backend_text', - nodeRef: node.ref, - surface, - platform: device.platform, - }, - }); - return fallbackText; - } catch (error) { - // ADR 0019 §2: a preferred operation's failure may fall back to the complete required path - // through a TYPED reason. `interaction_read_fallback` is that reason — structured, never - // sniffed from an error message — and the required path (the captured tree) is what answers. - emitDiagnostic({ - level: 'warn', - phase: 'interaction_read_fallback', - data: { - reason: 'backend_read_failed', - nodeRef: node.ref, - surface, - platform: device.platform, - error: error instanceof Error ? error.message : String(error), - }, - }); - return fallbackText; } } diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 26957ec35d..6658350570 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -166,11 +166,13 @@ export async function dispatchGetViaRuntime( // ADR 0012 step 4: a guarded replay dispatch must resolve through the // snapshot path so the post-resolution identity guard runs. const replayTargetGuard = req.internal?.replayTargetGuard; - if (target.target.kind === 'selector' && !replayTargetGuard) { - const directResponse = await dispatchDirectIosSelectorGet(params, sub, target.target.selector); - if (directResponse) return directResponse; - } + // ADR 0019: `get` declares `device-runtime`, so NOTHING in its request path may reach the + // device before resolve -> admit -> bind. Admission runs first for every target shape, + // including the ones the direct-iOS fast path below can answer: that path is a fast path + // *within* an admitted request, never a way around exact-owner facts or the one-binding + // invariant. (The query itself is still the shared root mechanic co-owned by `is`, `wait`, + // and the Wave 5 offscreen probe — this unit orders it, it does not claim it.) const boundRuntime = await resolveBoundGetRuntime({ session: params.sessionStore.get(params.sessionName), inspectFacts: params.inspectFacts, @@ -178,6 +180,12 @@ export async function dispatchGetViaRuntime( }); if (!boundRuntime.ok) return boundRuntime.response; params.consumedSnapshot ??= {}; + + if (target.target.kind === 'selector' && !replayTargetGuard) { + const directResponse = await dispatchDirectIosSelectorGet(params, sub, target.target.selector); + if (directResponse) return directResponse; + } + const runtime = createSelectorRuntimeForDevice({ ...params, session: boundRuntime.session, diff --git a/src/platform-runtime-element-text-host.ts b/src/platform-runtime-element-text-host.ts index 158b217392..cdd8ef7831 100644 --- a/src/platform-runtime-element-text-host.ts +++ b/src/platform-runtime-element-text-host.ts @@ -1,6 +1,8 @@ -import type { - ElementTextRuntimeHost, - ReadTextAtPointInput, +import { + elementTextRead, + type ElementTextReadOutcome, + type ElementTextRuntimeHost, + type ReadTextAtPointInput, } from '@agent-device/contracts/platform'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; @@ -17,7 +19,10 @@ export function createElementTextRuntimeHost(): ElementTextRuntimeHost { }); } -async function readTextAtPoint(device: DeviceInfo, input: ReadTextAtPointInput): Promise { +async function readTextAtPoint( + device: DeviceInfo, + input: ReadTextAtPointInput, +): Promise { if (device.platform === 'android') return await readAndroidText(device, input); if (device.platform === 'linux') return await readLinuxText(input); if (usesMacOsHelperSurface(device, input)) return await readMacOsSurfaceText(input); @@ -32,29 +37,38 @@ function usesMacOsHelperSurface(device: DeviceInfo, input: ReadTextAtPointInput) return isMacOs(device) && surface !== undefined && surface !== 'app'; } -async function readAndroidText(device: DeviceInfo, input: ReadTextAtPointInput): Promise { +// Each reader classifies its own owner's "nothing here" answer through `elementTextRead`. +// None of them catches: a transport or tooling failure is unexpected and propagates. + +async function readAndroidText( + device: DeviceInfo, + input: ReadTextAtPointInput, +): Promise { const { readAndroidTextAtPoint } = await import('./platforms/android/input-actions.ts'); - return (await readAndroidTextAtPoint(device, input.point.x, input.point.y)) ?? ''; + // uiautomator answers `undefined` when no node covers the point. + return elementTextRead(await readAndroidTextAtPoint(device, input.point.x, input.point.y)); } -async function readLinuxText(input: ReadTextAtPointInput): Promise { +async function readLinuxText(input: ReadTextAtPointInput): Promise { const { readLinuxTextAtPoint } = await import('./platforms/linux/snapshot.ts'); - return await readLinuxTextAtPoint(input.point.x, input.point.y, input.options?.surface); + return elementTextRead( + await readLinuxTextAtPoint(input.point.x, input.point.y, input.options?.surface), + ); } -async function readMacOsSurfaceText(input: ReadTextAtPointInput): Promise { +async function readMacOsSurfaceText(input: ReadTextAtPointInput): Promise { const { runMacOsReadTextAction } = await import('./platforms/apple/os/macos/helper.ts'); const result = await runMacOsReadTextAction(input.point.x, input.point.y, { bundleId: input.options?.appBundleId, surface: input.options?.surface, }); - return result.text; + return elementTextRead(result.text); } async function readAppleRunnerText( device: DeviceInfo, input: ReadTextAtPointInput, -): Promise { +): Promise { const { runAppleRunnerCommand } = await import('./platforms/apple/core/runner/runner-client.ts'); const result = await runAppleRunnerCommand( device, @@ -66,6 +80,10 @@ async function readAppleRunnerText( }, { ...input.execution }, ); - if (typeof result.text === 'string') return result.text; - return typeof result.message === 'string' ? result.message : ''; + if (typeof result.text === 'string') return elementTextRead(result.text); + // The runner answers `message` instead of `text` when it queried the element but could not + // render readable text from it — a declined read, not a failed one. + return typeof result.message === 'string' + ? elementTextRead(result.message) + : Object.freeze({ status: 'unreadable', reason: 'surface-not-readable' } as const); } From b565a5cb2ab18693b5c134da98e465caff823cf7 Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 19 Aug 2026 14:37:30 +0200 Subject: [PATCH 3/9] feat(daemon): land the selector capture seam with get as its first consumer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Takes ownership of the request-bound selector capture seam from #1876, which cannot ship standalone: with find's cutover deferred it had no consuming command (ADR 0019 §10) and was not dead-code clean (check:production-exports 19 -> 20). `get` is its first consumer, so it lands here. Adopts find's handoff as given. The one shape change, approved by the coordinator: the selector family gets its own capture uses carrying a PREFERRED `readTextAtPoint`, declared ALONGSIDE the snapshot uses so `snapshot`/`diff` keep binding exactly what they bind today. The read is surfaced through the existing arms of `bindSnapshotCaptureRuntime`, reusing the same selectActiveAppSnapshot / selectSnapshotWithoutActiveApp selectors — no second plan-to-operation dispatch. `get` now runs through `createBoundSelectorRuntime`; `resolveBoundGetRuntime` and its test are deleted as superseded, and `'get'` leaves the `createSelectorRuntime` capability union. The legacy read adapter survives for `find get text` and is selected by which command constructed the runtime — never by failure, family, environment, or flag — so `get` cannot reach it. It retires in find's cutover, where the last consumer moves. --- packages/contracts/src/facades/platform.ts | 15 +- .../src/platform-runtime-operations.ts | 112 +++++++---- .../layering/runtime-command-cutover-table.ts | 13 +- src/__tests__/test-utils/session-factories.ts | 9 + src/core/command-descriptor/registry.ts | 4 +- src/daemon/__tests__/get-runtime.test.ts | 184 ------------------ .../selector-capture-binding.test.ts | 163 ++++++++++++++++ .../__tests__/selector-capture-fixture.ts | 98 ++++++++++ src/daemon/get-runtime.ts | 129 ------------ .../interaction-get-runtime-fixture.ts | 4 + .../__tests__/interaction-touch-fixtures.ts | 8 +- src/daemon/handlers/snapshot-capture.ts | 62 +++--- src/daemon/selector-capture-binding.ts | 73 +++++++ src/daemon/selector-capture-runtime.ts | 42 ++-- src/daemon/selector-runtime-backend.ts | 116 +++++++---- src/daemon/selector-runtime.ts | 19 +- src/daemon/snapshot-runtime-binding.ts | 73 ++++++- 17 files changed, 660 insertions(+), 464 deletions(-) delete mode 100644 src/daemon/__tests__/get-runtime.test.ts create mode 100644 src/daemon/__tests__/selector-capture-binding.test.ts create mode 100644 src/daemon/__tests__/selector-capture-fixture.ts delete mode 100644 src/daemon/get-runtime.ts create mode 100644 src/daemon/selector-capture-binding.ts diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index f1166bd646..7e412861fc 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -214,12 +214,18 @@ export { captureSnapshotUse, defineUse, resolveScreenshotRuntimePlan, + resolveSelectorCaptureRuntimePlan, resolveSnapshotRuntimePlan, screenshotRuntimePlanUses, + selectorCaptureRuntimePlanUses, snapshotRuntimePlanUses, viewportRuntimeUse, } from '../platform-runtime-operations.ts'; -export type { ScreenshotRuntimePlan, SnapshotRuntimePlan } from '../platform-runtime-operations.ts'; +export type { + ScreenshotRuntimePlan, + SelectorCaptureRuntimePlan, + SnapshotRuntimePlan, +} from '../platform-runtime-operations.ts'; export type { PlatformRuntimeHost, PlatformRuntimeModule, @@ -235,13 +241,8 @@ export { appStateRuntimeUses, appStateUse, shutdownTargetUse, - elementReadRuntimeUse, - elementReadRuntimePlan, -} from '../platform-runtime-operations.ts'; -export type { - DeviceReadinessRuntimePlan, - ElementReadRuntimePlan, } from '../platform-runtime-operations.ts'; +export type { DeviceReadinessRuntimePlan } from '../platform-runtime-operations.ts'; export { bindLocalScreenshotInteractor, bindProviderScreenshotInteractor, diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 845a45caa0..d30c1b34da 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -71,30 +71,6 @@ export const bootTargetHeadlessUse = defineUse({ export const appsRuntimeUse = defineUse({ required: ['ensureReady', 'listApps'] }); export const captureSnapshotUse = defineUse({ required: ['captureSnapshot'] }); export const viewportRuntimeUse = defineUse({ required: ['setViewport'] }); -/** - * `get` reads one element's text or attributes. The required path answers from the captured - * tree on every supported cell; `readTextAtPoint` is the owner-provided live read that recovers - * fuller text for editable/expandable elements, so it is preferred rather than required (ADR - * 0019 §2). An owner without it still executes `get` completely. - */ -export const elementReadRuntimeUse = defineUse({ - required: ['captureSnapshot'], - preferred: ['readTextAtPoint'], -}); - -/** - * `get`'s use does not vary with its input, so there is nothing to resolve: the one plan is a - * frozen constant rather than a `resolve…RuntimePlan` over a single row. - */ -export type ElementReadRuntimePlan = Readonly<{ - kind: 'element-read'; - use: typeof elementReadRuntimeUse; -}>; - -export const elementReadRuntimePlan: ElementReadRuntimePlan = Object.freeze({ - kind: 'element-read', - use: elementReadRuntimeUse, -}); const captureSnapshotWithCustomActionsUse = defineUse({ required: ['captureSnapshot', 'captureSnapshotWithCustomActions'], }); @@ -109,6 +85,31 @@ const captureSnapshotWithCustomActionsWithoutActiveAppUse = defineUse({ ], }); +/** + * The selector family's capture uses. Declared ALONGSIDE the snapshot uses above, never in place + * of them: `snapshot`/`diff` keep binding exactly what they bind today. The only difference is the + * PREFERRED element read — every selector read's required path answers from the captured tree, so + * an owner without the read still executes the command completely (ADR 0019 §2), but an owner that + * has one lets `get text` return the live value a truncated snapshot node cannot. + */ +const selectorCaptureUse = defineUse({ + required: ['captureSnapshot'], + preferred: ['readTextAtPoint'], +}); +const selectorCaptureWithoutActiveAppUse = defineUse({ + required: ['captureSnapshot', 'captureSnapshotWithoutActiveApp'], + preferred: ['readTextAtPoint'], +}); + +/** + * The selector family (`find`, `get`, `is`, `wait`) resolves targets from the plain accessibility + * capture: it exposes no `--actions` surface, so only the active-app split applies. + */ +export const selectorCaptureRuntimePlanUses = Object.freeze([ + selectorCaptureUse, + selectorCaptureWithoutActiveAppUse, +] as const); + export const snapshotRuntimePlanUses = Object.freeze([ captureSnapshotUse, captureSnapshotWithCustomActionsUse, @@ -138,30 +139,67 @@ export type SnapshotRuntimePlan = use: typeof captureSnapshotWithoutActiveAppUse; }>; +/** + * Same two `kind`s the snapshot plan uses for this split — deliberately, so the shared + * admit-then-bind path keeps ONE set of arms rather than growing a parallel dispatch — but + * carrying the selector uses, which add the preferred element read. + */ +export type SelectorCaptureRuntimePlan = + | Readonly<{ + kind: 'selector-active-app'; + operation: 'captureSnapshot'; + use: typeof selectorCaptureUse; + }> + | Readonly<{ + kind: 'selector-without-active-app'; + operation: 'captureSnapshotWithoutActiveApp'; + use: typeof selectorCaptureWithoutActiveAppUse; + }>; + +/** + * The active-app split every selector capture selects from. The selector family exposes no + * `--actions` surface, so custom actions are outside its declaration. + */ +export function resolveSelectorCaptureRuntimePlan( + input: Readonly<{ hasActiveApp: boolean }>, +): SelectorCaptureRuntimePlan { + return input.hasActiveApp + ? Object.freeze({ + kind: 'selector-active-app', + operation: 'captureSnapshot', + use: selectorCaptureUse, + }) + : Object.freeze({ + kind: 'selector-without-active-app', + operation: 'captureSnapshotWithoutActiveApp', + use: selectorCaptureWithoutActiveAppUse, + }); +} + /** Selects one owner-fact-backed capture plan from normalized command/session intent. */ export function resolveSnapshotRuntimePlan(input: { customActions: boolean; hasActiveApp: boolean; }): SnapshotRuntimePlan { - if (input.customActions) { + if (!input.customActions) { return input.hasActiveApp - ? Object.freeze({ - kind: 'custom-actions-active-app', - operation: 'captureSnapshotWithCustomActions', - use: captureSnapshotWithCustomActionsUse, - }) + ? Object.freeze({ kind: 'active-app', operation: 'captureSnapshot', use: captureSnapshotUse }) : Object.freeze({ - kind: 'custom-actions-without-active-app', - operation: 'captureSnapshotWithCustomActions', - use: captureSnapshotWithCustomActionsWithoutActiveAppUse, + kind: 'without-active-app', + operation: 'captureSnapshotWithoutActiveApp', + use: captureSnapshotWithoutActiveAppUse, }); } return input.hasActiveApp - ? Object.freeze({ kind: 'active-app', operation: 'captureSnapshot', use: captureSnapshotUse }) + ? Object.freeze({ + kind: 'custom-actions-active-app', + operation: 'captureSnapshotWithCustomActions', + use: captureSnapshotWithCustomActionsUse, + }) : Object.freeze({ - kind: 'without-active-app', - operation: 'captureSnapshotWithoutActiveApp', - use: captureSnapshotWithoutActiveAppUse, + kind: 'custom-actions-without-active-app', + operation: 'captureSnapshotWithCustomActions', + use: captureSnapshotWithCustomActionsWithoutActiveAppUse, }); } diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index ddf0825fb9..b00827ed62 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -521,13 +521,18 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ routeNames: ['WEB_QUERY_COMMANDS_WITH_GET', 'HARMONYOS_GET_SUPPORT'], }, runtimeTypeNames: ['ElementTextRuntimeOperations', 'SnapshotRuntimeOperations'], - operations: { names: ['captureSnapshot', 'readTextAtPoint'] }, + operations: { + names: ['captureSnapshot', 'captureSnapshotWithoutActiveApp', 'readTextAtPoint'], + }, singularExecution: { routes: ['dispatchGetViaRuntime'], - operations: ['captureSnapshot', 'readTextAtPoint'], + operations: ['captureSnapshot', 'captureSnapshotWithoutActiveApp', 'readTextAtPoint'], + // `get` executes through the shared selector seam, so the capture owners are the SAME + // selectors `snapshot`/`diff` count; only the preferred element read is this unit's own. operationOwners: { - captureSnapshot: ['selectElementReadOperations'], - readTextAtPoint: ['selectElementTextRead'], + captureSnapshot: ['selectActiveAppSnapshot'], + captureSnapshotWithoutActiveApp: ['selectSnapshotWithoutActiveApp'], + readTextAtPoint: ['bindElementRead'], }, }, }, diff --git a/src/__tests__/test-utils/session-factories.ts b/src/__tests__/test-utils/session-factories.ts index 5c88d362e8..f7a6b95fd7 100644 --- a/src/__tests__/test-utils/session-factories.ts +++ b/src/__tests__/test-utils/session-factories.ts @@ -51,6 +51,15 @@ export function makeIosSession(name: string, overrides?: Partial): return makeSession(name, { device: IOS_SIMULATOR, ...overrides }); } +/** + * An iOS session with a tracked app — what `open ` produces. The shared snapshot + * runtime exposes capture on an iOS leaf only through the active-app plan row, so a test + * that captures on iOS needs this rather than a bare session. + */ +export function makeIosAppSession(name: string, overrides?: Partial): SessionState { + return makeIosSession(name, { appBundleId: 'com.example.app', ...overrides }); +} + export function makeAndroidSession(name: string, overrides?: Partial): SessionState { return makeSession(name, { device: ANDROID_EMULATOR, ...overrides }); } diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index ec926da2cc..7c29ca921b 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -26,7 +26,7 @@ import { readySendPushNotificationUse, openApplicationRuntimePlanUses, closeApplicationRuntimePlanUses, - elementReadRuntimeUse, + selectorCaptureRuntimePlanUses, snapshotRuntimePlanUses, prepareAppleRunnerRuntimeUse, runtimeCommandRuntimePlanUses, @@ -1191,7 +1191,7 @@ export const RAW_COMMAND_DESCRIPTORS = [ daemon: { route: 'interaction', refFrameEffect: 'preserve' }, timeoutPolicy: postActionObservationTimeoutPolicy('get', PRESERVE_DAEMON_TIMEOUT_POLICY), batchable: true, - platformExecution: { kind: 'device-runtime', uses: [elementReadRuntimeUse] as const }, + platformExecution: { kind: 'device-runtime', uses: selectorCaptureRuntimePlanUses }, }, { name: 'read', diff --git a/src/daemon/__tests__/get-runtime.test.ts b/src/daemon/__tests__/get-runtime.test.ts deleted file mode 100644 index 79707d51ef..0000000000 --- a/src/daemon/__tests__/get-runtime.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { - applicationLifecycleOperationFacts, - createUnavailablePlatformRuntimeFacts, - localRuntimeOwner, - narrowDeviceBinding, - providerRuntimeOwner, - type DeviceBinding, - type PlatformRuntimeOperations, - type RuntimeFacts, - type RuntimeOperationUnavailability, - type RuntimeOwnerRef, -} from '@agent-device/contracts/platform'; -import type { DeviceInfo } from '@agent-device/kernel/device'; -import { resolveBoundGetRuntime } from '../get-runtime.ts'; -import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; -import type { SessionState } from '../types.ts'; -import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; - -const available = Object.freeze({ available: true } as const); - -function unavailableFact(reason: RuntimeOperationUnavailability['reason'], hint?: string) { - return Object.freeze({ available: false, reason, ...(hint ? { hint } : {}) } as const); -} - -function facts( - device: DeviceInfo, - owner: RuntimeOwnerRef, - operations: Readonly<{ - captureSnapshot: RuntimeFacts['operations']['captureSnapshot']; - readTextAtPoint: RuntimeFacts['operations']['readTextAtPoint']; - }>, -): RuntimeFacts { - const missing = unavailableFact('owner-capability-missing'); - const base = createUnavailablePlatformRuntimeFacts(device, owner, { - appLog: missing, - network: missing, - viewport: missing, - elementText: missing, - lifecycle: applicationLifecycleOperationFacts({ - resolveOpenTarget: missing, - prepareApplicationOpen: missing, - openApplication: missing, - applyRuntimeHints: missing, - clearRuntimeHints: missing, - closeApplication: missing, - finalizeApplicationClose: missing, - prepareAppleRunner: missing, - configureProviderPortReverse: missing, - }), - }); - return Object.freeze({ - device: base.device, - operations: { ...base.operations, ...operations }, - }); -} - -function harness( - options: Readonly<{ - owner?: RuntimeOwnerRef; - captureSnapshot?: RuntimeFacts['operations']['captureSnapshot']; - readTextAtPoint?: RuntimeFacts['operations']['readTextAtPoint']; - }> = {}, -) { - const owner = options.owner ?? localRuntimeOwner('apple'); - const operations = { - captureSnapshot: options.captureSnapshot ?? available, - readTextAtPoint: options.readTextAtPoint ?? available, - }; - const captureSnapshot = vi.fn(async () => ({ backend: 'xctest', nodes: [] }) as never); - const readTextAtPoint = vi.fn(async () => 'live text'); - const inspectFacts = vi.fn(async (device: DeviceInfo) => - facts(device, owner, operations), - ) as InspectDeviceRuntimeFacts; - const bindDevice = vi.fn(async (device: DeviceInfo, use) => { - const binding = Object.freeze({ - device, - owner, - facts: facts(device, owner, operations), - operations: Object.freeze({ - ...(operations.captureSnapshot.available ? { captureSnapshot } : {}), - ...(operations.readTextAtPoint.available ? { readTextAtPoint } : {}), - }), - [Symbol.asyncDispose]: async () => undefined, - }) as unknown as DeviceBinding; - return narrowDeviceBinding(binding, use); - }) as BindDeviceRuntime; - return { inspectFacts, bindDevice, captureSnapshot, readTextAtPoint }; -} - -function session(): SessionState { - return makeIosSession('get-runtime', { appBundleId: 'com.example.app' }); -} - -describe('resolveBoundGetRuntime', () => { - it('refuses without an active session before touching facts or binding', async () => { - const seams = harness(); - const resolved = await resolveBoundGetRuntime({ session: undefined, ...seams }); - expect(resolved.ok).toBe(false); - expect(seams.inspectFacts).not.toHaveBeenCalled(); - expect(seams.bindDevice).not.toHaveBeenCalled(); - }); - - it('inspects owner facts exactly once and binds exactly once', async () => { - const seams = harness(); - const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); - expect(resolved.ok).toBe(true); - expect(seams.inspectFacts).toHaveBeenCalledTimes(1); - expect(seams.bindDevice).toHaveBeenCalledTimes(1); - }); - - it('binds the admitted device with the element-read use', async () => { - const seams = harness(); - const active = session(); - await resolveBoundGetRuntime({ session: active, ...seams }); - const [boundDevice, use] = vi.mocked(seams.bindDevice).mock.calls[0] ?? []; - expect(boundDevice?.id).toBe(active.device.id); - expect(use).toEqual({ required: ['captureSnapshot'], preferred: ['readTextAtPoint'] }); - }); - - it('refuses before binding when the required capture is unavailable', async () => { - const seams = harness({ - captureSnapshot: unavailableFact('unsupported-platform-leaf', 'no snapshot backend'), - }); - const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); - expect(resolved.ok).toBe(false); - if (!resolved.ok && !resolved.response.ok) { - expect(resolved.response.error.code).toBe('UNSUPPORTED_OPERATION'); - expect(resolved.response.error.hint).toBe('no snapshot backend'); - } - expect(seams.inspectFacts).toHaveBeenCalledTimes(1); - expect(seams.bindDevice).not.toHaveBeenCalled(); - }); - - it('still admits and binds when only the preferred read is unavailable', async () => { - const seams = harness({ readTextAtPoint: unavailableFact('unsupported-platform-leaf') }); - const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); - expect(resolved.ok).toBe(true); - if (resolved.ok) { - expect(resolved.operations.captureSnapshot).toBeTypeOf('function'); - expect(resolved.operations.readTextAtPoint).toBeUndefined(); - } - expect(seams.bindDevice).toHaveBeenCalledTimes(1); - }); - - it('exposes the preferred read when the owner advertises it', async () => { - const seams = harness(); - const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); - expect(resolved.ok).toBe(true); - if (resolved.ok) { - await resolved.operations.readTextAtPoint?.({ point: { x: 1, y: 2 } }); - expect(seams.readTextAtPoint).toHaveBeenCalledTimes(1); - } - }); - - // Provider ownership is authoritative: an unavailable provider fact fails closed rather than - // borrowing the local family runtime. - it('fails closed for a provider owner whose facts refuse the capture', async () => { - const seams = harness({ - owner: providerRuntimeOwner('webdriver', 'tenant-a'), - captureSnapshot: unavailableFact('unsupported-provider-mode'), - }); - const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); - expect(resolved.ok).toBe(false); - expect(seams.bindDevice).not.toHaveBeenCalled(); - }); - - it('binds a provider owner that advertises capture but no live read', async () => { - const seams = harness({ - owner: providerRuntimeOwner('webdriver', 'tenant-a'), - readTextAtPoint: unavailableFact('unsupported-provider-mode'), - }); - const resolved = await resolveBoundGetRuntime({ session: session(), ...seams }); - expect(resolved.ok).toBe(true); - if (resolved.ok) expect(resolved.operations.readTextAtPoint).toBeUndefined(); - }); - - it('refuses to bind without a runtime gateway', async () => { - const seams = harness(); - await expect( - resolveBoundGetRuntime({ session: session(), inspectFacts: seams.inspectFacts }), - ).rejects.toThrow(/binding is unavailable/i); - }); -}); diff --git a/src/daemon/__tests__/selector-capture-binding.test.ts b/src/daemon/__tests__/selector-capture-binding.test.ts new file mode 100644 index 0000000000..29045be675 --- /dev/null +++ b/src/daemon/__tests__/selector-capture-binding.test.ts @@ -0,0 +1,163 @@ +import { expect, test } from 'vitest'; +import type { CaptureSnapshotInput } from '@agent-device/contracts/platform'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import { + makeAndroidSession, + makeIosSession, +} from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inventory-gateways.ts'; +import { resolveBoundSelectorCapture } from '../selector-capture-binding.ts'; +import { createBoundSelectorRuntime } from '../selector-runtime-backend.ts'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; + +// The seam every selector unit consumes. `find` landed it; `get`, `is`, and `wait` migrate by +// naming their command, so these are the guarantees they inherit rather than re-derive. + +test('an available plan inspects once and binds once, on the admitted device', async () => { + const fixture = selectorCaptureFixture(); + + const bound = await resolveBoundSelectorCapture({ + command: 'find', + device: ANDROID_EMULATOR, + session: makeAndroidSession('selector'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(bound.ok).toBe(true); + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); +}); + +test('repeated captures reuse the one binding the plan was admitted for', async () => { + const fixture = selectorCaptureFixture(); + const bound = await resolveBoundSelectorCapture({ + command: 'wait', + device: ANDROID_EMULATOR, + session: makeAndroidSession('selector'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!bound.ok) throw new Error('expected an admitted capture'); + + await bound.operations.capture({}); + await bound.operations.capture({}); + + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); + expect(fixture.captures).toHaveLength(2); +}); + +test('a per-capture signal reaches the bound operation for a poll deadline', async () => { + const fixture = selectorCaptureFixture(); + const bound = await resolveBoundSelectorCapture({ + command: 'wait', + device: ANDROID_EMULATOR, + session: makeAndroidSession('selector'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!bound.ok) throw new Error('expected an admitted capture'); + const deadline = new AbortController(); + deadline.abort(new Error('poll deadline')); + + const input: CaptureSnapshotInput = { signal: deadline.signal }; + await expect(bound.operations.capture(input)).rejects.toThrow(/poll deadline/); +}); + +test('an unavailable required operation refuses before any bind', async () => { + const fixture = selectorCaptureFixture({ + capture: { available: false, reason: 'unsupported-platform-leaf' }, + }); + + const bound = await resolveBoundSelectorCapture({ + command: 'is', + device: ANDROID_EMULATOR, + session: makeAndroidSession('selector'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(bound).toMatchObject({ ok: false, response: { ok: false } }); + expect(fixture.binds).toEqual([]); +}); + +// The active-app split is the only plan axis selector commands have: no `--actions` surface, +// so `captureSnapshotWithCustomActions` is never required and never admitted for them. +test('a session without a tracked app selects the without-active-app plan', async () => { + const fixture = selectorCaptureFixture({ + withoutActiveApp: { available: false, reason: 'owner-capability-missing' }, + }); + + const withApp = await resolveBoundSelectorCapture({ + command: 'get', + device: IOS_SIMULATOR, + session: makeIosSession('with-app', { appBundleId: 'com.example.app' }), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + // The iOS refusal enriches its hint from device inventory, which request execution owns. + const withoutApp = await withTestDeviceInventory( + {}, + async () => + await resolveBoundSelectorCapture({ + command: 'get', + device: IOS_SIMULATOR, + session: makeIosSession('no-app'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ); + + expect(withApp.ok).toBe(true); + expect(withoutApp.ok).toBe(false); +}); + +// `createBoundSelectorRuntime` is the construction path `get`, `is`, and `wait` switch onto: +// admit, bind once, then build the selector runtime with the bound operations attached. It is +// exercised here rather than from a command route because no selector descriptor has cut over +// yet — `find`'s own cutover is deferred behind the Wave 5 `focus`/`type` surfaces. +test('the bound construction path admits and binds before it builds a runtime', async () => { + const fixture = selectorCaptureFixture(); + const sessionStore = makeSessionStore(); + sessionStore.set('bound', makeAndroidSession('bound')); + + const resolved = await createBoundSelectorRuntime( + { + req: { token: 't', session: 'bound', command: 'get', positionals: [], flags: {} }, + sessionName: 'bound', + logPath: '/tmp/bound.log', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }, + { requireSession: true, command: 'get' }, + ); + + expect(resolved.ok).toBe(true); + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); +}); + +test('the bound construction path refuses an unavailable operation without building a runtime', async () => { + const fixture = selectorCaptureFixture({ + capture: { available: false, reason: 'unsupported-platform-leaf' }, + }); + const sessionStore = makeSessionStore(); + sessionStore.set('bound', makeAndroidSession('bound')); + + const resolved = await createBoundSelectorRuntime( + { + req: { token: 't', session: 'bound', command: 'is', positionals: [], flags: {} }, + sessionName: 'bound', + logPath: '/tmp/bound.log', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }, + { requireSession: true, command: 'is' }, + ); + + expect(resolved).toMatchObject({ ok: false, response: { ok: false } }); + expect(fixture.binds).toEqual([]); +}); diff --git a/src/daemon/__tests__/selector-capture-fixture.ts b/src/daemon/__tests__/selector-capture-fixture.ts new file mode 100644 index 0000000000..4a70277778 --- /dev/null +++ b/src/daemon/__tests__/selector-capture-fixture.ts @@ -0,0 +1,98 @@ +import { + localRuntimeOwner, + narrowDeviceBinding, + providerRuntimeOwner, + snapshotRuntimeOperationFacts, + type CaptureSnapshotInput, + type PlatformRuntimeOperations, + type RuntimeFacts, + type RuntimeOperationFact, + type SnapshotResult, +} from '@agent-device/contracts/platform'; +import { deviceShape, type DeviceInfo } from '@agent-device/kernel/device'; +import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import { unavailableDeviceRuntimeGateway } from './test-device-runtime-gateway.ts'; + +const available: RuntimeOperationFact = Object.freeze({ available: true }); + +/** + * The request-bound capture seam every selector command (`find`, `get`, `is`, `wait`) + * consumes, faked at `inspectFacts` / `bindDevice` rather than at the legacy leaf + * dispatch. Records each bind and each capture so a test can assert the ADR 0019 §9 + * shape directly: one inspection, one bind, and every capture through the bound + * operation. + */ +export function selectorCaptureFixture( + params: Readonly<{ + capture?: RuntimeOperationFact; + withoutActiveApp?: RuntimeOperationFact; + snapshot?: (input: CaptureSnapshotInput, index: number) => SnapshotResult; + }> = {}, +): Readonly<{ + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; + inspections: DeviceInfo[]; + binds: DeviceInfo[]; + captures: CaptureSnapshotInput[]; +}> { + const inspections: DeviceInfo[] = []; + const binds: DeviceInfo[] = []; + const captures: CaptureSnapshotInput[] = []; + + const facts = async (device: DeviceInfo): Promise> => { + const base = await unavailableDeviceRuntimeGateway.inspectFacts(device); + return { + device: { + ...deviceShape(device), + providerMode: isActiveProviderDevice(device) ? 'provider-runtime' : 'local', + }, + operations: { + ...base.operations, + ...snapshotRuntimeOperationFacts({ + capture: params.capture ?? available, + customActions: { available: false, reason: 'unsupported-platform-leaf' }, + withoutActiveApp: params.withoutActiveApp ?? params.capture ?? available, + }), + }, + }; + }; + + const captureSnapshot = async (input: CaptureSnapshotInput): Promise => { + const index = captures.length; + captures.push(input); + input.signal?.throwIfAborted(); + return params.snapshot?.(input, index) ?? { nodes: [], backend: 'xctest' }; + }; + + return { + inspectFacts: async (device) => { + inspections.push(device); + return await facts(device); + }, + bindDevice: async (device, use) => { + binds.push(device); + const deviceFacts = await facts(device); + return narrowDeviceBinding( + { + device, + owner: + deviceFacts.device.providerMode === 'provider-runtime' + ? providerRuntimeOwner('test', 'selector-capture-fixture') + : localRuntimeOwner(device.platform), + facts: deviceFacts, + operations: { + captureSnapshot, + captureSnapshotWithCustomActions: captureSnapshot, + captureSnapshotWithoutActiveApp: captureSnapshot, + }, + [Symbol.asyncDispose]: async () => {}, + }, + use, + ); + }, + inspections, + binds, + captures, + }; +} diff --git a/src/daemon/get-runtime.ts b/src/daemon/get-runtime.ts deleted file mode 100644 index 4a1edf4d16..0000000000 --- a/src/daemon/get-runtime.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { - elementReadRuntimePlan, - type CaptureSnapshotInput, - type ElementReadRuntimePlan, - type ElementTextRuntimeOperations, - type ReadTextAtPointInput, - type SnapshotResult, -} from '@agent-device/contracts/platform'; -import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; -import type { DaemonResponse, SessionState } from './types.ts'; -import { noActiveSessionError } from './handlers/response.ts'; -import { - admitRuntimePlan, - requireRuntimeBinding, - unavailableRuntimeOperationResponse, - unwrapAdmittedRuntimePlan, - type AdmittedRuntimePlan, -} from './handlers/session-runtime-admission.ts'; - -/** - * The operations `get` executes with. `captureSnapshot` is required, so it is non-optional here; - * `readTextAtPoint` is the declared preferred operation and is present only on owners whose facts - * advertise it. Its absence changes which text `get` can read, never whether `get` can run. - */ -export type BoundGetRuntimeOperations = Readonly<{ - captureSnapshot(input: CaptureSnapshotInput): Promise; - readTextAtPoint?: ElementTextRuntimeOperations['readTextAtPoint']; -}>; - -export type ResolvedGetRuntime = - | Readonly<{ - ok: true; - session: SessionState; - device: SessionState['device']; - operations: BoundGetRuntimeOperations; - }> - | Readonly<{ ok: false; response: DaemonResponse }>; - -/** - * `get`'s one facts-first admission and one binding. The session owns the device, so there is no - * separate target resolution: admission inspects that device's owner facts once, refuses before - * binding when the required capture is unavailable, then binds exactly once on the admitted - * device through the admitted plan's token. - */ -export async function resolveBoundGetRuntime( - params: Readonly<{ - session: SessionState | undefined; - inspectFacts?: InspectDeviceRuntimeFacts; - bindDevice?: BindDeviceRuntime; - }>, -): Promise { - const { session } = params; - if (!session) return { ok: false, response: noActiveSessionError() }; - const device = session.device; - const admission = await admitRuntimePlan({ - device, - plan: elementReadRuntimePlan, - inspectFacts: params.inspectFacts, - }); - if (!admission.admitted) { - return { ok: false, response: unavailableRuntimeOperationResponse('get', admission.fact)! }; - } - return { - ok: true, - session, - device, - operations: await bindGetRuntime(params.bindDevice, admission), - }; -} - -/** - * Binds only an admitted plan, on the device it was admitted for: the token is minted by - * `admitRuntimePlan` alone and unwrapped by exact identity, so a bare plan, a separate device, or - * a look-alike cannot reach these operations. - */ -async function bindGetRuntime( - bindDevice: BindDeviceRuntime | undefined, - admission: AdmittedRuntimePlan, -): Promise { - const bind = requireRuntimeBinding(bindDevice); - const { device, plan } = unwrapAdmittedRuntimePlan(admission); - return selectElementReadOperations(await bind(device, plan.use)); -} - -/** - * Mirrors the snapshot binder's `BoundSnapshotOperation`: the operation this projection names is - * non-optional, so a value of this type IS the proof that the owner advertised it. - */ -type BoundElementReadOperation = Readonly<{ - operations: Readonly>; -}>; - -type BoundElementReadCatalog = Readonly<{ - captureSnapshot(input: CaptureSnapshotInput): Promise; - readTextAtPoint: ElementTextRuntimeOperations['readTextAtPoint']; -}>; - -type NarrowedElementReadRuntime = Readonly<{ - operations: Readonly<{ - captureSnapshot(input: CaptureSnapshotInput): Promise; - readTextAtPoint?: ElementTextRuntimeOperations['readTextAtPoint']; - }>; -}>; - -/** - * The lexical owner of `get`'s required capture call. The preferred read is handed to its own - * owner below only when the owner facts advertised it: the narrowed projection is *constructed* - * from a non-undefined local, so presence is carried by the type system rather than repaired - * with a non-null assertion or a defensive throw. - */ -function selectElementReadOperations( - runtime: NarrowedElementReadRuntime, -): BoundGetRuntimeOperations { - const readTextAtPoint = runtime.operations.readTextAtPoint; - return Object.freeze({ - captureSnapshot: async (input: CaptureSnapshotInput) => - await runtime.operations.captureSnapshot(input), - ...(readTextAtPoint - ? { readTextAtPoint: selectElementTextRead({ operations: { readTextAtPoint } }) } - : {}), - }); -} - -/** The lexical owner of `get`'s preferred element-text read. */ -function selectElementTextRead( - runtime: BoundElementReadOperation<'readTextAtPoint'>, -): ElementTextRuntimeOperations['readTextAtPoint'] { - return async (input: ReadTextAtPointInput) => await runtime.operations.readTextAtPoint(input); -} diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts index 751ef031cf..fd6c718810 100644 --- a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -75,6 +75,10 @@ function elementReadFacts(device: DeviceInfo): RuntimeFacts Promise; + captureData?: () => Promise; }; type SnapshotData = { @@ -84,6 +81,7 @@ export async function captureSnapshot( } export async function captureSnapshotData(params: CaptureSnapshotParams): Promise { + if (params.captureData) return await params.captureData(); const { device, session, logPath } = params; const context = contextFromFlags( logPath, @@ -91,33 +89,33 @@ export async function captureSnapshotData(params: CaptureSnapshotParams): Promis session?.appBundleId, session?.trace?.outPath, ); - const options = { - appBundleId: context.appBundleId, - interactiveOnly: context.snapshotInteractiveOnly, - preferredBackend: context.snapshotPreferredBackend, - depth: context.snapshotDepth, - scope: context.snapshotScope, - raw: context.snapshotRaw, - customActions: context.snapshotCustomActions, - includeRects: params.includeRects, - includeHiddenContentHints: context.snapshotIncludeHiddenContentHints, - surface: session?.surface, - }; - const execution = { - requestId: context.requestId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }; - if (params.captureData) return await params.captureData({ options, execution }); return await captureSnapshotWithInteractor({ device, - runnerContext: { ...execution, signal: params.signal, appBundleId: context.appBundleId }, - options: { ...options, signal: params.signal }, + runnerContext: { + requestId: context.requestId, + signal: params.signal, + appBundleId: context.appBundleId, + verbose: context.verbose, + logPath: context.logPath, + traceLogPath: context.traceLogPath, + iosXctestrunFile: context.iosXctestrunFile, + iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, + iosXctestEnvDir: context.iosXctestEnvDir, + runnerLeaseContext: context.runnerLeaseContext, + }, + options: { + appBundleId: context.appBundleId, + signal: params.signal, + interactiveOnly: context.snapshotInteractiveOnly, + preferredBackend: context.snapshotPreferredBackend, + depth: context.snapshotDepth, + scope: context.snapshotScope, + raw: context.snapshotRaw, + customActions: context.snapshotCustomActions, + includeRects: params.includeRects, + includeHiddenContentHints: context.snapshotIncludeHiddenContentHints, + surface: session?.surface, + }, }); } diff --git a/src/daemon/selector-capture-binding.ts b/src/daemon/selector-capture-binding.ts new file mode 100644 index 0000000000..4c92123966 --- /dev/null +++ b/src/daemon/selector-capture-binding.ts @@ -0,0 +1,73 @@ +import { + resolveSelectorCaptureRuntimePlan, + type CaptureSnapshotInput, + type ElementTextRuntimeOperations, + type SnapshotResult, +} from '@agent-device/contracts/platform'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; +import { admitAndBindSnapshotCapture } from './snapshot-runtime-binding.ts'; +import type { DaemonResponse, SessionState } from './types.ts'; + +/** The selector commands that resolve their targets from a request-bound capture. */ +export type SelectorCaptureCommand = 'find' | 'get' | 'is' | 'wait'; + +/** + * One request's bound accessibility capture. Selector commands capture repeatedly under one + * binding (polling, sparse recovery), so the operation is parametrized by intent rather than + * frozen at bind time and `input.signal` carries a poll's remaining budget. + */ +export type BoundSelectorCapture = (input: CaptureSnapshotInput) => Promise; + +/** + * The bound operations a selector command's runtime executes through. A record rather than a + * bare capture function on purpose: the next selector unit adds its own bound operation here + * (`get`'s preferred element read, whose platform branches `get` and `find get text` + * currently duplicate) without changing any signature on this seam. + */ +/** + * The owner's live element-text read, when its facts advertise one. Optional because it is a + * PREFERRED operation: every selector read's required path answers from the captured tree, so an + * owner without it still executes the command completely (ADR 0019 §2). + */ +export type BoundSelectorRead = ElementTextRuntimeOperations['readTextAtPoint']; + +export type BoundSelectorOperations = Readonly<{ + capture: BoundSelectorCapture; + readText?: BoundSelectorRead; +}>; + +export type ResolvedSelectorCapture = + | Readonly<{ ok: true; operations: BoundSelectorOperations }> + | Readonly<{ ok: false; response: DaemonResponse }>; + +/** + * The selector family's entry to the shared admit-then-bind path: it contributes the + * active-app plan and its command name for the refusal wording, and inherits one inspection, + * refusal-before-bind, and one binding. A sibling unit migrates by naming its command here. + */ +export async function resolveBoundSelectorCapture( + params: Readonly<{ + command: SelectorCaptureCommand; + device: SessionState['device']; + session: SessionState | undefined; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; + }>, +): Promise { + const bound = await admitAndBindSnapshotCapture({ + ...params, + plan: resolveSelectorCaptureRuntimePlan({ + hasActiveApp: params.session?.appBundleId !== undefined, + }), + }); + if (!bound.ok) return bound; + // The read is present only when the admitted owner advertised it; its absence is not a failure + // and not a fallback — every selector read's required path answers from the captured tree. + return { + ok: true, + operations: { + capture: bound.capture, + ...(bound.readTextAtPoint ? { readText: bound.readTextAtPoint } : {}), + }, + }; +} diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 747a936a04..32ef876c82 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -1,5 +1,4 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import type { CaptureSnapshotInput, SnapshotResult } from '@agent-device/contracts/platform'; import type { BackendSnapshotResult } from '../backend.ts'; import { buildSnapshotPresentationKey, @@ -13,10 +12,12 @@ import { captureSnapshot } from './handlers/snapshot-capture.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import { getActiveAndroidSnapshotFreshness } from './android-snapshot-freshness.ts'; import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts'; +import type { BoundSelectorCapture } from './selector-capture-binding.ts'; +import { buildRuntimeCaptureInput } from './snapshot-runtime-capture-input.ts'; const SELECTOR_CAPTURE_CACHE_TTL_MS = 750; -type SelectorCaptureRuntimeParams = { +export type SelectorCaptureRuntimeParams = { device: SessionState['device']; session: SessionState | undefined; sessionStore: SessionStore; @@ -27,11 +28,11 @@ type SelectorCaptureRuntimeParams = { // capture runtime reports every consumed snapshot here for response-level disclosures. consumedSnapshot?: { state?: SnapshotState }; /** - * Request-bound platform capture, supplied by a selector command that already bound its device - * runtime. Unmigrated selector commands omit it and keep the legacy interactor capture until - * their own descriptor cuts over. + * The request-bound capture from `resolveBoundSelectorCapture`: every cache tier, recovery + * re-capture, and poll below reaches the platform through it. Selector commands still on + * legacy admission pass nothing; the last one to migrate makes this required. */ - captureData?: (input: CaptureSnapshotInput) => Promise; + capture?: BoundSelectorCapture; }; /** @@ -177,19 +178,38 @@ async function runCapture( snapshotScope: string | undefined, interactiveOnly = request.flags?.snapshotInteractiveOnly, ): Promise { + const flags = { + ...request.flags, + snapshotInteractiveOnly: interactiveOnly, + }; + const boundCapture = params.capture; const capture = await captureSnapshot({ device: params.device, session: params.session, - flags: { - ...request.flags, - snapshotInteractiveOnly: interactiveOnly, - }, + flags, outPath: request.outPath ?? params.req.flags?.out, logPath: params.logPath ?? '', snapshotScope, includeRects: request.includeRects, signal: request.signal, - ...(params.captureData ? { captureData: params.captureData } : {}), + ...(boundCapture === undefined + ? {} + : { + captureData: async () => + await boundCapture( + buildRuntimeCaptureInput({ + flags, + logPath: params.logPath ?? '', + meta: params.req.meta, + session: params.session, + snapshotScope, + includeRects: request.includeRects, + // The poll's remaining budget, not the request's: `wait`/`find wait` + // abort a stalled capture at its deadline instead of racing it. + signal: request.signal, + }), + ), + }), }); return capture.snapshot; } diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index e7d4e0d275..1593913bdf 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -18,8 +18,12 @@ import type { ContextFromFlags } from './handlers/interaction-common.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; import { createSelectorCaptureRuntime } from './selector-capture-runtime.ts'; +import { + resolveBoundSelectorCapture, + type BoundSelectorOperations, + type SelectorCaptureCommand, +} from './selector-capture-binding.ts'; import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; -import type { BoundGetRuntimeOperations } from './get-runtime.ts'; import { isActiveProviderDevice } from '../provider-device-runtime.ts'; import { getRequestSignal } from '../request/cancel.ts'; import { snapshotOptionsToFlags } from '../backend-snapshot-options.ts'; @@ -30,28 +34,30 @@ export type SelectorRuntimeParams = { logPath?: string; sessionStore: SessionStore; contextFromFlags?: ContextFromFlags; - /** - * Request-bound device runtime seams. Migrated selector commands resolve their own plan from - * these; unmigrated ones ignore them and keep their legacy admission until their unit lands. - */ - inspectFacts?: InspectDeviceRuntimeFacts; - bindDevice?: BindDeviceRuntime; // Filled by the capture runtime with the snapshot each selector command actually consumed; // sessionless routes disclose from here because no session record stores the capture. consumedSnapshot?: { state?: SnapshotState }; signal?: AbortSignal; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; }; -type SelectorRuntimeDeviceParams = SelectorRuntimeParams & { +export type SelectorRuntimeDeviceParams = SelectorRuntimeParams & { session: SessionState | undefined; device: SessionState['device']; - /** - * The operations the calling command already bound. A migrated command passes them so its - * capture and element read execute through its own binding instead of the legacy adapter. - */ - operations?: BoundGetRuntimeOperations; + /** The request-bound operations this runtime executes through. Absent for selector + * commands still on legacy admission, until their own ADR 0019 unit lands. */ + bound?: BoundSelectorOperations; }; +type ResolvedSelectorRuntime = + | { ok: true; runtime: ReturnType } + | { ok: false; response: DaemonResponse }; + +type ResolvedSelectorDevice = + | { ok: true; session: SessionState | undefined; device: SessionState['device'] } + | { ok: false; response: DaemonResponse }; + type AppleRunnerFindTextTarget = { device: SessionState['device']; appBundleId: string; @@ -76,45 +82,87 @@ export function createSelectorRuntimeForDevice(params: SelectorRuntimeDevicePara }); } -export async function createSelectorRuntime( +/** The session/device a selector command runs against, before any admission decides. */ +async function resolveSelectorRuntimeDevice( params: SelectorRuntimeParams, - options: { requireSession: boolean; capability: 'find' | 'is' }, -): Promise< - | { ok: true; runtime: ReturnType } - | { ok: false; response: DaemonResponse } -> { + requireSession: boolean, +): Promise { params.consumedSnapshot ??= {}; const session = params.sessionStore.get(params.sessionName); - if (!session && options.requireSession) { - return { - ok: false, - response: noActiveSessionError(), - }; - } + if (!session && requireSession) return { ok: false, response: noActiveSessionError() }; const device = session?.device ?? (await resolveTargetDevice(params.req.flags ?? {})); if (!session) await ensureDeviceReady(device); - const unsupported = requireCommandSupported(options.capability, device); + return { ok: true, session, device }; +} + +/** + * A migrated selector command's runtime: facts-first admission, exactly one binding, and a + * backend whose every capture goes through the bound operation. A sibling unit migrates by + * naming its command here instead of passing a `capability` to {@link createSelectorRuntime}; + * nothing else in this module or `selector-capture-runtime.ts` needs to change. + */ +export async function createBoundSelectorRuntime( + params: SelectorRuntimeParams, + options: { requireSession: boolean; command: SelectorCaptureCommand }, +): Promise { + const resolved = await resolveSelectorRuntimeDevice(params, options.requireSession); + if (!resolved.ok) return resolved; + const bound = await resolveBoundSelectorCapture({ + command: options.command, + device: resolved.device, + session: resolved.session, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!bound.ok) return { ok: false, response: bound.response }; + return { + ok: true, + runtime: createSelectorRuntimeForDevice({ + ...params, + session: resolved.session, + device: resolved.device, + bound: bound.operations, + }), + }; +} + +/** + * The legacy capability-admitted selector runtime, for the selector commands whose ADR 0019 + * unit has not landed. The union narrows as each one migrates, and the last selector unit + * deletes this function together with its `requireCommandSupported` call. + */ +export async function createSelectorRuntime( + params: SelectorRuntimeParams, + options: { requireSession: boolean; capability: 'find' | 'is' }, +): Promise { + const resolved = await resolveSelectorRuntimeDevice(params, options.requireSession); + if (!resolved.ok) return resolved; + const unsupported = requireCommandSupported(options.capability, resolved.device); if (unsupported) return { ok: false, response: unsupported }; return { ok: true, runtime: createSelectorRuntimeForDevice({ ...params, - session, - device, + session: resolved.session, + device: resolved.device, }), }; } function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDeviceBackend { + // Which read the backend uses is fixed by WHICH COMMAND CONSTRUCTED THIS RUNTIME, never by + // failure, family, environment, or flag. A migrated command arrives with `bound` and its + // binding is authoritative — including when the owner advertised no read, which is the complete + // required path. An unmigrated selector command arrives without `bound` and keeps the legacy + // dispatch until its own descriptor cuts over (ADR 0019 §6 permits the unmigrated sibling's own + // path); a migrated command can never reach it. const { req, session, device, logPath, sessionName, sessionStore } = params; const resolveContextFromFlags: ContextFromFlags = params.contextFromFlags ?? ((flags, appBundleId, traceLogPath) => contextFromFlags(logPath ?? '', flags, appBundleId, traceLogPath)); - // A migrated command's binding is authoritative, including when it reports no live read; - // an unmigrated command keeps the legacy dispatch until its own descriptor cuts over. - const readTextAtPoint = params.operations - ? params.operations.readTextAtPoint + const readTextAtPoint = params.bound + ? params.bound.readText : legacyDispatchReadTextAtPoint({ device, flags: req.flags, @@ -129,7 +177,7 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice req, consumedSnapshot: params.consumedSnapshot, logPath, - captureData: params.operations?.captureSnapshot, + capture: params.bound?.capture, }); return { platform: publicPlatformString(device), @@ -158,13 +206,13 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice }, readText: async (_context, node: SnapshotNode) => ({ text: await readTextForNode({ + readTextAtPoint, device, node, flags: req.flags, appBundleId: session?.appBundleId, traceOutPath: session?.trace?.outPath, surface: session?.surface, - readTextAtPoint, contextFromFlags: resolveContextFromFlags, }), }), diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 6658350570..0c282a0728 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -48,11 +48,11 @@ import { } from './direct-ios-selector.ts'; import { isSessionRecording } from './session-script-publication-capability.ts'; import { + createBoundSelectorRuntime, createSelectorRuntime, createSelectorRuntimeForDevice, type SelectorRuntimeParams, } from './selector-runtime-backend.ts'; -import { resolveBoundGetRuntime } from './get-runtime.ts'; export type DirectIosSelectorQueryResult = { found: boolean; @@ -173,25 +173,18 @@ export async function dispatchGetViaRuntime( // *within* an admitted request, never a way around exact-owner facts or the one-binding // invariant. (The query itself is still the shared root mechanic co-owned by `is`, `wait`, // and the Wave 5 offscreen probe — this unit orders it, it does not claim it.) - const boundRuntime = await resolveBoundGetRuntime({ - session: params.sessionStore.get(params.sessionName), - inspectFacts: params.inspectFacts, - bindDevice: params.bindDevice, + const resolvedRuntime = await createBoundSelectorRuntime(params, { + requireSession: true, + command: 'get', }); - if (!boundRuntime.ok) return boundRuntime.response; - params.consumedSnapshot ??= {}; + if (!resolvedRuntime.ok) return resolvedRuntime.response; if (target.target.kind === 'selector' && !replayTargetGuard) { const directResponse = await dispatchDirectIosSelectorGet(params, sub, target.target.selector); if (directResponse) return directResponse; } - const runtime = createSelectorRuntimeForDevice({ - ...params, - session: boundRuntime.session, - device: boundRuntime.device, - operations: boundRuntime.operations, - }); + const runtime = resolvedRuntime.runtime; // #1076 + ADR 0014: a get @ref binds against the retained ref-frame evidence, // so it never silently retargets to a newer positional tree. Its warning is diff --git a/src/daemon/snapshot-runtime-binding.ts b/src/daemon/snapshot-runtime-binding.ts index 5c135e9d2f..c8e1e84108 100644 --- a/src/daemon/snapshot-runtime-binding.ts +++ b/src/daemon/snapshot-runtime-binding.ts @@ -1,7 +1,10 @@ import { resolveSnapshotRuntimePlan, type CaptureSnapshotInput, + type ElementTextRuntimeOperations, + type ReadTextAtPointInput, type RuntimeOperationFact, + type SelectorCaptureRuntimePlan, type SnapshotResult, type SnapshotRuntimeOperations, type SnapshotRuntimePlan, @@ -43,10 +46,22 @@ type ResolvedSnapshotCaptureRuntime = /** A capture operation parametrized by intent, so a polling caller can capture repeatedly * under the one binding it was admitted for. */ -type BoundSnapshotCapture = (input: CaptureSnapshotInput) => Promise; +export type BoundSnapshotCapture = (input: CaptureSnapshotInput) => Promise; -type AdmittedSnapshotCapture = - | Readonly<{ ok: true; capture: BoundSnapshotCapture }> +/** The owner's live element read, when its facts advertise one. */ +export type BoundElementRead = ElementTextRuntimeOperations['readTextAtPoint']; + +export type AdmittedSnapshotCapture = + | Readonly<{ + ok: true; + capture: BoundSnapshotCapture; + /** + * The owner's live element read, present only when the caller's plan declared it PREFERRED + * and the admitted owner advertised it. `snapshot`/`diff` plans declare no read, so this is + * simply absent for them — the member is additive and they are unchanged. + */ + readTextAtPoint?: BoundElementRead; + }> | Readonly<{ ok: false; response: DaemonResponse }>; /** @@ -55,12 +70,12 @@ type AdmittedSnapshotCapture = * the admitted device. `snapshot`/`diff` supply the four-way custom-actions plan and the * selector family the active-app plan; a new consumer supplies a plan and a command name. */ -async function admitAndBindSnapshotCapture( +export async function admitAndBindSnapshotCapture( params: Readonly<{ command: string; device: SessionState['device']; session: SessionState | undefined; - plan: SnapshotRuntimePlan; + plan: SnapshotRuntimePlan | SelectorCaptureRuntimePlan; inspectFacts?: InspectDeviceRuntimeFacts; bindDevice?: BindDeviceRuntime; }>, @@ -79,10 +94,11 @@ async function admitAndBindSnapshotCapture( }), }; } - const runtime = await bindSnapshotCaptureRuntime(params.bindDevice, admission); + const bound = await bindSnapshotCaptureRuntime(params.bindDevice, admission); return Object.freeze({ ok: true, - capture: async (input: CaptureSnapshotInput) => await runtime.captureSnapshot(input), + capture: async (input: CaptureSnapshotInput) => await bound.captureSnapshot(input), + ...(bound.readTextAtPoint ? { readTextAtPoint: bound.readTextAtPoint } : {}), }); } @@ -132,15 +148,28 @@ export async function resolveBoundSnapshotCaptureRuntime( */ async function bindSnapshotCaptureRuntime( bindDevice: BindDeviceRuntime | undefined, - admission: AdmittedRuntimePlan, -): Promise }>> { + admission: AdmittedRuntimePlan, +): Promise< + Readonly<{ + captureSnapshot(input: CaptureSnapshotInput): Promise; + readTextAtPoint?: BoundElementRead; + }> +> { const bind = requireRuntimeBinding(bindDevice); const { device, plan } = unwrapAdmittedRuntimePlan(admission); + // One switch, one set of operation selectors. The selector arms reuse the SAME + // `selectActiveAppSnapshot` / `selectSnapshotWithoutActiveApp` the snapshot arms use and only + // add the preferred element read; the discriminants differ solely so the compiler can narrow + // `plan.use` per family. No parallel plan-to-operation dispatch is introduced. switch (plan.kind) { case 'active-app': { const runtime = await bind(device, plan.use); return selectActiveAppSnapshot(runtime); } + case 'selector-active-app': { + const runtime = await bind(device, plan.use); + return { ...selectActiveAppSnapshot(runtime), ...selectElementRead(runtime) }; + } case 'custom-actions-active-app': { const runtime = await bind(device, plan.use); return selectCustomActionsSnapshot(runtime); @@ -149,6 +178,10 @@ async function bindSnapshotCaptureRuntime( const runtime = await bind(device, plan.use); return selectSnapshotWithoutActiveApp(runtime); } + case 'selector-without-active-app': { + const runtime = await bind(device, plan.use); + return { ...selectSnapshotWithoutActiveApp(runtime), ...selectElementRead(runtime) }; + } case 'custom-actions-without-active-app': { const runtime = await bind(device, plan.use); return selectCustomActionsSnapshot(runtime); @@ -156,6 +189,28 @@ async function bindSnapshotCaptureRuntime( } } +/** + * Projects the preferred element read when the admitted owner advertised it. A projection whose + * use never declared it simply has no such member, so this yields `{}` for `snapshot`/`diff`. + */ +function selectElementRead( + runtime: Readonly<{ operations: Readonly<{ readTextAtPoint?: BoundElementRead }> }>, +): Readonly<{ readTextAtPoint?: BoundElementRead }> { + const readTextAtPoint = runtime.operations.readTextAtPoint; + // Narrowed by CONSTRUCTION rather than by assertion: the projection below is only buildable + // from a non-undefined local, so presence is carried by the value that captured it. + return readTextAtPoint + ? { readTextAtPoint: bindElementRead({ operations: { readTextAtPoint } }) } + : {}; +} + +/** The one lexical owner of the narrowed `readTextAtPoint` call. */ +function bindElementRead( + runtime: Readonly<{ operations: Readonly<{ readTextAtPoint: BoundElementRead }> }>, +): BoundElementRead { + return async (input: ReadTextAtPointInput) => await runtime.operations.readTextAtPoint(input); +} + type BoundSnapshotOperation = Readonly<{ operations: Readonly>; }>; From 50f66c31151a7a77b210fa59ea1f10116be30abc Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 19 Aug 2026 17:23:32 +0200 Subject: [PATCH 4/9] refactor: retire the read dispatch alias across both selector consumers Read-only `find` now constructs a BOUND selector backend, so `get text` and `find get text` execute the same bound `readTextAtPoint` instead of one binding it and the other dispatching the legacy `read`. This moves find's READ LEG only: find's descriptor stays LEGACY_PLATFORM_EXECUTION and it claims no cutover row. With no consumer left, the whole chain goes: the `read` registry entry and its `dispatch: {}` projection, `DISPATCH_HANDLERS.read`, `handleReadCommand`, `interaction-read-legacy-dispatch.ts`, and the duplicate platform reader branches it carried. `read` was the only `dispatch-alias` descriptor, so that catalog group goes too. Deleting the registry entry drops 'read' from DescriptorDispatchCommandName, which makes a surviving DISPATCH_HANDLERS.read a compile error rather than something R36 has to police. R36 now claims the retirement it can prove. `find.test.ts` is over the size tripwire, so its handler invocation is extracted to find-handler-fixture.ts and the pin lowered 1237 -> 1221. --- .../layering/runtime-command-cutover-table.ts | 17 +++--- src/__tests__/test-file-size-ratchet.test.ts | 2 +- .../__tests__/parity.test.ts | 6 ++- src/core/command-descriptor/registry.ts | 11 ---- src/core/command-descriptor/types.ts | 2 +- src/core/dispatch-interactions.ts | 53 ------------------- src/core/dispatch.ts | 2 - .../__tests__/find-handler-fixture.ts | 29 ++++++++++ src/daemon/handlers/__tests__/find.test.ts | 38 ++++--------- .../interaction-get-runtime-fixture.ts | 23 +++++--- .../system-surface-disclosure.test.ts | 6 +++ src/daemon/handlers/find.ts | 5 ++ .../interaction-read-legacy-dispatch.ts | 45 ---------------- src/daemon/request-handler-chain.ts | 2 + src/daemon/selector-runtime-backend.ts | 21 ++------ src/daemon/selector-runtime.ts | 8 ++- 16 files changed, 98 insertions(+), 172 deletions(-) create mode 100644 src/daemon/handlers/__tests__/find-handler-fixture.ts delete mode 100644 src/daemon/handlers/interaction-read-legacy-dispatch.ts diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index b00827ed62..a7126fea38 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -512,13 +512,18 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ subject: 'element read', tier: 'request-scoped', execution: 'device-runtime', - // `get`'s legacy admission WAS its capability bucket plus the static family command sets the - // matrix augments it with; the row's automatic admission columns reject the bucket and the - // `requireCommandSupported('get', …)` call, and these names are the augmentation entries that - // had to disappear with them (`addWebCommandCapabilities` throws for a web-listed command - // with no matrix row, so the web entry could not be left behind). + // Two retirements. `get`'s own legacy admission was its capability bucket plus the static + // family command sets the matrix augments it with — the row's automatic admission columns + // reject the bucket and the `requireCommandSupported('get', …)` call. And the shared element + // read: both consumers of the selector backend's read (`get text` and read-only + // `find … get text`) now execute the bound `readTextAtPoint`, so the legacy `read` dispatch + // alias retires whole. Deleting its registry entry drops `'read'` from + // `DescriptorDispatchCommandName`, which makes a surviving `DISPATCH_HANDLERS.read` a COMPILE + // error rather than something this row has to police. legacyRetirement: { - routeNames: ['WEB_QUERY_COMMANDS_WITH_GET', 'HARMONYOS_GET_SUPPORT'], + modulePaths: ['src/daemon/handlers/interaction-read-legacy-dispatch.ts'], + importPatterns: [/(?:^|\/)handlers\/interaction-read-legacy-dispatch(?:\.[cm]?[jt]s)?$/], + routeNames: ['handleReadCommand'], }, runtimeTypeNames: ['ElementTextRuntimeOperations', 'SnapshotRuntimeOperations'], operations: { diff --git a/src/__tests__/test-file-size-ratchet.test.ts b/src/__tests__/test-file-size-ratchet.test.ts index 1f3a440e61..d3120b72fe 100644 --- a/src/__tests__/test-file-size-ratchet.test.ts +++ b/src/__tests__/test-file-size-ratchet.test.ts @@ -48,7 +48,7 @@ const PINNED_TEST_FILE_LINES: Readonly> = Object.freeze({ 'src/platforms/apple/core/__tests__/runner-command-retry.test.ts': 1327, 'src/__tests__/cli-client-commands.test.ts': 1317, 'src/__tests__/cli-config.test.ts': 1282, - 'src/daemon/handlers/__tests__/find.test.ts': 1223, + 'src/daemon/handlers/__tests__/find.test.ts': 1207, 'src/platforms/apple/core/__tests__/perf.test.ts': 1222, 'src/mcp/__tests__/command-tools.test.ts': 1218, 'src/daemon/handlers/__tests__/session-replay-divergence.test.ts': 1215, diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index f401543280..69e6d6d25e 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -193,7 +193,11 @@ test('platform dispatch command list is built from descriptor dispatch facets', .sort(); assert.deepEqual(listRegisteredDispatchCommandNames(), dispatchCommands); - assert.ok(dispatchCommands.includes('read'), 'read stays dispatch-only'); + assert.equal( + dispatchCommands.includes('read' as never), + false, + 'the read dispatch alias retired with the selector element-read cutover (#1739)', + ); assert.equal( dispatchCommands.includes(PUBLIC_COMMANDS.gesture), false, diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 7c29ca921b..74dbdc7d60 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -1193,17 +1193,6 @@ export const RAW_COMMAND_DESCRIPTORS = [ batchable: true, platformExecution: { kind: 'device-runtime', uses: selectorCaptureRuntimePlanUses }, }, - { - name: 'read', - deviceClaimPolicy: 'require-owner', - ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/handlers/interaction.ts'] as const } : {}), - catalog: { group: 'dispatch-alias' }, - recordsSessionAction: false, - dispatch: {}, - timeoutPolicy: DEFAULT_TIMEOUT_POLICY, - batchable: false, - platformExecution: LEGACY_PLATFORM_EXECUTION, - }, { name: 'is', deviceClaimPolicy: 'require-owner', diff --git a/src/core/command-descriptor/types.ts b/src/core/command-descriptor/types.ts index 9c8b53e527..1be0713408 100644 --- a/src/core/command-descriptor/types.ts +++ b/src/core/command-descriptor/types.ts @@ -103,7 +103,7 @@ export type DeviceClaimPolicy = | 'acquire-session' | 'release-session'; -export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli' | 'dispatch-alias'; +export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli'; /** * Which default tool set a framework adapter (`agent-device/ai-sdk`, the diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 24846aa8ae..3a17d60294 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -666,59 +666,6 @@ function parseScrollTarget(input: string): { return { direction: parseScrollDirection(input) }; } -export async function handleReadCommand( - device: DeviceInfo, - positionals: string[], - context: DispatchContext | undefined, -): Promise> { - const { x, y } = readPoint(positionals, 'read requires x y'); - if (device.platform === 'android') { - const { readAndroidTextAtPoint } = await import('../platforms/android/input-actions.ts'); - const text = await readAndroidTextAtPoint(device, x, y); - return { action: 'read', text: text ?? '' }; - } - if (device.platform === 'linux') { - const { readLinuxTextAtPoint } = await import('../platforms/linux/snapshot.ts'); - const text = await readLinuxTextAtPoint(x, y, context?.surface); - return { action: 'read', text }; - } - if (isMacOs(device) && context?.surface && context.surface !== 'app') { - const { runMacOsReadTextAction } = await import('../platforms/apple/os/macos/helper.ts'); - const result = await runMacOsReadTextAction(x, y, { - bundleId: context.appBundleId, - surface: context.surface, - }); - return { action: 'read', text: result.text }; - } - // macOS app sessions run through the XCUITest runner; only desktop/menubar surfaces use the helper. - const { runAppleRunnerCommand } = await import('../platforms/apple/core/runner/runner-client.ts'); - const result = await runAppleRunnerCommand( - device, - { - command: 'readText', - x, - y, - appBundleId: context?.appBundleId, - }, - { - verbose: context?.verbose, - logPath: context?.logPath, - traceLogPath: context?.traceLogPath, - requestId: context?.requestId, - iosXctestrunFile: context?.iosXctestrunFile, - iosXctestDerivedDataPath: context?.iosXctestDerivedDataPath, - iosXctestEnvDir: context?.iosXctestEnvDir, - }, - ); - const text = - typeof result.text === 'string' - ? result.text - : typeof result.message === 'string' - ? result.message - : ''; - return { action: 'read', text }; -} - function findMistargetedTypeRef(positionals: string[]): string | null { return findMistargetedTypeRefToken(positionals[0]); } diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 4d4345e4e1..c0232b8e2e 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -18,7 +18,6 @@ import { handleHoverCommand, handleLongPressCommand, handlePressCommand, - handleReadCommand, handleScrollCommand, handleTypeCommand, } from './dispatch-interactions.ts'; @@ -180,7 +179,6 @@ const DISPATCH_HANDLERS: Record = { handleTvRemoteCommand(device, interactor, positionals, context), settings: ({ device, interactor, positionals, context }) => handleSettingsCommand(device, interactor, positionals, context), - read: ({ device, positionals, context }) => handleReadCommand(device, positionals, context), }; /** diff --git a/src/daemon/handlers/__tests__/find-handler-fixture.ts b/src/daemon/handlers/__tests__/find-handler-fixture.ts new file mode 100644 index 0000000000..94ce11baf5 --- /dev/null +++ b/src/daemon/handlers/__tests__/find-handler-fixture.ts @@ -0,0 +1,29 @@ +import type { SessionStore } from '../../session-store.ts'; +import type { DaemonRequest, DaemonResponse } from '../../types.ts'; +import { handleFindCommands } from '../find.ts'; +import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; + +/** + * One `handleFindCommands` invocation shape. + * + * Read-only `find` constructs a BOUND selector backend — it shares the element read with `get` — + * so every caller needs the request-runtime seams. It lives here rather than in `find.test.ts` + * because that file is over the module-size tripwire and may only shrink. + */ +export function invokeFindHandler(params: { + sessionName: string; + sessionStore: SessionStore; + positionals: string[]; + flags?: DaemonRequest['flags']; + invoke: (req: DaemonRequest) => Promise; +}) { + const { sessionName, sessionStore, positionals, flags } = params; + return handleFindCommands({ + req: { token: 't', session: sessionName, command: 'find', positionals, flags: flags ?? {} }, + sessionName, + logPath: '/tmp/test.log', + sessionStore, + ...getRuntimeBindings(), + invoke: params.invoke, + }); +} diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 146c0cc6ab..55c12896e6 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -28,9 +28,13 @@ vi.mock('../snapshot-interactor-capture.ts', async () => { import { dispatchCommand } from '../../../core/dispatch.ts'; +import { resetGetRuntimeFixture } from './interaction-get-runtime-fixture.ts'; +import { invokeFindHandler } from './find-handler-fixture.ts'; + const mockDispatch = vi.mocked(dispatchCommand); beforeEach(() => { + resetGetRuntimeFixture(); mockDispatch.mockReset(); mockDispatch.mockImplementation(async (_device: unknown, command: string) => { return command === 'snapshot' ? { nodes: [] } : {}; @@ -63,17 +67,11 @@ async function runFindClickScenario(options: { } const invokeCalls: DaemonRequest[] = []; - const response = await handleFindCommands({ - req: { - token: 't', - session: sessionName, - command: 'find', - positionals: options.positionals, - flags: options.flags ?? {}, - }, + const response = await invokeFindHandler({ sessionName, - logPath: '/tmp/test.log', sessionStore, + positionals: options.positionals, + flags: options.flags, invoke: async (req) => { invokeCalls.push(req); const data = options.invoke ? await options.invoke(req) : {}; @@ -1039,17 +1037,10 @@ test('read-only find while recording is intentionally deferred from target-v1 ev return {}; }); - const response = await handleFindCommands({ - req: { - token: 't', - session: sessionName, - command: 'find', - positionals: ['text', 'Save', 'exists'], - flags: {}, - }, + const response = await invokeFindHandler({ sessionName, - logPath: '/tmp/test.log', sessionStore, + positionals: ['text', 'Save', 'exists'], invoke: async () => ({ ok: true, data: {} }), }); @@ -1134,17 +1125,10 @@ async function runFindThroughLeaf(options: { ); const invokeCalls: DaemonRequest[] = []; - const response = await handleFindCommands({ - req: { - token: 't', - session: sessionName, - command: 'find', - positionals: options.positionals, - flags: {}, - }, + const response = await invokeFindHandler({ sessionName, - logPath: '/tmp/test.log', sessionStore, + positionals: options.positionals, invoke: async (req) => { invokeCalls.push(req); if (options.divergeBeforeDispatch) { diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts index fd6c718810..f0939fb484 100644 --- a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -90,17 +90,23 @@ const mockInspectElementReadFacts: InspectDeviceRuntimeFacts = vi.fn(async (devi const mockBindElementReadRuntime: BindDeviceRuntime = vi.fn(async (device: DeviceInfo, use) => { const facts = elementReadFacts(device); + // Delegates to the interactor capture the surrounding suites already mock, so only the two + // selector operations are fixture-owned here. + const capture = async (input: CaptureSnapshotInput) => + await captureSnapshotWithInteractor({ + device, + runnerContext: { ...input.execution, appBundleId: input.options?.appBundleId }, + options: { ...input.options }, + }); const binding: DeviceBinding = Object.freeze({ device, owner: localRuntimeOwner('apple'), facts, operations: Object.freeze({ - captureSnapshot: async (input: CaptureSnapshotInput) => - await captureSnapshotWithInteractor({ - device, - runnerContext: { ...input.execution, appBundleId: input.options?.appBundleId }, - options: { ...input.options }, - }), + captureSnapshot: capture, + // The selector plan takes this row on a session with no tracked app, so an owner that + // advertises it must implement it or `narrowDeviceBinding` rejects the contract. + captureSnapshotWithoutActiveApp: capture, ...(elementReadFixtureState.readTextAtPointAvailable ? { readTextAtPoint: mockReadTextAtPoint } : {}), @@ -110,7 +116,10 @@ const mockBindElementReadRuntime: BindDeviceRuntime = vi.fn(async (device: Devic return narrowDeviceBinding(binding, use); }) as BindDeviceRuntime; -/** Spread into any interaction-handler params so `get` can admit and bind. */ +/** + * Spread into a handler's params so a selector command can admit and bind. Consumed by `get` and + * by read-only `find`, which share the bound element read. + */ export function getRuntimeBindings(): Readonly<{ inspectFacts: InspectDeviceRuntimeFacts; bindDevice: BindDeviceRuntime; diff --git a/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts index 4919ef60a7..aed6b991cc 100644 --- a/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts +++ b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts @@ -1,5 +1,6 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { handleFindCommands } from '../find.ts'; +import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; import { dispatchFindReadOnlyViaRuntime, dispatchWaitViaRuntime } from '../../selector-runtime.ts'; import type { DaemonRequest, DaemonResponse } from '../../types.ts'; import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../core/android-system-surface-disclosure.ts'; @@ -105,6 +106,7 @@ test('read-only find exists on a system-surface capture discloses the occlusion' sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -130,6 +132,7 @@ test('wait timeout for app text hidden behind a system surface discloses the occ sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response.ok).toBe(false); @@ -153,6 +156,7 @@ test('sessionless read-only find still discloses the occluding system surface', sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -213,6 +217,7 @@ test('sessionless wait success on shade content still discloses the occluding sy sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response.ok).toBe(true); @@ -238,6 +243,7 @@ test('sessionless wait timeout still discloses the occluding system surface', as sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response.ok).toBe(false); diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 89184d80a7..82b38d6a03 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -20,6 +20,7 @@ import { recordSessionAction } from './handler-utils.ts'; import { stripInternalInteractionFlags } from '../interaction-outcome-policy.ts'; import { resolveFindMatch } from './find-match-resolution.ts'; import { dispatchFindReadOnlyViaRuntime } from '../selector-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; import { createFindTargetCapture, sparseFindSnapshotResponse } from './find-target-capture.ts'; import { isSparseSnapshotQualityVerdict } from '../../snapshot-quality/verdict.ts'; @@ -57,6 +58,8 @@ export async function handleFindCommands(params: { logPath: string; sessionStore: SessionStore; invoke: DaemonInvokeFn; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; }): Promise { const { req, sessionName, logPath, sessionStore, invoke } = params; const command = req.command; @@ -83,6 +86,8 @@ export async function handleFindCommands(params: { sessionName, logPath, sessionStore, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }); if (runtimeResponse) return runtimeResponse; // Read-only find actions (exists/wait/list/get_text/get_attrs) always return from diff --git a/src/daemon/handlers/interaction-read-legacy-dispatch.ts b/src/daemon/handlers/interaction-read-legacy-dispatch.ts deleted file mode 100644 index a609cdbea7..0000000000 --- a/src/daemon/handlers/interaction-read-legacy-dispatch.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { dispatchCommand } from '../../core/dispatch.ts'; -import type { SessionState } from '../types.ts'; -import type { ContextFromFlags } from './interaction-common.ts'; -import type { CommandFlags } from '@agent-device/contracts/command'; -import { elementTextRead } from '@agent-device/contracts/platform'; -import type { ReadElementTextAtPoint } from './interaction-read.ts'; - -/** - * The legacy `read` dispatch, adapted to the neutral point-read shape. - * - * OWNED DEBT, with a named retirement trigger (#1739): `readText` on the shared selector backend - * serves `get text` and read-only `find … get text`. `get` is migrated and passes its bound - * `readTextAtPoint` instead of this adapter; `find` is a separate unit and still reaches the - * platform through the `read` dispatch alias. This module, the `read` registry entry, its - * `dispatch: {}` projection, `DISPATCH_HANDLERS.read`, and `handleReadCommand` all retire - * together in the unit that migrates `find`'s read-only path — the last selector-read consumer. - * - * It is not a fallback: no command chooses between this and a bound operation. Which one the - * shared backend receives is fixed by whether the calling command has cut over. - */ -export function legacyDispatchReadTextAtPoint(params: { - device: SessionState['device']; - flags: CommandFlags | undefined; - surface?: SessionState['surface']; - contextFromFlags: ContextFromFlags; -}): ReadElementTextAtPoint { - return async (input) => { - const rawData = await dispatchCommand( - params.device, - 'read', - [String(input.point.x), String(input.point.y)], - undefined, - { - ...params.contextFromFlags( - params.flags, - input.options?.appBundleId, - input.execution?.traceLogPath, - ), - surface: params.surface, - }, - ); - const data = rawData && typeof rawData === 'object' ? rawData : undefined; - return elementTextRead(typeof data?.text === 'string' ? data.text : undefined); - }; -} diff --git a/src/daemon/request-handler-chain.ts b/src/daemon/request-handler-chain.ts index 0b1240a863..2b757fc8dc 100644 --- a/src/daemon/request-handler-chain.ts +++ b/src/daemon/request-handler-chain.ts @@ -224,6 +224,8 @@ async function runFindHandler( logPath: params.logPath, sessionStore: params.sessionStore, invoke: params.invoke, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }), ); } diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 1593913bdf..6b88b659db 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -12,7 +12,6 @@ import { createDaemonRuntimeSessionStore } from './runtime-session.ts'; import { contextFromFlags } from './context.ts'; import { ensureDeviceReady } from './device-ready.ts'; import { readTextForNode } from './handlers/interaction-read.ts'; -import { legacyDispatchReadTextAtPoint } from './handlers/interaction-read-legacy-dispatch.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import type { ContextFromFlags } from './handlers/interaction-common.ts'; import { SessionStore } from './session-store.ts'; @@ -133,7 +132,7 @@ export async function createBoundSelectorRuntime( */ export async function createSelectorRuntime( params: SelectorRuntimeParams, - options: { requireSession: boolean; capability: 'find' | 'is' }, + options: { requireSession: boolean; capability: 'is' }, ): Promise { const resolved = await resolveSelectorRuntimeDevice(params, options.requireSession); if (!resolved.ok) return resolved; @@ -150,25 +149,15 @@ export async function createSelectorRuntime( } function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDeviceBackend { - // Which read the backend uses is fixed by WHICH COMMAND CONSTRUCTED THIS RUNTIME, never by - // failure, family, environment, or flag. A migrated command arrives with `bound` and its - // binding is authoritative — including when the owner advertised no read, which is the complete - // required path. An unmigrated selector command arrives without `bound` and keeps the legacy - // dispatch until its own descriptor cuts over (ADR 0019 §6 permits the unmigrated sibling's own - // path); a migrated command can never reach it. + // The bound operation is the ONLY element read. Both consumers of the shared backend read — + // `get text` and read-only `find … get text` — construct a bound backend, so there is no second + // read path to choose between and nothing reaches the retired `read` dispatch. const { req, session, device, logPath, sessionName, sessionStore } = params; const resolveContextFromFlags: ContextFromFlags = params.contextFromFlags ?? ((flags, appBundleId, traceLogPath) => contextFromFlags(logPath ?? '', flags, appBundleId, traceLogPath)); - const readTextAtPoint = params.bound - ? params.bound.readText - : legacyDispatchReadTextAtPoint({ - device, - flags: req.flags, - surface: session?.surface, - contextFromFlags: resolveContextFromFlags, - }); + const readTextAtPoint = params.bound?.readText; const captureRuntime = createSelectorCaptureRuntime({ device, session, diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 0c282a0728..a8a38a276b 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -88,9 +88,13 @@ export async function dispatchFindReadOnlyViaRuntime( const action = parsed.action; if (!isReadOnlyFindAction(action)) return null; - const resolvedRuntime = await createSelectorRuntime(params, { + // Read-only `find` shares the element read with `get`, so it constructs a BOUND backend and + // the two consume one bound operation instead of one binding it and the other dispatching the + // legacy `read`. This moves find's READ LEG only: find's descriptor stays + // `LEGACY_PLATFORM_EXECUTION`, it claims no cutover row, and its mutating actions are untouched. + const resolvedRuntime = await createBoundSelectorRuntime(params, { requireSession: false, - capability: 'find', + command: 'find', }); if (!resolvedRuntime.ok) return resolvedRuntime.response; From fbeba2f4b35975f31b1014c083ae91c2e105efc4 Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 19 Aug 2026 17:28:32 +0200 Subject: [PATCH 5/9] refactor(daemon): apply the seam addendum after #1876 was re-scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two edits, per find's ADDENDUM.md: 1. `includeRects` returns to `buildRuntimeCaptureInput`. It was removed from #1876 as unconsumed; the selector capture path is genuinely its first consumer (a Web rect capture requests bounds explicitly), so it lands here under the same rule that moved the seam. `snapshot`/`diff` pass nothing. 2. The per-capture `signal` is dropped, not restored. `CaptureSnapshotInput` has no such field on this stack — it moved to `wait` (#1875) with the regression that proves per-poll abort and quiescence. `get` captures once per resolution and never polls, so nothing here needs it. The seam test and fixture coverage for it moves with the contract rather than being kept against a field that no longer exists. --- .../src/platform-runtime-unavailable.test.ts | 1 + .../src/platform-runtime-unavailable.ts | 2 -- packages/platform-android/src/runtime.ts | 1 + packages/platform-apple/src/runtime.ts | 1 + packages/platform-linux/src/runtime.ts | 2 ++ .../__tests__/selector-capture-binding.test.ts | 18 ------------------ .../__tests__/selector-capture-fixture.ts | 1 - .../interaction-get-runtime-fixture.ts | 1 + .../__tests__/session-capabilities.fixtures.ts | 5 ----- src/daemon/selector-capture-binding.ts | 2 +- src/daemon/selector-capture-runtime.ts | 3 --- src/daemon/snapshot-runtime-capture-input.ts | 6 ++++++ src/platform-runtime-gateway.test.ts | 1 + 13 files changed, 14 insertions(+), 30 deletions(-) diff --git a/packages/contracts/src/platform-runtime-unavailable.test.ts b/packages/contracts/src/platform-runtime-unavailable.test.ts index 749a034248..c250bb97bb 100644 --- a/packages/contracts/src/platform-runtime-unavailable.test.ts +++ b/packages/contracts/src/platform-runtime-unavailable.test.ts @@ -45,6 +45,7 @@ test('generic unavailable binding preserves exact provider ownership and mode', assert.deepEqual(binding.facts.operations.captureScreenshot, { available: false, reason: 'unsupported-device-kind', + }); assert.deepEqual(binding.facts.operations.readTextAtPoint, { available: false, reason: 'unsupported-provider-mode', diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index cc5138dab4..c0b1eb6ae0 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -144,8 +144,6 @@ function freezeUnavailableFacts( readiness: orNetwork(unavailable.readiness), shutdown: orNetwork(unavailable.shutdown), elementText: Object.freeze({ ...unavailable.elementText }), - readiness: Object.freeze({ ...(unavailable.readiness ?? unavailable.network) }), - shutdown: Object.freeze({ ...(unavailable.shutdown ?? unavailable.network) }), lifecycle: applicationLifecycleOperationFacts(unavailable.lifecycle), }); } diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 976648e98c..408735d0e8 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -218,6 +218,7 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, }) + : {}), ...(facts.operations.readTextAtPoint.available ? bindElementTextRuntime({ device: request.device, host: host.elementText }) : {}), diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index a8721188c7..27ae133e86 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -286,6 +286,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, }) + : {}), ...(facts.operations.readTextAtPoint.available ? bindElementTextRuntime({ device: request.device, host: host.elementText }) : {}), diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 2ddeaf0b5b..261b2d7df3 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -95,6 +95,7 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR signal: request.scope.signal, resolveInteractor: host.localInteractors.resolve, }) + : {}), ...(facts.operations.readTextAtPoint.available ? bindElementTextRuntime({ device: request.device, host: host.elementText }) : {}), @@ -140,6 +141,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts }), ...screenshotRuntimeOperationFacts({ capture: device.kind === 'device' ? supported : screenshotKindUnavailable, + }), // The Linux read is value-first (AXValue/title/description) where the captured tree is // label-first, so the desktop row genuinely reads differently from its snapshot text. ...elementTextRuntimeOperationFacts({ diff --git a/src/daemon/__tests__/selector-capture-binding.test.ts b/src/daemon/__tests__/selector-capture-binding.test.ts index 29045be675..4aa78fb791 100644 --- a/src/daemon/__tests__/selector-capture-binding.test.ts +++ b/src/daemon/__tests__/selector-capture-binding.test.ts @@ -1,5 +1,4 @@ import { expect, test } from 'vitest'; -import type { CaptureSnapshotInput } from '@agent-device/contracts/platform'; import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; import { makeAndroidSession, @@ -48,23 +47,6 @@ test('repeated captures reuse the one binding the plan was admitted for', async expect(fixture.captures).toHaveLength(2); }); -test('a per-capture signal reaches the bound operation for a poll deadline', async () => { - const fixture = selectorCaptureFixture(); - const bound = await resolveBoundSelectorCapture({ - command: 'wait', - device: ANDROID_EMULATOR, - session: makeAndroidSession('selector'), - inspectFacts: fixture.inspectFacts, - bindDevice: fixture.bindDevice, - }); - if (!bound.ok) throw new Error('expected an admitted capture'); - const deadline = new AbortController(); - deadline.abort(new Error('poll deadline')); - - const input: CaptureSnapshotInput = { signal: deadline.signal }; - await expect(bound.operations.capture(input)).rejects.toThrow(/poll deadline/); -}); - test('an unavailable required operation refuses before any bind', async () => { const fixture = selectorCaptureFixture({ capture: { available: false, reason: 'unsupported-platform-leaf' }, diff --git a/src/daemon/__tests__/selector-capture-fixture.ts b/src/daemon/__tests__/selector-capture-fixture.ts index 4a70277778..2f974d5c3b 100644 --- a/src/daemon/__tests__/selector-capture-fixture.ts +++ b/src/daemon/__tests__/selector-capture-fixture.ts @@ -61,7 +61,6 @@ export function selectorCaptureFixture( const captureSnapshot = async (input: CaptureSnapshotInput): Promise => { const index = captures.length; captures.push(input); - input.signal?.throwIfAborted(); return params.snapshot?.(input, index) ?? { nodes: [], backend: 'xctest' }; }; diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts index f0939fb484..fd5a944f53 100644 --- a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -58,6 +58,7 @@ function elementReadFacts(device: DeviceInfo): RuntimeFacts Promise; diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 32ef876c82..6eda22b0f8 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -204,9 +204,6 @@ async function runCapture( session: params.session, snapshotScope, includeRects: request.includeRects, - // The poll's remaining budget, not the request's: `wait`/`find wait` - // abort a stalled capture at its deadline instead of racing it. - signal: request.signal, }), ), }), diff --git a/src/daemon/snapshot-runtime-capture-input.ts b/src/daemon/snapshot-runtime-capture-input.ts index d89688b96a..7b593195ba 100644 --- a/src/daemon/snapshot-runtime-capture-input.ts +++ b/src/daemon/snapshot-runtime-capture-input.ts @@ -15,6 +15,11 @@ export function buildRuntimeCaptureInput( meta?: DaemonRequest['meta']; session: SessionState | undefined; snapshotScope: string | undefined; + /** + * Web rect captures request bounds explicitly. Lands here with the selector capture path, + * its first consumer; `snapshot`/`diff` pass nothing and are unaffected. + */ + includeRects?: boolean; }>, ): CaptureSnapshotInput { const { flags, logPath, meta, session, snapshotScope } = params; @@ -37,6 +42,7 @@ export function buildRuntimeCaptureInput( raw: flags?.snapshotRaw, customActions: flags?.snapshotCustomActions, includeHiddenContentHints: flags?.snapshotIncludeHiddenContentHints, + includeRects: params.includeRects, surface, }, execution: { diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index 93366c26c8..ba2e386a09 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -48,6 +48,7 @@ describe('composed platform runtime gateway', () => { appLog: unavailable, network: unavailable, screenshot: unavailable, + elementText: unavailable, viewport: unavailable, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: unavailable, From 378c4f3b7fb489eac9206afb6d96f796f84990cc Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 19 Aug 2026 18:08:30 +0200 Subject: [PATCH 6/9] refactor(get): retire the direct-iOS selector shortcut MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `get` declares device-runtime, so its request path must reach the platform only through operations R36 declares. `dispatchDirectIosSelectorGet` reached `runAppleRunnerCommand` through a path the row declares no operation for; admitting before a bypass is not executing through the seam, so the bypass is removed rather than ordered after admission. Every target shape — including the simple iOS `id=` selector — now resolves through the bound capture. `queryDirectIosSelector` itself stays: `offscreen-target-probe.ts` still consumes it and it remains single-copy. `dispatchDirectIosSelectorIs` belongs to `is` (#1883). Two get-only helpers (`readDirectIosGetSelector`, `buildDirectIosGetResult`) became unreachable and are deleted with the caller. Declaring `querySelector` as a fact-admitted preferred operation was rejected on duplication, not correctness: the offscreen probe takes a plain session and cannot consume a bound operation, so it would ship the query twice until Wave 5 moves the probe — the deferred-duplication shape this PR was already overruled for on the `read` alias. It returns as a declared, §9-measured operation in a later unit that also moves the probe. Cost, stated plainly: `get text id=…` loses its tree-capture skip on iOS. No fallback was added and the latency is not recovered elsewhere. R36's singularExecution claim is now what the code does rather than aspirational. --- .../layering/runtime-command-cutover-table.ts | 9 ++- .../handlers/__tests__/interaction.test.ts | 81 +++++-------------- src/daemon/selector-runtime.ts | 70 ++-------------- 3 files changed, 35 insertions(+), 125 deletions(-) diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index a7126fea38..ecdf21ca35 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -523,7 +523,12 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ legacyRetirement: { modulePaths: ['src/daemon/handlers/interaction-read-legacy-dispatch.ts'], importPatterns: [/(?:^|\/)handlers\/interaction-read-legacy-dispatch(?:\.[cm]?[jt]s)?$/], - routeNames: ['handleReadCommand'], + // `dispatchDirectIosSelectorGet` was `get`'s last route to the platform outside the seam: + // it reached `runAppleRunnerCommand` through a path this row declares no operation for. + // Admitting before a bypass is not executing through the seam, so the bypass is retired + // rather than ordered after admission. `queryDirectIosSelector` itself stays — the Wave 5 + // offscreen-target probe still consumes it and it remains single-copy. + routeNames: ['handleReadCommand', 'dispatchDirectIosSelectorGet'], }, runtimeTypeNames: ['ElementTextRuntimeOperations', 'SnapshotRuntimeOperations'], operations: { @@ -534,6 +539,8 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ operations: ['captureSnapshot', 'captureSnapshotWithoutActiveApp', 'readTextAtPoint'], // `get` executes through the shared selector seam, so the capture owners are the SAME // selectors `snapshot`/`diff` count; only the preferred element read is this unit's own. + // With the direct-iOS bypass retired these names are now the ONLY routes from + // `dispatchGetViaRuntime` to the platform, so the claim states what the code does. operationOwners: { captureSnapshot: ['selectActiveAppSnapshot'], captureSnapshotWithoutActiveApp: ['selectSnapshotWithoutActiveApp'], diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 27a73899e6..b34583f8d8 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -1,5 +1,4 @@ import { test, expect, vi, beforeEach } from 'vitest'; -import { AppError } from '@agent-device/kernel/errors'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { WEB_DESKTOP_DEVICE } from '../../../__tests__/test-utils/device-fixtures.ts'; import { @@ -263,17 +262,29 @@ test('an eligible direct iOS selector cannot operate before admission', async () expect(mockDispatch).not.toHaveBeenCalled(); }); -test('get text simple iOS id selector uses runner query without snapshot', async () => { +// The direct-iOS shortcut is RETIRED (#1739): `get` declares `device-runtime`, so a simple +// `id=` selector resolves through the bound capture like every other shape rather than through a +// raw runner query R36 declares no operation for. The cost is real and accepted — this selector +// no longer skips the tree capture. +test('get text simple iOS id selector resolves through the bound capture, not a runner query', async () => { const sessionStore = makeSessionStore(); const sessionName = 'get-text-ios-direct-selector'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockRunAppleRunnerCommand.mockResolvedValue({ - found: true, - text: 'Ada Lovelace', + mockDispatch.mockResolvedValue({ + backend: 'xctest', nodes: [ { index: 0, depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 393, height: 852 }, + enabled: true, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, type: 'TextField', label: 'Name', identifier: 'field-name', @@ -300,29 +311,16 @@ test('get text simple iOS id selector uses runner query without snapshot', async }); expect(response?.ok).toBe(true); - expect(mockRunAppleRunnerCommand).toHaveBeenCalledWith( - expect.anything(), - { - command: 'querySelector', - selectorKey: 'id', - selectorValue: 'field-name', - appBundleId: 'com.example.app', - }, - expect.anything(), - ); - expect(mockDispatch).not.toHaveBeenCalledWith( - expect.anything(), - 'snapshot', - expect.anything(), - expect.anything(), - expect.anything(), - ); if (response?.ok) { expect(response.data?.text).toBe('Ada Lovelace'); expect(response.data?.selector).toBe('id="field-name"'); } - const recorded = sessionStore.get(sessionName)?.actions.at(-1); - expect(recorded?.result?.selectorChain).toEqual(['id="field-name"']); + // No querySelector: the retired bypass was the only caller on this path. + expect(mockRunAppleRunnerCommand).not.toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ command: 'querySelector' }), + expect.anything(), + ); }); test('get text iOS label selector uses snapshot disambiguation instead of runner query', async () => { @@ -397,41 +395,6 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner } }); -test('get text simple iOS id selector does not snapshot-fallback on ambiguous runner match', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'get-text-ios-direct-selector-ambiguous'; - sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockRunAppleRunnerCommand.mockRejectedValue( - new AppError('AMBIGUOUS_MATCH', 'selector matched multiple elements'), - ); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'get', - positionals: ['text', 'id="field-name"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(false); - if (response?.ok === false) { - expect(response.error.code).toBe('AMBIGUOUS_MATCH'); - } - expect(mockDispatch).not.toHaveBeenCalledWith( - expect.anything(), - 'snapshot', - expect.anything(), - expect.anything(), - expect.anything(), - ); -}); - test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { const sessionStore = makeSessionStore(); const sessionName = 'snapshot-flags'; diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index a8a38a276b..ea53978f2a 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -171,23 +171,17 @@ export async function dispatchGetViaRuntime( // snapshot path so the post-resolution identity guard runs. const replayTargetGuard = req.internal?.replayTargetGuard; - // ADR 0019: `get` declares `device-runtime`, so NOTHING in its request path may reach the - // device before resolve -> admit -> bind. Admission runs first for every target shape, - // including the ones the direct-iOS fast path below can answer: that path is a fast path - // *within* an admitted request, never a way around exact-owner facts or the one-binding - // invariant. (The query itself is still the shared root mechanic co-owned by `is`, `wait`, - // and the Wave 5 offscreen probe — this unit orders it, it does not claim it.) + // ADR 0019: `get` declares `device-runtime`, so its request path reaches the device ONLY + // through operations R36 declares. Every target shape — including the simple iOS `id=` selector + // a direct runner query used to answer without a capture — resolves through the bound capture. + // Admission before a bypass is not the same as executing through the seam, so the bypass is + // gone rather than merely ordered after admission. const resolvedRuntime = await createBoundSelectorRuntime(params, { requireSession: true, command: 'get', }); if (!resolvedRuntime.ok) return resolvedRuntime.response; - if (target.target.kind === 'selector' && !replayTargetGuard) { - const directResponse = await dispatchDirectIosSelectorGet(params, sub, target.target.selector); - if (directResponse) return directResponse; - } - const runtime = resolvedRuntime.runtime; // #1076 + ADR 0014: a get @ref binds against the retained ref-frame evidence, @@ -382,44 +376,6 @@ function readRecordedResolutionTarget( return { node: node as SnapshotNode, preActionNodes: preActionNodes as SnapshotNode[] }; } -function readDirectIosGetSelector( - session: SessionState | undefined, - property: 'text' | 'attrs', - selectorExpression: string, -): DirectIosSelectorTarget | null { - // ADR 0012 decision 3: recording requires the snapshot path so target - // evidence can be computed from the resolution tree. - if (!session || isSessionRecording(session)) return null; - const selector = readSimpleIosSelectorTarget({ session, selectorExpression }); - // get text intentionally disambiguates label/text/value triplets from snapshots; the runner - // direct query rejects those ambiguous matches before the shared selector resolver can rank them. - if (property === 'text' && selector?.key !== 'id') return null; - return selector; -} - -async function dispatchDirectIosSelectorGet( - params: SelectorRuntimeParams, - property: 'text' | 'attrs', - selectorExpression: string, -): Promise { - const session = params.sessionStore.get(params.sessionName); - const selector = readDirectIosGetSelector(session, property, selectorExpression); - if (!session || !selector) return null; - - const result = await queryDirectIosSelectorOrFallback(params, session, selector); - if (isDirectIosSelectorErrorResult(result)) return result.response; - if (!result) return null; - const payload = buildDirectIosGetResult(property, selector.raw, result); - if (!payload) return null; - recordIfSession( - params.sessionStore, - params.sessionName, - params.req, - buildGetRecordResult(payload, property), - ); - return { ok: true, data: toDaemonGetData(payload) }; -} - async function dispatchDirectIosSelectorIs( params: SelectorRuntimeParams, predicate: IsPredicate, @@ -555,22 +511,6 @@ function isDirectIosSelectorErrorResult( return result !== null && 'kind' in result && result.kind === 'error'; } -function buildDirectIosGetResult( - property: 'text' | 'attrs', - selector: string, - result: DirectIosSelectorQueryResult, -) { - if (!result.found || !result.node) return null; - const base = { - target: { kind: 'selector' as const, selector }, - node: result.node, - selectorChain: [selector], - }; - if (property === 'attrs') return { kind: 'attrs' as const, ...base }; - if (typeof result.text !== 'string') return null; - return { kind: 'text' as const, ...base, text: result.text }; -} - function buildDirectIosIsResult( predicate: Exclude, expectedText: string, From a3978d46baf4146ebea2fcf9c7058324e842fea0 Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 19 Aug 2026 21:42:09 +0200 Subject: [PATCH 7/9] refactor: ride the Interactor seam for the element read; drop the bespoke host Two operations of the same class were reaching their mechanics two different ways: `findText` rides `Interactor` via `localInteractors.resolve`, while `readTextAtPoint` had its own host port. That is duplication of MECHANISM, so the read now rides the same seam. `Interactor` gains `readTextAtPoint?`, implemented on the Apple, Android and Linux interactors where those mechanics already live. `src/platform-runtime-element-text-host.ts` and its `elementText` host wiring are deleted; the contract binds through the resolver exactly as the snapshot runtime does. Size honesty: this removes an 89-line module but the four readers still have to exist, so they moved into the interactors rather than vanishing. Net production change is ~4 lines, not ~89. The duplication of mechanism is what is actually fixed; Wave 5/6 retires the seam for both operations together. Also from the size investigation: - `ElementTextRuntimeExecution` was byte-identical to `SnapshotRuntimeExecution`; removed and reused, as `find-text-runtime.ts` does. - Removed a stranded, stale comment in `selector-capture-binding.ts` that still claimed a duplication this branch had already retired. - `FrozenUnavailablePlatformRuntimeFacts` is derived from its input type rather than restated, removing a 14-line clone group my new cell had pushed over the detector threshold. --- .../contracts/src/element-text-runtime.ts | 54 +++++++---- packages/contracts/src/facades/platform.ts | 3 +- packages/contracts/src/interactor-types.ts | 10 +++ .../src/platform-runtime-operations.ts | 6 +- .../src/platform-runtime-unavailable.ts | 23 ++--- packages/platform-android/src/runtime.ts | 6 +- packages/platform-apple/src/runtime.ts | 6 +- packages/platform-linux/src/runtime.ts | 6 +- src/core/interactors/android.ts | 5 ++ src/core/interactors/linux.ts | 6 ++ src/daemon/selector-capture-binding.ts | 11 ++- src/platform-runtime-element-text-host.ts | 89 ------------------- src/platform-runtime-operation-host.ts | 2 - .../interactor-runner-provider.test.ts | 6 ++ src/platforms/apple/interactor.ts | 42 ++++++++- 15 files changed, 136 insertions(+), 139 deletions(-) delete mode 100644 src/platform-runtime-element-text-host.ts diff --git a/packages/contracts/src/element-text-runtime.ts b/packages/contracts/src/element-text-runtime.ts index 61ca6cb334..8936418f9a 100644 --- a/packages/contracts/src/element-text-runtime.ts +++ b/packages/contracts/src/element-text-runtime.ts @@ -1,11 +1,9 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Point } from '@agent-device/kernel/snapshot'; -import type { RunnerContext } from './interactor-types.ts'; +import type { Interactor, RunnerContext } from './interactor-types.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; import type { SessionSurface } from './session-surface.ts'; - -/** Runner metadata the selected read implementation needs, without request-owned state. */ -export type ElementTextRuntimeExecution = Readonly>; +import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; /** * Neutral intent for one point-addressed element read. The point is already resolved from the @@ -14,7 +12,8 @@ export type ElementTextRuntimeExecution = Readonly; - execution?: ElementTextRuntimeExecution; + /** Same runner metadata a capture needs; reuses that type rather than restating it. */ + execution?: SnapshotRuntimeExecution; }>; /** @@ -71,24 +70,47 @@ export function elementTextRuntimeOperationFacts( return Object.freeze({ readTextAtPoint: input.readTextAtPoint }); } +/** Resolves the selected owner's interactor, exactly as the snapshot runtime does. */ +export type ElementTextInteractorResolver = ( + device: DeviceInfo, + runner: RunnerContext, +) => Promise; + /** - * The existing per-family read mechanics, injected by composition. Families reach their own - * tools through this port rather than importing root modules, matching the snapshot runtime's - * interactor-resolver seam. + * Binds the owner's live point read for the lifetime of a request binding. + * + * Rides the same `Interactor` seam `findText` uses rather than a bespoke host port: two + * operations of the same class reaching their mechanics two different ways is duplication of + * mechanism, and Wave 5/6 retires the seam for both together. */ -export type ElementTextRuntimeHost = Readonly<{ - readTextAtPoint(device: DeviceInfo, input: ReadTextAtPointInput): Promise; -}>; - -/** Captures one selected owner's read authority for the lifetime of a request binding. */ export function bindElementTextRuntime( params: Readonly<{ device: DeviceInfo; - host: ElementTextRuntimeHost; + signal: AbortSignal; + resolveInteractor: ElementTextInteractorResolver; }>, ): ElementTextRuntimeOperations { return Object.freeze({ - readTextAtPoint: async (input: ReadTextAtPointInput) => - await params.host.readTextAtPoint(params.device, input), + readTextAtPoint: async (input: ReadTextAtPointInput) => { + const signal = params.signal; + signal.throwIfAborted(); + const interactor = await params.resolveInteractor(params.device, { + ...input.execution, + appBundleId: input.options?.appBundleId, + signal, + }); + // An owner whose facts advertised the read but whose interactor has none is a runtime + // contract error surfaced as a declined read, not a silent empty answer. + if (!interactor.readTextAtPoint) { + return Object.freeze({ status: 'unreadable', reason: 'surface-not-readable' } as const); + } + return elementTextRead( + await interactor.readTextAtPoint(input.point, { + appBundleId: input.options?.appBundleId, + surface: input.options?.surface, + signal, + }), + ); + }, }); } diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index 7e412861fc..45c1d8d509 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -283,8 +283,7 @@ export { } from '../element-text-runtime.ts'; export type { ElementTextReadOutcome, - ElementTextRuntimeExecution, - ElementTextRuntimeHost, + ElementTextInteractorResolver, ElementTextRuntimeOperationFacts, ElementTextRuntimeOperations, ElementTextUnreadableReason, diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index d9b0164f3d..48041f3d4a 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -222,6 +222,16 @@ export type Interactor = { screenshot(outPath: string, options?: ScreenshotOptions): Promise; setViewport?(width: number, height: number): Promise | void>; snapshot(options?: SnapshotOptions): Promise; + /** + * Native reading of the live text at a point, when the backend has one. Answers the text the + * owner can see right now, which can exceed what an already-captured node carries (an editable + * field whose value is longer than its label). Optional: a backend without it leaves the + * captured tree as the complete answer. + */ + readTextAtPoint?( + point: Point, + options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal }, + ): Promise; gestureViewport?(): Promise; back(mode?: BackMode): Promise; home(): Promise; diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index d30c1b34da..cb9234f4d3 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -15,10 +15,7 @@ import type { ScreenRecordingRuntimeOperations } from './screen-recording-runtim import type { ScreenshotRuntimeOperations } from './screenshot-runtime.ts'; import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; -import type { - ElementTextRuntimeHost, - ElementTextRuntimeOperations, -} from './element-text-runtime.ts'; +import type { ElementTextRuntimeOperations } from './element-text-runtime.ts'; import type { DeviceReadinessRuntimeHost, DeviceReadinessRuntimeOperations, @@ -291,7 +288,6 @@ export type PlatformRuntimeHost = AppLogRuntimeHost & }>; screenRecording: ScreenRecordingRuntimeHost; snapshot: SnapshotRuntimeHost; - elementText: ElementTextRuntimeHost; deviceReadiness: DeviceReadinessRuntimeHost; deviceShutdown: DeviceShutdownRuntimeHost; localInteractors: LocalApplicationInteractorHost; diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index c0b1eb6ae0..7a995a6a82 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -35,21 +35,14 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{ lifecycle: ApplicationLifecycleOperationFacts; }>; -type FrozenUnavailablePlatformRuntimeFacts = Readonly<{ - appLog: RuntimeOperationUnavailability; - apps: RuntimeOperationUnavailability; - appDeployment: RuntimeOperationUnavailability; - appState: RuntimeOperationUnavailability; - network: RuntimeOperationUnavailability; - screenRecording: RuntimeOperationUnavailability; - screenshot: RuntimeOperationUnavailability; - snapshot: RuntimeOperationUnavailability; - viewport: RuntimeOperationUnavailability; - elementText: RuntimeOperationUnavailability; - readiness: RuntimeOperationUnavailability; - shutdown: RuntimeOperationUnavailability; - lifecycle: ApplicationLifecycleOperationFacts; -}>; +/** + * The same cells with every optional one resolved. Derived from the input type rather than + * restated, so a new cell cannot be added to one and forgotten in the other. + */ +type FrozenUnavailablePlatformRuntimeFacts = Readonly< + Required> & + Readonly<{ lifecycle: ApplicationLifecycleOperationFacts }> +>; export function createUnavailablePlatformRuntimeBinding( device: DeviceInfo, diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 408735d0e8..7d11af67b2 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -220,7 +220,11 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor }) : {}), ...(facts.operations.readTextAtPoint.available - ? bindElementTextRuntime({ device: request.device, host: host.elementText }) + ? bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) : {}), ensureReady: async (input: EnsureReadyInput) => await ensureAndroidReady( diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 27ae133e86..1b94570c3d 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -288,7 +288,11 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR }) : {}), ...(facts.operations.readTextAtPoint.available - ? bindElementTextRuntime({ device: request.device, host: host.elementText }) + ? bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) : {}), ...(facts.operations.ensureReady.available ? { diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 261b2d7df3..f8842b4f20 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -97,7 +97,11 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR }) : {}), ...(facts.operations.readTextAtPoint.available - ? bindElementTextRuntime({ device: request.device, host: host.elementText }) + ? bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) : {}), }), [Symbol.asyncDispose]: async () => undefined, diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 3295372225..7358d4419c 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -65,6 +65,11 @@ export function createAndroidInteractor( performGesture: (plan) => executeAndroidTouchPlan(device, plan), gestureViewport: () => readAndroidGestureViewport(device), screenshot: (outPath, options) => screenshotAndroid(device, outPath, options), + // uiautomator reads the node covering a point; `undefined` means nothing covers it. + readTextAtPoint: async (point) => { + const { readAndroidTextAtPoint } = await import('../../platforms/android/input-actions.ts'); + return (await readAndroidTextAtPoint(device, point.x, point.y)) ?? undefined; + }, snapshot: async (options) => { const snapshotOptions = options ?? {}; const result = await withDiagnosticTimer( diff --git a/src/core/interactors/linux.ts b/src/core/interactors/linux.ts index 9d57c990d5..5d7f21f167 100644 --- a/src/core/interactors/linux.ts +++ b/src/core/interactors/linux.ts @@ -45,6 +45,12 @@ export function createLinuxInteractor(): Interactor { await swipeLinux(start.x, start.y, end.x, end.y, plan.durationMs); }, screenshot: (outPath, options) => screenshotLinux(outPath, options), + // The Linux read is value-first (AXValue/title/description) where the captured tree is + // label-first, so this genuinely reads differently from its snapshot text. + readTextAtPoint: async (point, options) => { + const { readLinuxTextAtPoint } = await import('../../platforms/linux/snapshot.ts'); + return await readLinuxTextAtPoint(point.x, point.y, options?.surface); + }, snapshot: async (options) => { return await withDiagnosticTimer( 'snapshot_capture', diff --git a/src/daemon/selector-capture-binding.ts b/src/daemon/selector-capture-binding.ts index 73673e46b3..48c259e148 100644 --- a/src/daemon/selector-capture-binding.ts +++ b/src/daemon/selector-capture-binding.ts @@ -18,12 +18,6 @@ export type SelectorCaptureCommand = 'find' | 'get' | 'is' | 'wait'; */ export type BoundSelectorCapture = (input: CaptureSnapshotInput) => Promise; -/** - * The bound operations a selector command's runtime executes through. A record rather than a - * bare capture function on purpose: the next selector unit adds its own bound operation here - * (`get`'s preferred element read, whose platform branches `get` and `find get text` - * currently duplicate) without changing any signature on this seam. - */ /** * The owner's live element-text read, when its facts advertise one. Optional because it is a * PREFERRED operation: every selector read's required path answers from the captured tree, so an @@ -31,6 +25,11 @@ export type BoundSelectorCapture = (input: CaptureSnapshotInput) => Promise - await readTextAtPoint(device, input), - }); -} - -async function readTextAtPoint( - device: DeviceInfo, - input: ReadTextAtPointInput, -): Promise { - if (device.platform === 'android') return await readAndroidText(device, input); - if (device.platform === 'linux') return await readLinuxText(input); - if (usesMacOsHelperSurface(device, input)) return await readMacOsSurfaceText(input); - // macOS app sessions run through the XCUITest runner; only desktop/menubar surfaces use the - // helper, and every other Apple leaf reaches the runner directly. - return await readAppleRunnerText(device, input); -} - -/** Only non-app macOS surfaces are helper-read; an app session is runner-read like any Apple leaf. */ -function usesMacOsHelperSurface(device: DeviceInfo, input: ReadTextAtPointInput): boolean { - const surface = input.options?.surface; - return isMacOs(device) && surface !== undefined && surface !== 'app'; -} - -// Each reader classifies its own owner's "nothing here" answer through `elementTextRead`. -// None of them catches: a transport or tooling failure is unexpected and propagates. - -async function readAndroidText( - device: DeviceInfo, - input: ReadTextAtPointInput, -): Promise { - const { readAndroidTextAtPoint } = await import('./platforms/android/input-actions.ts'); - // uiautomator answers `undefined` when no node covers the point. - return elementTextRead(await readAndroidTextAtPoint(device, input.point.x, input.point.y)); -} - -async function readLinuxText(input: ReadTextAtPointInput): Promise { - const { readLinuxTextAtPoint } = await import('./platforms/linux/snapshot.ts'); - return elementTextRead( - await readLinuxTextAtPoint(input.point.x, input.point.y, input.options?.surface), - ); -} - -async function readMacOsSurfaceText(input: ReadTextAtPointInput): Promise { - const { runMacOsReadTextAction } = await import('./platforms/apple/os/macos/helper.ts'); - const result = await runMacOsReadTextAction(input.point.x, input.point.y, { - bundleId: input.options?.appBundleId, - surface: input.options?.surface, - }); - return elementTextRead(result.text); -} - -async function readAppleRunnerText( - device: DeviceInfo, - input: ReadTextAtPointInput, -): Promise { - const { runAppleRunnerCommand } = await import('./platforms/apple/core/runner/runner-client.ts'); - const result = await runAppleRunnerCommand( - device, - { - command: 'readText', - x: input.point.x, - y: input.point.y, - appBundleId: input.options?.appBundleId, - }, - { ...input.execution }, - ); - if (typeof result.text === 'string') return elementTextRead(result.text); - // The runner answers `message` instead of `text` when it queried the element but could not - // render readable text from it — a declined read, not a failed one. - return typeof result.message === 'string' - ? elementTextRead(result.message) - : Object.freeze({ status: 'unreadable', reason: 'surface-not-readable' } as const); -} diff --git a/src/platform-runtime-operation-host.ts b/src/platform-runtime-operation-host.ts index 1423fc3b20..5efe999939 100644 --- a/src/platform-runtime-operation-host.ts +++ b/src/platform-runtime-operation-host.ts @@ -28,7 +28,6 @@ import { createAndroidApplicationTools } from './platform-runtime-android-applic import { createLocalApplicationInteractorHost } from './platform-runtime-local-application-interactors.ts'; import { createApplicationResourceLifecycle } from './platform-runtime-application-resources.ts'; import { createSnapshotRuntimeHost } from './snapshot/snapshot-desktop-surface.ts'; -import { createElementTextRuntimeHost } from './platform-runtime-element-text-host.ts'; export function createPlatformRuntimeHost(options: { sessionsDir: string; @@ -107,7 +106,6 @@ export function createPlatformRuntimeHost(options: { ), screenRecording: createScreenRecordingRuntimeHost(), snapshot: createSnapshotRuntimeHost(), - elementText: createElementTextRuntimeHost(), localInteractors: createLocalApplicationInteractorHost(), appleApplications, androidApplications, diff --git a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts index 0c9d07a752..dc222fdd70 100644 --- a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts +++ b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts @@ -41,6 +41,12 @@ const RUNNER_TRANSPORT_METHODS: Record< }, gestureViewport: { invoke: (i) => i.gestureViewport!(), runnerCommand: 'gestureViewport' }, snapshot: { invoke: (i) => i.snapshot(), runnerCommand: 'snapshot' }, + // Runner-routed for every provider-backed device: the macOS-helper branch is reachable only + // for a local desktop/menubar surface, which a provider-owned mobile device never carries. + readTextAtPoint: { + invoke: (i) => i.readTextAtPoint!({ x: 10, y: 20 }), + runnerCommand: 'readText', + }, back: { invoke: (i) => i.back(), runnerCommand: 'backInApp' }, home: { invoke: (i) => i.home(), runnerCommand: 'home' }, setOrientation: { invoke: (i) => i.setOrientation('portrait'), runnerCommand: 'rotate' }, diff --git a/src/platforms/apple/interactor.ts b/src/platforms/apple/interactor.ts index 060e67a026..8fc8a9bcb5 100644 --- a/src/platforms/apple/interactor.ts +++ b/src/platforms/apple/interactor.ts @@ -18,13 +18,14 @@ import { type AppleRunnerProvider, } from './core/runner/runner-provider.ts'; import { toAppleTvRemoteButton } from '@agent-device/contracts/interaction'; +import type { SessionSurface } from '@agent-device/contracts/session'; import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { isMacOs, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { withMethodScope } from '../../utils/method-scope.ts'; -import type { RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; +import type { Point, RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; import type { Interactor, RunnerCallOptions, @@ -65,6 +66,12 @@ export function createAppleInteractor( close: (app) => closeIosApp(device, app, runnerOpts), screenshot: (outPath, options) => runAppleScreenshot(device, outPath, options, runnerOpts), snapshot: async (options) => await captureAppleSnapshot(device, options, runnerOpts), + // The live text at a point: helper for macOS desktop/menubar surfaces, XCTest runner for + // every other Apple leaf including a macOS app session. + readTextAtPoint: async (point, options) => + usesMacOsHelperSurface(device, options?.surface) + ? await readMacOsSurfaceTextAtPoint(point, options) + : await readRunnerTextAtPoint(device, point, options, runnerOpts), back: async (mode) => { if (isTvOsDevice(device)) { // tvOS focus-only navigation: the Menu button pops focus, not a coordinate tap. @@ -330,3 +337,36 @@ function readAppleSnapshotResult(result: Record): { : undefined, }; } + +/** Only non-app macOS surfaces are helper-read; an app session is runner-read like any leaf. */ +function usesMacOsHelperSurface(device: DeviceInfo, surface: SessionSurface | undefined): boolean { + return isMacOs(device) && surface !== undefined && surface !== 'app'; +} + +async function readMacOsSurfaceTextAtPoint( + point: Point, + options?: { appBundleId?: string; surface?: SessionSurface }, +): Promise { + const { runMacOsReadTextAction } = await import('./os/macos/helper.ts'); + const result = await runMacOsReadTextAction(point.x, point.y, { + bundleId: options?.appBundleId, + surface: options?.surface, + }); + return result.text; +} + +async function readRunnerTextAtPoint( + device: DeviceInfo, + point: Point, + options: { appBundleId?: string; signal?: AbortSignal } | undefined, + runnerOpts: RunnerCallOptions, +): Promise { + const result = await runAppleRunnerCommand( + device, + { command: 'readText', x: point.x, y: point.y, appBundleId: options?.appBundleId }, + options?.signal ? { ...runnerOpts, signal: options.signal } : runnerOpts, + ); + if (typeof result.text === 'string') return result.text; + // The runner answers `message` when it reached the element but rendered no readable text. + return typeof result.message === 'string' ? result.message : undefined; +} From 4a1cf8520dc212f3fe016742082ddca128ba3383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 13:12:36 +0200 Subject: [PATCH 8/9] refactor: migrate is to the request-bound device runtime (#1883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: migrate is to the request-bound device runtime `is` declares the shared selector capture use, admits once from exact owner facts, refuses before binding, and binds exactly once. Its capability bucket, the static HarmonyOS/Web command sets that augmented it, and `requireCommandSupported` admission for `is` are gone; `'is'` leaves the `createSelectorRuntime` capability union. Admission now runs BEFORE the direct-iOS selector fast path. ADR 0019 requires resolve -> admit -> bind before anything in a `device-runtime` command's request path reaches the device, so that query becomes a fast path *within* an admitted request rather than a way around exact-owner facts. The rule is documented once, on `createBoundSelectorRuntime`, replacing the two duplicated call-site comments `get` and `is` were each carrying. Declared behaviour change: `is` takes the active-app plan split, so the facts decide per family. On iOS `appBundleId` is the XCUITest attach target — with no tracked app the runner's own process comes to the foreground, displaces the app under test, and the capture then answers confidently about the runner's own blank screen. An iOS `is` on a session with no tracked app is now a typed SESSION_NOT_FOUND refusal carrying the `open` hint. Refusing beats displacing-and-lying. Android captures the real launcher in that state and is unchanged, which is what the platform facts already encoded. The two Apple watchOS cells move from capability-admitted-then-runner-failure to a typed unavailable refusal, the same classification snapshot, diff, and get already landed. R37 is the new parametrized cutover row. `find` keeps `createSelectorRuntime` and its `requireCommandSupported` call, so `captureData` stays optional and `captureSnapshotWithInteractor` stays: this unit is not the last selector unit. * fix(is): a failing iOS assertion fails instead of exiting zero Reverses part of #557, on thymikee's explicit instruction. `is` is an assertion: the docs state it "exits non-zero on failure". The direct-iOS fast path broke that contract — it reported a failed predicate as a completed command, so on device $ agent-device is text id=… "Wrong Expected Text" Passed: is text (exit 0) because `{ok: true, pass: false}` reaches `isCliOutput`, which renders "Passed: is " without reading `pass`. A failing assertion reported as success lets a replay run on past a broken state. Now: Error (COMMAND_FAILED): is text failed for selector id=…: expected="Wrong Expected Text" actual="Apple Account, …" (exit 1) The renderer needed no patch: a negative can no longer produce a success envelope, so it is correct by construction. Direction chosen deliberately. Making the two paths agree could have gone either way, and "an agent asked a question and got an answer" is a real argument for the other one. This follows the DOCUMENTED contract rather than merely the incumbent behaviour, and the alternative is a far larger change: a zero-exit `is` would alter every platform and path, break scripts that rely on it failing the shell, and needs its own PR, docs, and probably a major version. It is also already how `is hidden` and `is exists` behave end to end. PASSING assertion, and that arm still answers with zero captures (pinned). Only the negative falls through — what #557's own summary asked for, "preserving snapshot fallback for misses", refusing fallback only for hard failures like ambiguity. The fall-through was #557's own design, never armed: the `| null` return and the caller's `if (!payload) return null;` guard were unreachable. This makes that dead guard live. Measured on iPhone 17 (median of 9, warm daemon): predicate holds 0.14s / 0 snapshots, unchanged; predicate fails 0.25s / 1 snapshot. ~+0.11s on failing assertions only. Correctness gain beyond the envelope: the fast path evaluates a ONE-NODE tree, so `visible` cannot see the ancestor geometry a list row inherits and its negative can be wrong. Falling through re-asks the real tree and can turn a spurious negative into a pass. The #557 pin moved with its reasoning at the pin site. * fix(layering): let a cutover row state a data-only admission retirement Review blocker on #1883: R37 claimed `legacyRetirement.routeNames: ['WEB_QUERY_COMMANDS_WITH_IS', 'HARMONYOS_IS_SUPPORT']`. Neither identifier has ever existed. They satisfied the non-empty shape check while proving nothing — the vacuous registry claim AGENTS.md warns about, and a green gate that would stay green if the deletion were reverted. The cause was the model, not the row. Every `LegacyRetirementClaim` form names something that must NOT exist, which a row can always satisfy by inventing a name. `is` retired no module, route, or dispatch projection because it had none: its legacy admission was a capability bucket plus membership in two static platform command sets, so its real retirement is a DATA deletion the model could not express. Rather than patch around that with sentinels or a per-command policy file — both forbidden by the playbook — this generalizes the model. `staticCommandSets` names the sets themselves and is proven from both sides: each must still be DECLARED in production source, and must no longer list the command. A fictional set fails the first half; a skipped deletion fails the second. That is what an identifier-shaped claim cannot state. R37 now claims HARMONYOS_SUPPORTED_COMMANDS and WEB_QUERY_COMMANDS, which is the deletion it actually performed. Planted red, both halves, against the real gate: [R37 is-runtime-cutover] 2 violation(s): (is cutover row):1 — claims retired static command set 'WEB_QUERY_COMMANDS_WITH_IS', which no production source declares (is cutover row):1 — claims retired static command set 'HARMONYOS_IS_SUPPORT', which no production source declares [R37 is-runtime-cutover] 2 violation(s): src/core/capabilities.ts:59 — static command set WEB_QUERY_COMMANDS still admits is so the exact claim that shipped is now rejected by name, and so is restoring the membership it claims to have removed. Mechanism cases live with the other planted-row tests; layering goes 177 -> 181. * test(is): pin the exit-code guarantee independently of what answers the predicate Prep for the Blocker 1 retirement, which deletes `buildDirectIosIsResult` — the function the #557 reversal fixed. The reversal's guarantee must not evaporate with it, so it gets a case that does not know how the daemon decided. `is` is documented to "exit non-zero on failure". The reversal proved that at the JSON envelope; nothing pinned it at the CLI boundary, which is where the defect was actually visible (`Passed: is text`, exit 0). This asserts the CLI contract directly: a `predicate_failed` response exits 1 and never renders as passed. It survives the retirement untouched, because it asserts the outcome rather than the path. Planted red with the exact pre-#1739 envelope the shortcut produced (`{ok: true, data: {pass: false}}`): `exitSpy.calls` is `[]` — no exit call at all — so the case fails, which is the regression it exists to catch. Unpushed on purpose: the restack will carry it into the retirement cycle. * refactor(is): retire the direct-iOS selector shortcut thymikee's ruling (option b). `is` declares `device-runtime`, so its request path must reach the device only through the operations R37 declares. It did not: a simple iOS `id=`/`label=` target was answered by a direct XCUITest querySelector without any capture, ordered after admission but not executing through the seam. This is not retired because it was wrong. `wait` hypothesized that the degenerate one-node evaluation mis-answers `is visible` for off-viewport nodes, traced it through the code convincingly, then tested it on device and it did not reproduce — XCUITest's own query is conservative about visibility, so the degenerate evaluation never gets the chance. It is retired because it was an undeclared, unmeasured bypass that made R37's singularExecution claim false: the same class of untruth as the sentinel retirement names fixed in the previous commit. Declaring querySelector as a real operation instead was rejected for a concrete reason: offscreen-target-probe.ts consumes queryDirectIosSelector with a plain session and cannot take a bound operation, so declaring it now would ship it twice until Wave 5 moves the probe — the deferred-duplication shape that got get's read deferral overruled. It returns as a declared, fact-admitted, section 9-measured operation in the unit that also moves the probe. Retired: dispatchDirectIosSelectorIs, its call site, buildDirectIosIsResult, and resolveDirectIosSelectorQuery — each had exactly one caller, all on this path — plus the ResolvedDirectIosSelectorQuery type they orphaned and two imports. queryDirectIosSelector itself stays: the offscreen probe still consumes it and it remains single-copy. Latency cost, stated plainly and not softened: a held predicate on a simple iOS selector goes from ~0.14s with no capture to ~0.25s with one, measured as the median of 9 warm runs on iPhone 17. There is no fallback and no fast path. R37's comment finally describes the code: "every predicate answers from the resolved tree" was written while the shortcut existed. Its scope is now stated too, so it is not read as absolute — the Android foreground-blocker diagnostic still reaches adb on the failure path, where it cannot produce or change a verdict; that edge is pre-existing, co-owned with wait, and recorded as Wave 6 denominator work with R22's appState as its declared replacement. Seven tests lost their subject. Those whose only content was the shortcut's own mechanics are deleted; the outcome-level ones are retargeted and keep asserting what survives. --------- Co-authored-by: agent --- scripts/layering/check.ts | 4 +- .../layering/runtime-command-cutover-model.ts | 12 + .../runtime-command-cutover-policy.test.ts | 54 ++++ .../runtime-command-cutover-policy.ts | 54 ++++ .../layering/runtime-command-cutover-table.ts | 40 ++- src/__tests__/cli-exit-paths.test.ts | 36 +++ src/core/__tests__/capabilities.test.ts | 3 - .../capability-plugin-routing-parity.test.ts | 3 - src/core/capabilities.ts | 3 +- .../__tests__/parity.test.ts | 1 + src/core/command-descriptor/registry.ts | 3 +- src/daemon/__tests__/is-runtime.test.ts | 271 ++++++++++++++++++ .../handlers/__tests__/interaction.test.ts | 135 +-------- src/daemon/selector-runtime-backend.ts | 40 +-- src/daemon/selector-runtime.ts | 115 +------- .../provider-scenarios/ios-world.ts | 31 +- 16 files changed, 517 insertions(+), 288 deletions(-) create mode 100644 src/daemon/__tests__/is-runtime.test.ts diff --git a/scripts/layering/check.ts b/scripts/layering/check.ts index 7817c3b011..1795b55f06 100644 --- a/scripts/layering/check.ts +++ b/scripts/layering/check.ts @@ -36,8 +36,8 @@ // composition file; premature implementation loading and forbidden cross-boundary edges fail (R13). // - Over COMMAND-ATOMIC RUNTIME CUTOVERS: one parametrized gate reads the migrated-command // table (appstate R22, shutdown R23, boot R20, apps R21, install/deploy R24-R27, -// lifecycle R28-R31, devices R17, logs R14, network R15, record R16, snapshot R32, diff R33) -// and proves each command keeps +// lifecycle R28-R31, devices R17, logs R14, network R15, record R16, snapshot R32, diff R33, +// viewport R34, get R36, is R37 — R35 reserved for find) and proves each command keeps // exactly one platform-execution path — retired routes, admission, modules, and widened // runtime access cannot coexist with its operation-fact-derived descriptor and handler. // - Over CONTRACTS PRODUCTION SOURCE: contracts owns vocabulary only — host, process, and timer diff --git a/scripts/layering/runtime-command-cutover-model.ts b/scripts/layering/runtime-command-cutover-model.ts index 55519b615d..9674860ada 100644 --- a/scripts/layering/runtime-command-cutover-model.ts +++ b/scripts/layering/runtime-command-cutover-model.ts @@ -28,6 +28,17 @@ export type LegacyRetirementClaim = Readonly<{ daemonOnlyProviderMethods?: readonly string[]; /** `PlatformPlugin` facet keys retired with the legacy adapter. */ pluginFacetKeys?: readonly string[]; + /** + * Static platform command sets this command's admission DATA was removed from — the whole + * retirement of a command whose legacy admission was a capability bucket plus set membership, + * with no adapter module, route, or dispatch projection to name. + * + * Every other form above names something that must NOT exist, which a row can satisfy by + * inventing a name that never existed. This one is two-sided and cannot: each named set must + * still EXIST in production source, and must no longer list the command. A fictional set fails + * the first half, a skipped deletion the second. + */ + staticCommandSets?: readonly string[]; }>; /** @@ -157,6 +168,7 @@ const RETIREMENT_FORMS = [ 'daemonOnlyRouteNames', 'daemonOnlyProviderMethods', 'pluginFacetKeys', + 'staticCommandSets', ] as const satisfies readonly (keyof LegacyRetirementClaim)[]; /** diff --git a/scripts/layering/runtime-command-cutover-policy.test.ts b/scripts/layering/runtime-command-cutover-policy.test.ts index 6f5d26a942..f21f6d1780 100644 --- a/scripts/layering/runtime-command-cutover-policy.test.ts +++ b/scripts/layering/runtime-command-cutover-policy.test.ts @@ -285,3 +285,57 @@ test('every shipped row states its claims', () => { [], ); }); + +// A data-only admission retirement — a capability bucket plus static-set membership, with no +// module, route, or dispatch projection to name as gone. Both halves are planted, because the +// half that matters is the one an identifier-shaped claim cannot state: a set that never existed. +const DATA_ONLY_ROW: MigratedCommandCutover = { + ...PLANTED_ROW, + legacyRetirement: { staticCommandSets: ['WEB_QUERY_COMMANDS'] }, +}; + +test('a data-only retirement is a stated claim, so a row needs no invented identifier', () => { + assert.deepEqual(cutoverRowDefects(DATA_ONLY_ROW), []); +}); + +test('planted red: a row claiming a static command set that does not exist is rejected', () => { + assert.deepEqual( + summariesFor( + PLANTED_RULE, + [['src/core/capabilities.ts', `const WEB_QUERY_COMMANDS = ['find'];`]], + [ + { + ...DATA_ONLY_ROW, + legacyRetirement: { staticCommandSets: ['WEB_QUERY_COMMANDS_WITH_PLANTED'] }, + }, + ], + ).filter((summary) => summary.includes('static command set')), + [ + "(planted cutover row): claims retired static command set 'WEB_QUERY_COMMANDS_WITH_PLANTED', which no production source declares", + ], + ); +}); + +test('planted red: a claimed static command set that still lists the command is rejected', () => { + assert.deepEqual( + summariesFor( + PLANTED_RULE, + [['src/core/capabilities.ts', `const WEB_QUERY_COMMANDS = ['find', 'planted'];`]], + [DATA_ONLY_ROW], + ).filter((summary) => summary.includes('still admits')), + ['src/core/capabilities.ts: static command set WEB_QUERY_COMMANDS still admits planted'], + ); +}); + +test('a claimed static command set that exists and dropped the command passes', () => { + // Scoped to this column: the planted row's singular-execution claims are unrelated here and + // have their own cases above. + assert.deepEqual( + summariesFor( + PLANTED_RULE, + [['src/core/capabilities.ts', `const WEB_QUERY_COMMANDS = ['find'];`]], + [DATA_ONLY_ROW], + ).filter((summary) => summary.includes('static command set')), + [], + ); +}); diff --git a/scripts/layering/runtime-command-cutover-policy.ts b/scripts/layering/runtime-command-cutover-policy.ts index fe43354132..a48f058acb 100644 --- a/scripts/layering/runtime-command-cutover-policy.ts +++ b/scripts/layering/runtime-command-cutover-policy.ts @@ -79,6 +79,7 @@ function rowViolations( violations.push(...narrowingViolations(row, file, program)); } violations.push(...exactCallViolations(row, files, programs)); + violations.push(...staticCommandSetViolations(row, files, programs)); const sources = new Map(files.map(({ path, source }) => [path, source])); for (const check of rowChecks(row)) violations.push(...check(sources)); return violations; @@ -356,6 +357,59 @@ function isAdmissionMember( ); } +/** + * A data-only admission retirement, proven from both sides. + * + * A command whose legacy admission was a capability bucket plus membership in a static platform + * command set retires no module, route, or dispatch projection — there is no identifier to name + * as gone. Naming an invented one satisfies the non-empty shape check while proving nothing, so + * the row names the sets themselves: each must still be DECLARED in production source, and must + * no longer carry this command. + * + * The existence half is what an identifier-shaped claim cannot express. The membership half + * overlaps the automatic static-set column for `WEB`/`HARMONY`-named sets, deliberately: stating + * it here keeps the declared claim self-sufficient rather than dependent on that regex. + */ +function staticCommandSetViolations( + row: MigratedCommandCutover, + files: readonly ProductionSource[], + programs: ReadonlyMap, +): UnruledViolation[] { + const declared = row.legacyRetirement.staticCommandSets ?? []; + if (declared.length === 0) return []; + const violations: UnruledViolation[] = []; + const seen = new Set(); + for (const file of files) { + const program = programs.get(file.path); + if (!program) continue; + visitAst(program, (node) => { + const name = staticCommandSetName(node, declared); + if (name === undefined) return; + seen.add(name); + if (containsStringLiteral(node['init'], row.command)) { + violations.push(at(file, node, `static command set ${name} still admits ${row.command}`)); + } + }); + } + for (const name of declared) { + if (seen.has(name)) continue; + violations.push({ + file: `(${row.command} cutover row)`, + line: 1, + message: `claims retired static command set '${name}', which no production source declares`, + }); + } + return violations; +} + +function staticCommandSetName(node: AstNode, declared: readonly string[]): string | undefined { + if (node['type'] !== 'VariableDeclarator') return undefined; + const id = node['id'] as AstNode | undefined; + if (id?.['type'] !== 'Identifier') return undefined; + const name = String(id['name']); + return declared.includes(name) ? name : undefined; +} + function containsStringLiteral(node: unknown, expected: string): boolean { let found = false; visitAst(node, (candidate) => { diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index ecdf21ca35..1e5783e3fc 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -25,7 +25,8 @@ import { retiredDispatchProjectionViolations } from './runtime-command-cutover-d * A row id is a report heading, so it must be unique across every stack that adds rows here. * `cutoverTableDefects` rejects a duplicate; lifecycle starts at R28 after the accepted * shutdown, install/deploy, and application-lifecycle allocations. Snapshot starts at R32; - * diff follows at R33, viewport at R34, and get at R36 (R35 is reserved for find). + * diff follows at R33, viewport at R34, get at R36, and is at R37. R35 stays reserved for + * find, whose cutover is deferred behind the Wave 5 `focus`/`type` surfaces. */ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ { @@ -548,6 +549,43 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ }, }, }, + { + rule: 'R37 is-runtime-cutover', + command: 'is', + subject: 'element predicate', + tier: 'request-scoped', + execution: 'device-runtime', + // `is` retired no module, route, or dispatch projection — it had none. Its whole legacy + // admission was the capability bucket (rejected by this row's automatic descriptor column) + // plus membership in these two static sets, which is a DATA deletion. Naming the sets proves + // it from both sides: each must still be declared in production source and must no longer + // list `is`, so neither an invented name nor a skipped deletion can satisfy it. + legacyRetirement: { + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS', 'WEB_QUERY_COMMANDS'], + }, + runtimeTypeNames: ['SnapshotRuntimeOperations'], + operations: { names: ['captureSnapshot', 'captureSnapshotWithoutActiveApp'] }, + singularExecution: { + routes: ['dispatchIsViaRuntime'], + operations: ['captureSnapshot', 'captureSnapshotWithoutActiveApp'], + // `is` executes through the shared selector seam, so its capture owners are the SAME + // selectors `snapshot`/`diff`/`get` count. It declares no operation of its own: every + // predicate answers from the resolved tree, so `readTextAtPoint` stays R36's alone. + // + // Scope, stated so this is not read as absolute: the claim covers how a predicate is + // EXECUTED. Since the direct-iOS selector shortcut retired, the bound capture is the only + // thing that answers one. It does NOT claim the route makes no other device call — the + // Android foreground-blocker diagnostic still reaches adb through + // `platforms/android/app-lifecycle.ts`, on the FAILURE path only, where it can enrich an + // already-failed response's message but can never produce or change a verdict. That edge + // is pre-existing, co-owned with `wait`, and recorded as Wave 6 denominator work; R22's + // `appState` is its declared replacement. + operationOwners: { + captureSnapshot: ['selectActiveAppSnapshot'], + captureSnapshotWithoutActiveApp: ['selectSnapshotWithoutActiveApp'], + }, + }, + }, { rule: 'R34 viewport-runtime-cutover', command: 'viewport', diff --git a/src/__tests__/cli-exit-paths.test.ts b/src/__tests__/cli-exit-paths.test.ts index 4c97cf77e5..73ceb8e524 100644 --- a/src/__tests__/cli-exit-paths.test.ts +++ b/src/__tests__/cli-exit-paths.test.ts @@ -222,3 +222,39 @@ test('a --debug failure caps the daemon-log-tail dump instead of printing it unb 'expected the byte cap to drop the oldest lines, not just the 200-line cap', ); }); + +// The end-to-end half of `is`'s documented contract: "is evaluates UI predicates against a +// selector expression and exits non-zero on failure" (website/docs/docs/commands.md). +// +// This deliberately does NOT know how the daemon decided. It was written when the direct-iOS +// shortcut answered some predicates itself and returned `{ok: true, pass: false}`, which the CLI +// rendered as `Passed: is text` with exit 0 (#1739). The shortcut is retired and every predicate +// now answers from the bound capture, so the guarantee is structural rather than guard-based — +// and this case survives that change untouched, because a failed assertion must exit non-zero +// whatever produced the failure. +test('a failed `is` predicate exits non-zero, whatever answered it', async () => { + const restoreEnv = installIsolatedCliTestEnv(); + const exitSpy = installExitSpy(); + const stderr = captureStderr(); + const sendToDaemon = async (): Promise => ({ + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'is text failed for selector id=greeting: expected="Welcome" actual="Goodbye"', + details: { command: 'is', reason: 'predicate_failed', predicate: 'text' }, + }, + }); + + try { + await runCli(['is', 'text', 'id=greeting', 'Welcome'], { sendToDaemon }); + } finally { + stderr.restore(); + exitSpy.restore(); + restoreEnv(); + } + + assert.deepEqual(exitSpy.calls, [1]); + const output = stderr.read(); + assert.ok(output.includes('COMMAND_FAILED'), 'expected the typed failure on stderr'); + assert.ok(!output.includes('Passed'), 'a failed assertion must never render as passed'); +}); diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index cf7cb28787..0b4eb5f5cf 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -232,7 +232,6 @@ test('macOS supports the Apple runner interaction core but excludes mobile-only 'find', 'focus', 'get', - 'is', 'longpress', 'logs', 'perf', @@ -306,7 +305,6 @@ test('Linux supports desktop interaction commands and blocks mobile/unsupported 'focus', 'get', 'home', - 'is', 'longpress', 'press', 'screenshot', @@ -334,7 +332,6 @@ test('web supports only the initial browser interaction slice', () => { 'find', 'get', 'hover', - 'is', 'press', 'record', 'screenshot', diff --git a/src/core/__tests__/capability-plugin-routing-parity.test.ts b/src/core/__tests__/capability-plugin-routing-parity.test.ts index e2f8375ec3..3abb51df82 100644 --- a/src/core/__tests__/capability-plugin-routing-parity.test.ts +++ b/src/core/__tests__/capability-plugin-routing-parity.test.ts @@ -180,11 +180,9 @@ const HARMONYOS_SUPPORTED_COMMANDS_REF = new Set([ 'fill', 'find', 'focus', - 'get', 'home', 'gesture', 'keyboard', - 'is', 'longpress', 'press', 'screenshot', @@ -271,7 +269,6 @@ test('HarmonyOS static capabilities omit runtime-backed command admissions', () 'focus', 'gesture', 'home', - 'is', 'keyboard', 'longpress', 'perf', diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index e63cc1f59c..2a5e4c2aa4 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -47,7 +47,6 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'home', 'gesture', 'keyboard', - 'is', 'longpress', 'press', 'scroll', @@ -56,7 +55,7 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'type', 'wait', ]); -const WEB_QUERY_COMMANDS = ['audio', 'find', 'is', 'wait'] as const; +const WEB_QUERY_COMMANDS = ['audio', 'find', 'wait'] as const; const WEB_INTERACTION_COMMANDS = [ 'click', 'fill', diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index 69e6d6d25e..1f5c328638 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -60,6 +60,7 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.get, PUBLIC_COMMANDS.install, PUBLIC_COMMANDS.installFromSource, + PUBLIC_COMMANDS.is, PUBLIC_COMMANDS.logs, PUBLIC_COMMANDS.network, PUBLIC_COMMANDS.open, diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 74dbdc7d60..cf79556c7f 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -1203,10 +1203,9 @@ export const RAW_COMMAND_DESCRIPTORS = [ recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'interaction', refFrameEffect: 'preserve' }, - capability: ALL_DEVICE_COMMAND_CAPABILITY, timeoutPolicy: postActionObservationTimeoutPolicy('is', PRESERVE_DAEMON_TIMEOUT_POLICY), batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: selectorCaptureRuntimePlanUses }, }, // -- generic (route: generic) -- diff --git a/src/daemon/__tests__/is-runtime.test.ts b/src/daemon/__tests__/is-runtime.test.ts new file mode 100644 index 0000000000..add368a0ed --- /dev/null +++ b/src/daemon/__tests__/is-runtime.test.ts @@ -0,0 +1,271 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { SnapshotResult } from '@agent-device/contracts/platform'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import { + makeAndroidSession, + makeIosAppSession, + makeIosSession, +} from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inventory-gateways.ts'; +import type { DaemonRequest } from '../types.ts'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; + +const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn() })); + +vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => { + const actual = + await importOriginal(); + return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; +}); + +import { dispatchIsViaRuntime } from '../selector-runtime.ts'; + +beforeEach(() => { + mockRunAppleRunnerCommand.mockReset(); + mockRunAppleRunnerCommand.mockResolvedValue({}); +}); + +// `is` answers every one of its seven predicates from the resolved capture — `isCommand` never +// reaches `backend.readText`. So its whole platform execution is the request-bound capture, and +// these cases bind at `inspectFacts` / `bindDevice`, never at `core/dispatch.ts`. + +const unavailableCapture = { available: false, reason: 'unsupported-device-kind' } as const; +const activeAppRequired = { available: false, reason: 'owner-capability-missing' } as const; + +/** One resolvable button, so a predicate has something real to answer about. */ +function buttonSnapshot(): SnapshotResult { + return { + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + ], + backend: 'android', + }; +} + +function isRequest(session: string, positionals: readonly string[]): DaemonRequest { + return { token: 't', session, command: 'is', positionals: [...positionals], flags: {} }; +} + +test('an admitted is inspects once, binds once, and answers through the bound capture', async () => { + const fixture = selectorCaptureFixture({ snapshot: () => buttonSnapshot() }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-bound', makeAndroidSession('is-bound', { appBundleId: 'com.example.app' })); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-bound', ['visible', 'id=auth_continue']), + sessionName: 'is-bound', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(true); + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); + expect(fixture.captures.length).toBeGreaterThan(0); +}); + +test('an unavailable capture fact refuses before any bind', async () => { + // The watchOS sentinel shape: capability-supported today, no snapshot backend at the owner. + const fixture = selectorCaptureFixture({ capture: unavailableCapture }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-refused', makeAndroidSession('is-refused', { appBundleId: 'com.a' })); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-refused', ['visible', 'id=auth_continue']), + sessionName: 'is-refused', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(false); + // The inspection is what makes this a typed admission refusal rather than a runtime failure: + // exact owner facts were read once, side-effect-free, and nothing bound or captured after. + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([]); + expect(fixture.captures).toEqual([]); +}); + +// The correctness fix this unit declares. On iOS `appBundleId` is the XCUITest attach target: +// with no tracked app the runner's own process comes to the foreground, DISPLACES the app under +// test, and the capture then answers confidently about the runner's own blank screen. Refusing +// beats displacing-and-lying. Android captures the real launcher in the same state, and the +// platform facts already encode that asymmetry — so `is` asks the facts rather than branching. +test('an iOS session with no tracked app is refused with the open hint, not answered from a displaced capture', async () => { + const fixture = selectorCaptureFixture({ + withoutActiveApp: activeAppRequired, + snapshot: () => buttonSnapshot(), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-no-app', makeIosSession('is-no-app')); + + const response = await withTestDeviceInventory( + {}, + async () => + await dispatchIsViaRuntime({ + req: isRequest('is-no-app', ['visible', 'id=auth_continue']), + sessionName: 'is-no-app', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ); + + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error?.code).toBe('SESSION_NOT_FOUND'); + expect(response.error?.message).toMatch(/requires an active app session/); + } + expect(fixture.binds).toEqual([]); + expect(fixture.captures).toEqual([]); +}); + +test('an iOS session WITH a tracked app still answers, so the refusal is the plan split and not an iOS ban', async () => { + const fixture = selectorCaptureFixture({ + withoutActiveApp: activeAppRequired, + snapshot: () => buttonSnapshot(), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-with-app', makeIosAppSession('is-with-app')); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-with-app', ['visible', 'label=Continue']), + sessionName: 'is-with-app', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(true); + expect(fixture.binds).toEqual([IOS_SIMULATOR]); +}); + +test('an Android session with no tracked app proceeds, because the owner advertises the without-active-app capture', async () => { + const fixture = selectorCaptureFixture({ snapshot: () => buttonSnapshot() }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-android-no-app', makeAndroidSession('is-android-no-app')); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-android-no-app', ['visible', 'id=auth_continue']), + sessionName: 'is-android-no-app', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(true); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); +}); + +// ADR 0019: a refused request reaches the device not at all. This used to guard the direct-iOS +// selector query, which could answer a simple `id=` target without a capture; that shortcut is +// retired, so the runner assertion below now proves the stronger property — on an unavailable +// fact, `is` makes no device call by any route. +test('a refused request reaches the device by no route at all', async () => { + const fixture = selectorCaptureFixture({ capture: unavailableCapture }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-direct-refused', makeIosAppSession('is-direct-refused')); + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + nodes: [ + { + index: 0, + type: 'Button', + label: 'Pickup', + identifier: 'shipping-pickup', + selected: true, + rect: { x: 126, y: 555, width: 75, height: 38 }, + enabled: true, + hittable: true, + }, + ], + }); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-direct-refused', ['selected', 'id="shipping-pickup"']), + sessionName: 'is-direct-refused', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(false); + expect(mockRunAppleRunnerCommand).not.toHaveBeenCalled(); + expect(fixture.binds).toEqual([]); +}); + +// `is` is an assertion: it "exits non-zero on failure" (website/docs/docs/commands.md). A +// direct-iOS shortcut used to answer some predicates itself and reported a failed one as a +// completed command — `is text id=… "Wrong Expected Text"` printed `Passed: is text` and exited 0 +// on device (#1739). That shortcut is retired, so the guarantee is now structural rather than +// guard-based: the bound capture is the only thing that answers a predicate, and `isCommand` +// raises COMMAND_FAILED when one fails. +// +// The CLI half — that such a response actually exits non-zero — lives in +// `src/__tests__/cli-exit-paths.test.ts`, at a layer that does not know how the daemon decided. +test('a failing predicate answers COMMAND_FAILED from the bound capture', async () => { + const fixture = selectorCaptureFixture({ + snapshot: () => ({ + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Apple Account', + identifier: 'account_row', + rect: { x: 10, y: 20, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + ], + backend: 'xctest', + }), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-direct-false', makeIosAppSession('is-direct-false')); + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + text: 'Apple Account', + nodes: [ + { + index: 0, + type: 'Button', + label: 'Apple Account', + identifier: 'account_row', + rect: { x: 10, y: 20, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + ], + }); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-direct-false', ['text', 'id=account_row', 'Wrong Expected Text']), + sessionName: 'is-direct-false', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + // A failed assertion is a failed command on every other path and in the docs; it is one here. + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error?.code).toBe('COMMAND_FAILED'); + expect(response.error?.details?.reason).toBe('predicate_failed'); + } + // The bound capture is what answered it. + expect(fixture.captures.length).toBeGreaterThan(0); +}); diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index b34583f8d8..4b30900b9c 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -513,69 +513,14 @@ test('is visible recaptures web snapshots when cached nodes may lack rects', asy }); }); -test('is selected simple iOS id selector uses runner query without snapshot', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'is-selected-ios-direct-selector'; - sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockRunAppleRunnerCommand.mockResolvedValue({ - found: true, - text: 'Pickup', - nodes: [ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Pickup', - identifier: 'shipping-pickup', - selected: true, - rect: { x: 126, y: 555, width: 75, height: 38 }, - enabled: true, - hittable: true, - }, - ], - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['selected', 'id="shipping-pickup"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(mockRunAppleRunnerCommand).toHaveBeenCalledWith( - expect.anything(), - { - command: 'querySelector', - selectorKey: 'id', - selectorValue: 'shipping-pickup', - appBundleId: 'com.example.app', - }, - expect.anything(), - ); - expect(mockDispatch).not.toHaveBeenCalledWith( - expect.anything(), - 'snapshot', - expect.anything(), - expect.anything(), - expect.anything(), - ); - if (response?.ok) { - expect(response.data?.predicate).toBe('selected'); - expect(response.data?.pass).toBe(true); - } - const recorded = sessionStore.get(sessionName)?.actions.at(-1); - expect(recorded?.result?.selectorChain).toEqual(['id="shipping-pickup"']); -}); - -test('is simple iOS selector returns false directly when runner predicate fails', async () => { +// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots +// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero +// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit +// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; +// the shortcut is now retired outright, so the bound capture answers every predicate and this is +// simply what `is` does. The assertion below is unchanged across both edits because it was always +// about the OUTCOME, not about which path produced it. +test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { const sessionStore = makeSessionStore(); const sessionName = 'is-selected-ios-direct-selector-false'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); @@ -608,67 +553,15 @@ test('is simple iOS selector returns false directly when runner predicate fails' ...getRuntimeBindings(), }); - expect(response?.ok).toBe(true); - expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0); - if (response?.ok) { - expect(response.data?.predicate).toBe('selected'); - expect(response.data?.pass).toBe(false); + // The session snapshot has no `id=submit`, so the bound capture reports the typed selector + // failure. Nothing can report a failed assertion as a completed command. + expect(response?.ok).toBe(false); + expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); + if (response?.ok === false) { + expect(response.error?.code).toBe('COMMAND_FAILED'); } }); -test('is simple iOS selector falls back to snapshot while gesture stabilization is pending', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'is-selected-ios-stabilizing'; - const session = makeIosSession(sessionName, { appBundleId: 'com.example.app' }); - session.postGestureStabilization = { action: 'swipe', positionals: [], markedAt: Date.now() }; - sessionStore.set(sessionName, session); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Pickup', - identifier: 'shipping-pickup', - selected: true, - rect: { x: 126, y: 555, width: 75, height: 38 }, - enabled: true, - hittable: true, - }, - ], - backend: 'xctest', - }; - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['selected', 'id="shipping-pickup"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(mockRunAppleRunnerCommand).not.toHaveBeenCalled(); - expect(mockDispatch.mock.calls.some((call) => call[1] === 'snapshot')).toBe(true); -}); - test('is visible passes for list text that inherits viewport visibility from an ancestor', async () => { const sessionStore = makeSessionStore(); const sessionName = 'visible-list-item'; diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 6b88b659db..7c0158477f 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -2,7 +2,7 @@ import type { AgentDeviceBackend, BackendSnapshotResult } from '../backend.ts'; import { resolveTargetDevice } from '../core/dispatch.ts'; import { createAgentDevice } from '../runtime.ts'; import { isMacOs, isApplePlatform, publicPlatformString } from '@agent-device/kernel/device'; -import { noActiveSessionError, requireCommandSupported } from './handlers/response.ts'; +import { noActiveSessionError } from './handlers/response.ts'; import type { SnapshotState, SnapshotNode } from '@agent-device/kernel/snapshot'; import { findNodeByLabel } from '../core/snapshot-node-lookup.ts'; import { runAppleRunnerCommand } from '../platforms/apple/core/runner/runner-client.ts'; @@ -95,10 +95,17 @@ async function resolveSelectorRuntimeDevice( } /** - * A migrated selector command's runtime: facts-first admission, exactly one binding, and a - * backend whose every capture goes through the bound operation. A sibling unit migrates by - * naming its command here instead of passing a `capability` to {@link createSelectorRuntime}; - * nothing else in this module or `selector-capture-runtime.ts` needs to change. + * THE selector runtime: facts-first admission, exactly one binding, and a backend whose every + * capture goes through the bound operation. Since `is` (R37) there is no other one — the legacy + * capability-admitted `createSelectorRuntime` and its `requireCommandSupported` call were its + * last consumer and retired with it, so a selector command cannot reach the device on a + * capability bucket even by mistake. + * + * ADR 0019 §6: a `device-runtime` command reaches the device only after resolve -> admit -> + * bind, so THIS CALL COMES FIRST in its route — ahead of every shortcut, including the + * direct-iOS selector query that answers some targets without a capture. That query is a fast + * path *within* an admitted request, never a way around exact-owner facts or the one-binding + * invariant. `get` (R36) and `is` (R37) both order it this way. */ export async function createBoundSelectorRuntime( params: SelectorRuntimeParams, @@ -125,29 +132,6 @@ export async function createBoundSelectorRuntime( }; } -/** - * The legacy capability-admitted selector runtime, for the selector commands whose ADR 0019 - * unit has not landed. The union narrows as each one migrates, and the last selector unit - * deletes this function together with its `requireCommandSupported` call. - */ -export async function createSelectorRuntime( - params: SelectorRuntimeParams, - options: { requireSession: boolean; capability: 'is' }, -): Promise { - const resolved = await resolveSelectorRuntimeDevice(params, options.requireSession); - if (!resolved.ok) return resolved; - const unsupported = requireCommandSupported(options.capability, resolved.device); - if (unsupported) return { ok: false, response: unsupported }; - return { - ok: true, - runtime: createSelectorRuntimeForDevice({ - ...params, - session: resolved.session, - device: resolved.device, - }), - }; -} - function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDeviceBackend { // The bound operation is the ONLY element read. Both consumers of the shared backend read — // `get text` and read-only `find … get text` — construct a bound backend, so there is no second diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index ea53978f2a..1f255ecb21 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -17,9 +17,7 @@ import { checkIsArgs, checkWaitText, checkFindArgs, - evaluateIsPredicate, isReadOnlyFindAction, - type IsPredicate, } from '@agent-device/selectors'; import { refSnapshotFlagGuardResponse } from './handlers/interaction-flags.ts'; import { parseVersionedRefPositional } from './handlers/interaction-touch-targets.ts'; @@ -49,7 +47,6 @@ import { import { isSessionRecording } from './session-script-publication-capability.ts'; import { createBoundSelectorRuntime, - createSelectorRuntime, createSelectorRuntimeForDevice, type SelectorRuntimeParams, } from './selector-runtime-backend.ts'; @@ -67,15 +64,6 @@ type DirectIosSelectorFallbackResult = | DirectIosSelectorErrorResult | null; -type ResolvedDirectIosSelectorQuery = - | { - session: SessionState; - selector: DirectIosSelectorTarget; - result: DirectIosSelectorQueryResult; - } - | DirectIosSelectorErrorResult - | null; - export async function dispatchFindReadOnlyViaRuntime( params: SelectorRuntimeParams, ): Promise { @@ -234,26 +222,19 @@ export async function dispatchIsViaRuntime( checked.hint ? { hint: checked.hint } : undefined, ); } - const { predicate, expectedText } = checked; - const split = { selectorExpression: checked.selectorExpression }; - // ADR 0012 decision 3 / #1349: recording and a guarded replay dispatch both - // require the snapshot path — evidence and the post-resolution identity - // guard are computed from the resolution tree. + const { predicate, selectorExpression, expectedText } = checked; + // ADR 0012 decision 3 / #1349: a guarded replay dispatch resolves through the snapshot path so + // the post-resolution identity guard runs against the resolution tree. const replayTargetGuard = req.internal?.replayTargetGuard; - const recordingSession = isSessionRecording(params.sessionStore.get(params.sessionName)); - if (!replayTargetGuard && !recordingSession) { - const directResponse = await dispatchDirectIosSelectorIs( - params, - predicate as IsPredicate, - split.selectorExpression, - expectedText, - ); - if (directResponse) return directResponse; - } - const resolvedRuntime = await createSelectorRuntime(params, { + // ADR 0019: `is` declares `device-runtime`, so its request path reaches the device ONLY through + // the operations R37 declares. Every predicate — including the simple iOS `id=` selector a + // direct runner query used to answer without a capture — resolves through the bound capture. + // Admission before a bypass is not the same as executing through the seam, so the bypass is + // gone rather than merely ordered after admission. + const resolvedRuntime = await createBoundSelectorRuntime(params, { requireSession: true, - capability: 'is', + command: 'is', }); if (!resolvedRuntime.ok) return resolvedRuntime.response; @@ -261,8 +242,8 @@ export async function dispatchIsViaRuntime( const result = await resolvedRuntime.runtime.selectors.is({ session: params.sessionName, requestId: req.meta?.requestId, - predicate: predicate as IsPredicate, - selector: split.selectorExpression, + predicate, + selector: selectorExpression, expectedText, expectedResolvedTarget: replayTargetGuard, }); @@ -322,7 +303,7 @@ export async function dispatchWaitViaRuntime( mintedGeneration: versionedRef.generation, }); } - // Wait builds its runtime directly (no createSelectorRuntime), so the consumed-snapshot slot + // Wait builds its runtime directly (no createBoundSelectorRuntime), so the consumed-snapshot slot // must be initialized here too or sessionless waits have nowhere to report the capture from. params.consumedSnapshot ??= {}; const execute = async () => { @@ -376,38 +357,6 @@ function readRecordedResolutionTarget( return { node: node as SnapshotNode, preActionNodes: preActionNodes as SnapshotNode[] }; } -async function dispatchDirectIosSelectorIs( - params: SelectorRuntimeParams, - predicate: IsPredicate, - selectorExpression: string, - expectedText: string, -): Promise { - if (predicate === 'hidden') return null; - const directQuery = await resolveDirectIosSelectorQuery(params, selectorExpression); - if (isDirectIosSelectorErrorResult(directQuery)) return directQuery.response; - if (!directQuery?.result.found || !directQuery.result.node) return null; - - const payload = - predicate === 'exists' - ? { - predicate, - pass: true, - selector: directQuery.selector.raw, - matches: 1, - selectorChain: [directQuery.selector.raw], - } - : buildDirectIosIsResult( - predicate, - expectedText, - directQuery.selector.raw, - directQuery.session, - directQuery.result.node, - ); - if (!payload) return null; - recordIfSession(params.sessionStore, params.sessionName, params.req, payload); - return { ok: true, data: stripSelectorChain(payload) }; -} - async function dispatchDirectIosSelectorWait( params: SelectorRuntimeParams & { session: SessionState | undefined; @@ -439,19 +388,6 @@ async function dispatchDirectIosSelectorWait( ); } -async function resolveDirectIosSelectorQuery( - params: SelectorRuntimeParams, - selectorExpression: string, -): Promise { - const session = params.sessionStore.get(params.sessionName); - const selector = readSimpleIosSelectorTarget({ session, selectorExpression }); - if (!session || !selector) return null; - const result = await queryDirectIosSelectorOrFallback(params, session, selector); - if (isDirectIosSelectorErrorResult(result)) return result; - if (!result) return null; - return { session, selector, result }; -} - /** * The single querySelector client for the local XCTest runner: a live, * tree-independent read (and its found/text/node shape) for exactly one @@ -506,34 +442,11 @@ async function queryDirectIosSelectorOrFallback( } function isDirectIosSelectorErrorResult( - result: DirectIosSelectorFallbackResult | ResolvedDirectIosSelectorQuery, + result: DirectIosSelectorFallbackResult, ): result is DirectIosSelectorErrorResult { return result !== null && 'kind' in result && result.kind === 'error'; } -function buildDirectIosIsResult( - predicate: Exclude, - expectedText: string, - selector: string, - session: SessionState, - node: SnapshotNode, -): Record | null { - const result = evaluateIsPredicate({ - predicate, - node, - nodes: [node], - expectedText, - platform: session.device.platform, - }); - return { - predicate, - pass: result.pass, - selector, - ...(predicate === 'text' ? { text: result.actualText } : {}), - selectorChain: [selector], - }; -} - function readDirectIosSelectorNode(data: Record): SnapshotNode | undefined { const nodes = data.nodes; if (!Array.isArray(nodes)) return undefined; diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index 5705e94118..5b3cc23282 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -142,31 +142,12 @@ export async function createIosSettingsWorld(): Promise { }, result: { transformed: true }, }, - { - command: 'ios.runner.querySelector', - deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - platform: 'apple', - request: { - command: 'querySelector', - selectorKey: 'label', - selectorValue: 'General', - appBundleId: 'com.apple.Preferences', - }, - result: { - found: true, - nodes: [ - { - index: 0, - type: 'XCUIElementTypeCell', - label: 'General', - identifier: 'General', - rect: { x: 16, y: 100, width: 360, height: 44 }, - enabled: true, - hittable: true, - }, - ], - }, - }, + // `is visible label=General` answered from a direct `querySelector` here until R37 retired + // that shortcut; it now resolves through the bound capture like every other predicate, so it + // consumes a snapshot and issues no runner query at all. The second snapshot is + // `find attrs by label`. This transcript is the scripted proof that the bypass is gone: an + // unexpected `querySelector` would fail the scenario rather than pass unnoticed. + runnerSnapshot(), runnerSnapshot(), { command: 'ios.runner.findText', From 7d3e5e6e48a4ae5e2f41945a5b8d892c7573acff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 20 Aug 2026 13:55:20 +0200 Subject: [PATCH 9/9] fix(contracts): a falsely advertised element read fails as a contract bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An owner whose facts advertised `readTextAtPoint` but whose interactor cannot perform it was reported as `{ status: 'unreadable', reason: 'surface-not-readable' }`. That put a contract violation inside the closed reason set that licenses falling back to the captured tree, so `get text` answered from potentially stale snapshot text precisely because the runtime lied about itself. ADR 0019 §2 requires the mismatch to fail as `runtime-contract-invalid`; it now throws. Removing the only producer of `surface-not-readable` made that reason dead: no path can reach it, since an interactor that HAS the read maps a blank or absent answer to `no-text-at-point` via `elementTextRead`. Dropped from the union, its consumer switch arm, and both test lists. `classifiedFallbackReason`'s `never` arm stays — it is what makes adding a reason a compile error rather than a silent untyped fallback. Deduplication found while auditing the change: - `invalidRuntimeContract` was module-private in `platform-runtime.ts`. It now owns its own module so both runtime modules share one construction. It is deliberately not exported through the platform facade: that facade must stay exhaustive over its sources, which would make this a public symbol with no external consumer. - The 8-field runner execution projection was written out three times (`snapshot-runtime-capture-input.ts`, `interaction-read.ts`, `screenshot-runtime.ts`). One `runtimeExecutionFromContext` now serves all three; `screenshotExecutionFromContext` keeps its name and delegates, since `ScreenshotRuntimeExecution` and `SnapshotRuntimeExecution` are the same type. Dropping a field here silently strips request id, log/trace paths, XCUITest overrides, or runner lease context — an operation that still answers but runs unconfigured, which is exactly the defect the wait unit hit as a P1. Red before green: with the old guard restored the new regression fails with "Missing expected rejection" — the call resolves instead of throwing, which is the silent degradation it exists to forbid. --- .../src/element-text-runtime.test.ts | 35 ++++++++++++-- .../contracts/src/element-text-runtime.ts | 18 ++++--- packages/contracts/src/platform-runtime.ts | 8 +--- .../contracts/src/runtime-contract-error.ts | 21 ++++++++ .../__tests__/interaction-read.test.ts | 22 ++++----- src/daemon/handlers/interaction-read.ts | 14 +----- src/daemon/screenshot-runtime.ts | 20 ++++---- src/daemon/snapshot-runtime-capture-input.ts | 48 ++++++++++++++----- 8 files changed, 123 insertions(+), 63 deletions(-) create mode 100644 packages/contracts/src/runtime-contract-error.ts diff --git a/packages/contracts/src/element-text-runtime.test.ts b/packages/contracts/src/element-text-runtime.test.ts index 143148c2dd..f9804d61f6 100644 --- a/packages/contracts/src/element-text-runtime.test.ts +++ b/packages/contracts/src/element-text-runtime.test.ts @@ -1,10 +1,14 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; import { + bindElementTextRuntime, elementTextRead, type ElementTextReadOutcome, type ElementTextUnreadableReason, } from './element-text-runtime.ts'; +import type { Interactor } from './interactor-types.ts'; +import type { DeviceInfo } from '@agent-device/kernel/device'; /** * The reasons this suite exercises. Kept local on purpose: exhaustiveness is enforced at the @@ -12,10 +16,7 @@ import { * so a second exported runtime list would be an unconsumed parallel source of truth that could * silently drift. The annotation is what ties this list back to the union. */ -const UNREADABLE_REASONS: readonly ElementTextUnreadableReason[] = [ - 'no-text-at-point', - 'surface-not-readable', -]; +const UNREADABLE_REASONS: readonly ElementTextUnreadableReason[] = ['no-text-at-point']; /** * ADR 0019 §2 contract coverage for the preferred element-text read. @@ -68,3 +69,29 @@ test('read outcomes are frozen so a consumer cannot mutate a classification', () assert.ok(Object.isFrozen(elementTextRead('value'))); assert.ok(Object.isFrozen(elementTextRead(''))); }); + +/** + * A runtime owner whose facts advertised `readTextAtPoint` but whose interactor cannot perform + * it is a CONTRACT BUG, not a refusal. Classifying it as an unreadable reason would place it + * inside the closed set that licenses falling back to already-captured text — so the command + * would answer from a stale tree precisely because the runtime lied about itself. + * + * Reverting the guard to `{ status: 'unreadable', reason: … }` makes this test fail: the call + * resolves instead of rejecting, which is the exact silent degradation it exists to forbid. + */ +test('an advertised read with no interactor implementation fails as a contract bug', async () => { + const runtime = bindElementTextRuntime({ + device: { platform: 'ios' } as unknown as DeviceInfo, + signal: new AbortController().signal, + // An interactor with NO readTextAtPoint — the mismatch the facts promised away. + resolveInteractor: async () => ({}) as unknown as Interactor, + }); + + await assert.rejects( + () => runtime.readTextAtPoint({ point: { x: 1, y: 2 } }), + (error: unknown) => + error instanceof AppError && + error.details?.reason === 'runtime-contract-invalid' && + /advertised readTextAtPoint/.test(error.message), + ); +}); diff --git a/packages/contracts/src/element-text-runtime.ts b/packages/contracts/src/element-text-runtime.ts index 8936418f9a..abfda48730 100644 --- a/packages/contracts/src/element-text-runtime.ts +++ b/packages/contracts/src/element-text-runtime.ts @@ -1,6 +1,7 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Point } from '@agent-device/kernel/snapshot'; import type { Interactor, RunnerContext } from './interactor-types.ts'; +import { invalidRuntimeContract } from './runtime-contract-error.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; import type { SessionSurface } from './session-surface.ts'; import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; @@ -26,9 +27,7 @@ export type ReadTextAtPointInput = Readonly<{ */ export type ElementTextUnreadableReason = /** The owner queried successfully and there is nothing readable at this point. */ - | 'no-text-at-point' - /** The owner's read surface exists but declined this query (unsupported element/surface). */ - | 'surface-not-readable'; + 'no-text-at-point'; /** The closed outcome of one live element-text read. */ export type ElementTextReadOutcome = @@ -99,10 +98,15 @@ export function bindElementTextRuntime( appBundleId: input.options?.appBundleId, signal, }); - // An owner whose facts advertised the read but whose interactor has none is a runtime - // contract error surfaced as a declined read, not a silent empty answer. - if (!interactor.readTextAtPoint) { - return Object.freeze({ status: 'unreadable', reason: 'surface-not-readable' } as const); + // Facts advertised the read but the owner's interactor cannot perform it. That is a + // contract violation, not a refusal: classifying it as `surface-not-readable` would put + // it inside the closed reason set and license the caller to fall back to already-captured + // text, answering from a stale tree because the runtime lied. Fail as the contract bug it + // is (ADR 0019 §2) so no consumer can silently degrade. + if (typeof interactor.readTextAtPoint !== 'function') { + throw invalidRuntimeContract( + 'Runtime owner advertised readTextAtPoint without an interactor implementation', + ); } return elementTextRead( await interactor.readTextAtPoint(input.point, { diff --git a/packages/contracts/src/platform-runtime.ts b/packages/contracts/src/platform-runtime.ts index da85afb827..5d5f8355ff 100644 --- a/packages/contracts/src/platform-runtime.ts +++ b/packages/contracts/src/platform-runtime.ts @@ -10,6 +10,7 @@ import { AppError } from '@agent-device/kernel/errors'; import type { PlatformModuleMetadata } from './platform-module.ts'; import type { PlatformRequestScope } from './platform-runtime-host.ts'; import type { ApplicationLifecycleResourceLifecycle } from './application-lifecycle-runtime.ts'; +import { invalidRuntimeContract } from './runtime-contract-error.ts'; type RuntimeOperation = (...args: never[]) => unknown; @@ -283,10 +284,3 @@ function unsupportedRuntimeOperation(key: string, fact: RuntimeOperationUnavaila hint: fact.hint, }); } - -function invalidRuntimeContract(message: string): AppError { - return new AppError('COMMAND_FAILED', message, { - reason: 'runtime-contract-invalid', - hint: 'This is an agent-device runtime contract bug; report the selected device and command.', - }); -} diff --git a/packages/contracts/src/runtime-contract-error.ts b/packages/contracts/src/runtime-contract-error.ts new file mode 100644 index 0000000000..905227bf4f --- /dev/null +++ b/packages/contracts/src/runtime-contract-error.ts @@ -0,0 +1,21 @@ +import { AppError } from '@agent-device/kernel/errors'; + +/** + * A runtime owner whose facts advertised an operation it cannot actually perform. + * + * This is a contract bug, never a legitimate refusal. Consumers must not classify it as an + * unavailable or declined operation: doing so places it inside whatever closed reason set + * licenses a fallback, and the command then answers from stale data precisely because the + * runtime lied about itself (ADR 0019 §2). + * + * Deliberately its own module rather than an export of `platform-runtime.ts`: the façade + * re-exports that module and must stay exhaustive over it, which would make this a public + * symbol with no external consumer — the unused-export defect. Here it stays internal to + * `packages/contracts` and both runtime modules share one construction. + */ +export function invalidRuntimeContract(message: string): AppError { + return new AppError('COMMAND_FAILED', message, { + reason: 'runtime-contract-invalid', + hint: 'This is an agent-device runtime contract bug; report the selected device and command.', + }); +} diff --git a/src/daemon/handlers/__tests__/interaction-read.test.ts b/src/daemon/handlers/__tests__/interaction-read.test.ts index 794deffabb..46f808ea9d 100644 --- a/src/daemon/handlers/__tests__/interaction-read.test.ts +++ b/src/daemon/handlers/__tests__/interaction-read.test.ts @@ -99,17 +99,17 @@ describe('readTextForNode', () => { }); // ADR 0019 §2: the ONLY fallbacks are the contract's classified reasons. - it.each(['no-text-at-point', 'surface-not-readable'] as const)( - 'falls back to the captured tree for the classified reason %s', - async (reason) => { - readTextAtPoint.mockResolvedValueOnce({ status: 'unreadable', reason }); - const text = await readTextForNode({ - ...baseParams, - node: node({ type: 'textfield', value: 'snap' }), - }); - expect(text).toBe('snap'); - }, - ); + it('falls back to the captured tree for the classified reason no-text-at-point', async () => { + readTextAtPoint.mockResolvedValueOnce({ + status: 'unreadable', + reason: 'no-text-at-point', + }); + const text = await readTextForNode({ + ...baseParams, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + }); it('classifies a blank live read as no-text-at-point rather than reading blank text', async () => { readTextAtPoint.mockResolvedValueOnce(elementTextRead(' ')); diff --git a/src/daemon/handlers/interaction-read.ts b/src/daemon/handlers/interaction-read.ts index a558b035f8..2294e1beba 100644 --- a/src/daemon/handlers/interaction-read.ts +++ b/src/daemon/handlers/interaction-read.ts @@ -4,6 +4,7 @@ import type { ElementTextUnreadableReason, } from '@agent-device/contracts/platform'; import { isIosFamily } from '@agent-device/kernel/device'; +import { runtimeExecutionFromContext } from '../snapshot-runtime-capture-input.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { SessionState } from '../types.ts'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; @@ -58,16 +59,7 @@ export async function readTextForNode(params: { const outcome = await readTextAtPoint({ point: center, options: { appBundleId, surface }, - execution: { - requestId: context.requestId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }, + execution: runtimeExecutionFromContext(context), }); if (outcome.status === 'read') return outcome.text; emitDiagnostic({ @@ -92,8 +84,6 @@ function classifiedFallbackReason(reason: ElementTextUnreadableReason): string { switch (reason) { case 'no-text-at-point': return 'no_text_at_point'; - case 'surface-not-readable': - return 'surface_not_readable'; default: { const unhandled: never = reason; return unhandled; diff --git a/src/daemon/screenshot-runtime.ts b/src/daemon/screenshot-runtime.ts index 40e63ebb4a..9da78cba1e 100644 --- a/src/daemon/screenshot-runtime.ts +++ b/src/daemon/screenshot-runtime.ts @@ -18,6 +18,7 @@ import { assertSupportedScreenshotPixelDensity, readScreenshotResultMetadata, } from '../utils/screenshot-density.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; import type { DaemonCommandContext } from './context.ts'; import { captureSnapshotData } from './handlers/snapshot-capture.ts'; import { buildSnapshotState } from './snapshot-state.ts'; @@ -123,20 +124,17 @@ export async function captureScreenshotArtifact( */ type CapturedScreenshot = Readonly<{ path: string; message?: string }>; -/** Runner metadata the capture needs; cancellation comes from the request binding, not from here. */ +/** + * Runner metadata the capture needs; cancellation comes from the request binding, not from here. + * + * `ScreenshotRuntimeExecution` and `SnapshotRuntimeExecution` are the same type — both + * `Readonly>` — so the projection is shared rather + * than restated. Keeping the screenshot-facing name preserves this module's callers. + */ export function screenshotExecutionFromContext( context: DaemonCommandContext, ): ScreenshotRuntimeExecution { - return { - requestId: context.requestId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }; + return runtimeExecutionFromContext(context); } async function executeScreenshot( diff --git a/src/daemon/snapshot-runtime-capture-input.ts b/src/daemon/snapshot-runtime-capture-input.ts index 7b593195ba..3e8dd51f0a 100644 --- a/src/daemon/snapshot-runtime-capture-input.ts +++ b/src/daemon/snapshot-runtime-capture-input.ts @@ -1,5 +1,8 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import type { CaptureSnapshotInput } from '@agent-device/contracts/platform'; +import type { + CaptureSnapshotInput, + SnapshotRuntimeExecution, +} from '@agent-device/contracts/platform'; import { contextFromFlags } from './context.ts'; import type { DaemonRequest, SessionState } from './types.ts'; @@ -45,15 +48,38 @@ export function buildRuntimeCaptureInput( includeRects: params.includeRects, surface, }, - execution: { - requestId: context.requestId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }, + execution: runtimeExecutionFromContext(context), + }; +} + +/** + * Projects the runner execution metadata a platform operation needs out of a resolved command + * context. Every request-bound operation — capture and element read alike — must forward the + * SAME set: dropping a field silently strips request id, log/trace paths, XCUITest overrides, or + * runner lease context, so the operation still answers but runs unconfigured and its diagnostics + * land nowhere. One projection means a new field reaches every operation at once and cannot be + * forgotten at one call site. + */ +export function runtimeExecutionFromContext( + context: Readonly<{ + requestId?: string; + verbose?: boolean; + logPath?: string; + traceLogPath?: string; + iosXctestrunFile?: string; + iosXctestDerivedDataPath?: string; + iosXctestEnvDir?: string; + runnerLeaseContext?: SnapshotRuntimeExecution['runnerLeaseContext']; + }>, +): SnapshotRuntimeExecution { + return { + requestId: context.requestId, + verbose: context.verbose, + logPath: context.logPath, + traceLogPath: context.traceLogPath, + iosXctestrunFile: context.iosXctestrunFile, + iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, + iosXctestEnvDir: context.iosXctestEnvDir, + runnerLeaseContext: context.runnerLeaseContext, }; }