diff --git a/packages/contracts/src/element-text-runtime.test.ts b/packages/contracts/src/element-text-runtime.test.ts new file mode 100644 index 000000000..f9804d61f --- /dev/null +++ b/packages/contracts/src/element-text-runtime.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + bindElementTextRuntime, + elementTextRead, + type ElementTextReadOutcome, + type ElementTextUnreadableReason, +} from './element-text-runtime.ts'; +import type { Interactor } from './interactor-types.ts'; +import type { DeviceInfo } from '@agent-device/kernel/device'; + +/** + * The reasons this suite exercises. Kept local on purpose: exhaustiveness is enforced at the + * CONSUMER by `classifiedFallbackReason`'s `never` arm (a new reason is a compile error there), + * so a second exported runtime list would be an unconsumed parallel source of truth that could + * silently drift. The annotation is what ties this list back to the union. + */ +const UNREADABLE_REASONS: readonly ElementTextUnreadableReason[] = ['no-text-at-point']; + +/** + * ADR 0019 §2 contract coverage for the preferred element-text read. + * + * A preferred operation may fall its consumer back to the required path only through a TYPED + * reason. These tests pin that the reason set is closed and exhaustively enumerated, so a new + * reason cannot be added without a consumer having to classify it — which is what keeps the + * retired generic `catch` from creeping back as "some other failure, just fall back". + */ + +test('the outcome union is closed: every value is a read or a classified unreadable', () => { + const outcomes: readonly ElementTextReadOutcome[] = [ + elementTextRead('live value'), + ...UNREADABLE_REASONS.map((reason) => ({ status: 'unreadable', reason }) as const), + ]; + for (const outcome of outcomes) { + if (outcome.status === 'read') { + assert.equal(typeof outcome.text, 'string'); + continue; + } + assert.ok( + (UNREADABLE_REASONS as readonly string[]).includes(outcome.reason), + `unreadable outcome carries an unclassified reason: ${outcome.reason}`, + ); + } +}); + +test('a non-blank owner answer is a read that preserves the exact text', () => { + const outcome = elementTextRead(' padded value '); + assert.deepEqual(outcome, { status: 'read', text: ' padded value ' }); +}); + +// Blank is a classification, not a read: an owner answering with whitespace has said there is +// nothing at this point, and saying so by reason keeps consumers off "empty or failed?" guesswork. +for (const [label, value] of [ + ['empty string', ''], + ['whitespace', ' \n\t '], + ['undefined', undefined], + ['null', null], +] as const) { + test(`a ${label} owner answer classifies as no-text-at-point`, () => { + assert.deepEqual(elementTextRead(value), { + status: 'unreadable', + reason: 'no-text-at-point', + }); + }); +} + +test('read outcomes are frozen so a consumer cannot mutate a classification', () => { + assert.ok(Object.isFrozen(elementTextRead('value'))); + assert.ok(Object.isFrozen(elementTextRead(''))); +}); + +/** + * A runtime owner whose facts advertised `readTextAtPoint` but whose interactor cannot perform + * it is a CONTRACT BUG, not a refusal. Classifying it as an unreadable reason would place it + * inside the closed set that licenses falling back to already-captured text — so the command + * would answer from a stale tree precisely because the runtime lied about itself. + * + * Reverting the guard to `{ status: 'unreadable', reason: … }` makes this test fail: the call + * resolves instead of rejecting, which is the exact silent degradation it exists to forbid. + */ +test('an advertised read with no interactor implementation fails as a contract bug', async () => { + const runtime = bindElementTextRuntime({ + device: { platform: 'ios' } as unknown as DeviceInfo, + signal: new AbortController().signal, + // An interactor with NO readTextAtPoint — the mismatch the facts promised away. + resolveInteractor: async () => ({}) as unknown as Interactor, + }); + + await assert.rejects( + () => runtime.readTextAtPoint({ point: { x: 1, y: 2 } }), + (error: unknown) => + error instanceof AppError && + error.details?.reason === 'runtime-contract-invalid' && + /advertised readTextAtPoint/.test(error.message), + ); +}); diff --git a/packages/contracts/src/element-text-runtime.ts b/packages/contracts/src/element-text-runtime.ts new file mode 100644 index 000000000..abfda4873 --- /dev/null +++ b/packages/contracts/src/element-text-runtime.ts @@ -0,0 +1,120 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Point } from '@agent-device/kernel/snapshot'; +import type { Interactor, RunnerContext } from './interactor-types.ts'; +import { invalidRuntimeContract } from './runtime-contract-error.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; +import type { SessionSurface } from './session-surface.ts'; +import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; + +/** + * Neutral intent for one point-addressed element read. The point is already resolved from the + * node the caller matched, so the operation names no command, request, session, or CLI flag. + */ +export type ReadTextAtPointInput = Readonly<{ + point: Point; + options?: Readonly<{ appBundleId?: string; surface?: SessionSurface }>; + /** Same runner metadata a capture needs; reuses that type rather than restating it. */ + execution?: SnapshotRuntimeExecution; +}>; + +/** + * Why an owner that HAS a live read still produced no text for this point. + * + * Closed on purpose (ADR 0019 §2): a consumer may fall back to the required path only for a + * reason named here. Anything else — a runner transport failure, a helper crash, a bug — is an + * unexpected error and propagates, because silently answering from a stale captured tree after + * an unclassified failure is exactly the "generic catch fallback" the ADR forbids. + */ +export type ElementTextUnreadableReason = + /** The owner queried successfully and there is nothing readable at this point. */ + 'no-text-at-point'; + +/** The closed outcome of one live element-text read. */ +export type ElementTextReadOutcome = + | Readonly<{ status: 'read'; text: string }> + | Readonly<{ status: 'unreadable'; reason: ElementTextUnreadableReason }>; + +/** + * Normalizes a raw owner read into the closed outcome. Blank text is not a read: an owner that + * answers with whitespace has told us there is nothing at this point, and saying so by reason + * keeps every consumer off "did it fail or is it empty?" guesswork. + */ +export function elementTextRead(text: string | undefined | null): ElementTextReadOutcome { + if (typeof text !== 'string' || text.trim().length === 0) { + return Object.freeze({ status: 'unreadable', reason: 'no-text-at-point' } as const); + } + return Object.freeze({ status: 'read', text } as const); +} + +export type ElementTextRuntimeOperations = Readonly<{ + /** + * The live text an owner reads at a point, which can exceed the readable text carried by an + * already-captured snapshot node (an editable field whose value is longer than its label). + * Declared `preferred`, never `required`: every consumer's required path answers from the + * snapshot tree, so an owner without this operation still executes the command completely. + * + * Returns a closed typed outcome rather than a bare string, so a consumer never has to + * distinguish "no text here" from "the read blew up" by catching. + */ + readTextAtPoint(input: ReadTextAtPointInput): Promise; +}>; + +export type ElementTextRuntimeOperationFacts = Readonly<{ + readTextAtPoint: RuntimeOperationFact; +}>; + +export function elementTextRuntimeOperationFacts( + input: ElementTextRuntimeOperationFacts, +): ElementTextRuntimeOperationFacts { + return Object.freeze({ readTextAtPoint: input.readTextAtPoint }); +} + +/** Resolves the selected owner's interactor, exactly as the snapshot runtime does. */ +export type ElementTextInteractorResolver = ( + device: DeviceInfo, + runner: RunnerContext, +) => Promise; + +/** + * Binds the owner's live point read for the lifetime of a request binding. + * + * Rides the same `Interactor` seam `findText` uses rather than a bespoke host port: two + * operations of the same class reaching their mechanics two different ways is duplication of + * mechanism, and Wave 5/6 retires the seam for both together. + */ +export function bindElementTextRuntime( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: ElementTextInteractorResolver; + }>, +): ElementTextRuntimeOperations { + return Object.freeze({ + readTextAtPoint: async (input: ReadTextAtPointInput) => { + const signal = params.signal; + signal.throwIfAborted(); + const interactor = await params.resolveInteractor(params.device, { + ...input.execution, + appBundleId: input.options?.appBundleId, + signal, + }); + // Facts advertised the read but the owner's interactor cannot perform it. That is a + // contract violation, not a refusal: classifying it as `surface-not-readable` would put + // it inside the closed reason set and license the caller to fall back to already-captured + // text, answering from a stale tree because the runtime lied. Fail as the contract bug it + // is (ADR 0019 §2) so no consumer can silently degrade. + if (typeof interactor.readTextAtPoint !== 'function') { + throw invalidRuntimeContract( + 'Runtime owner advertised readTextAtPoint without an interactor implementation', + ); + } + return elementTextRead( + await interactor.readTextAtPoint(input.point, { + appBundleId: input.options?.appBundleId, + surface: input.options?.surface, + signal, + }), + ); + }, + }); +} diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index 6ac3c411d..45c1d8d50 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -214,12 +214,18 @@ export { captureSnapshotUse, defineUse, resolveScreenshotRuntimePlan, + resolveSelectorCaptureRuntimePlan, resolveSnapshotRuntimePlan, screenshotRuntimePlanUses, + selectorCaptureRuntimePlanUses, snapshotRuntimePlanUses, viewportRuntimeUse, } from '../platform-runtime-operations.ts'; -export type { ScreenshotRuntimePlan, SnapshotRuntimePlan } from '../platform-runtime-operations.ts'; +export type { + ScreenshotRuntimePlan, + SelectorCaptureRuntimePlan, + SnapshotRuntimePlan, +} from '../platform-runtime-operations.ts'; export type { PlatformRuntimeHost, PlatformRuntimeModule, @@ -270,6 +276,19 @@ export type { ViewportRuntimeOperationFacts, ViewportRuntimeOperations, } from '../viewport-runtime.ts'; +export { + bindElementTextRuntime, + elementTextRead, + elementTextRuntimeOperationFacts, +} from '../element-text-runtime.ts'; +export type { + ElementTextReadOutcome, + ElementTextInteractorResolver, + ElementTextRuntimeOperationFacts, + ElementTextRuntimeOperations, + ElementTextUnreadableReason, + ReadTextAtPointInput, +} from '../element-text-runtime.ts'; export type { AppStateRuntimeCommand, AppStateRuntimeCommandResult, diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index d9b0164f3..48041f3d4 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -222,6 +222,16 @@ export type Interactor = { screenshot(outPath: string, options?: ScreenshotOptions): Promise; setViewport?(width: number, height: number): Promise | void>; snapshot(options?: SnapshotOptions): Promise; + /** + * Native reading of the live text at a point, when the backend has one. Answers the text the + * owner can see right now, which can exceed what an already-captured node carries (an editable + * field whose value is longer than its label). Optional: a backend without it leaves the + * captured tree as the complete answer. + */ + readTextAtPoint?( + point: Point, + options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal }, + ): Promise; gestureViewport?(): Promise; back(mode?: BackMode): Promise; home(): Promise; diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 53f4e0d04..cb9234f4d 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -15,6 +15,7 @@ import type { ScreenRecordingRuntimeOperations } from './screen-recording-runtim import type { ScreenshotRuntimeOperations } from './screenshot-runtime.ts'; import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; +import type { ElementTextRuntimeOperations } from './element-text-runtime.ts'; import type { DeviceReadinessRuntimeHost, DeviceReadinessRuntimeOperations, @@ -47,6 +48,7 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & ScreenshotRuntimeOperations & SnapshotRuntimeOperations & ViewportRuntimeOperations & + ElementTextRuntimeOperations & DeviceReadinessRuntimeOperations & DeviceShutdownRuntimeOperations & ApplicationLifecycleRuntimeOperations; @@ -80,6 +82,31 @@ const captureSnapshotWithCustomActionsWithoutActiveAppUse = defineUse({ ], }); +/** + * The selector family's capture uses. Declared ALONGSIDE the snapshot uses above, never in place + * of them: `snapshot`/`diff` keep binding exactly what they bind today. The only difference is the + * PREFERRED element read — every selector read's required path answers from the captured tree, so + * an owner without the read still executes the command completely (ADR 0019 §2), but an owner that + * has one lets `get text` return the live value a truncated snapshot node cannot. + */ +const selectorCaptureUse = defineUse({ + required: ['captureSnapshot'], + preferred: ['readTextAtPoint'], +}); +const selectorCaptureWithoutActiveAppUse = defineUse({ + required: ['captureSnapshot', 'captureSnapshotWithoutActiveApp'], + preferred: ['readTextAtPoint'], +}); + +/** + * The selector family (`find`, `get`, `is`, `wait`) resolves targets from the plain accessibility + * capture: it exposes no `--actions` surface, so only the active-app split applies. + */ +export const selectorCaptureRuntimePlanUses = Object.freeze([ + selectorCaptureUse, + selectorCaptureWithoutActiveAppUse, +] as const); + export const snapshotRuntimePlanUses = Object.freeze([ captureSnapshotUse, captureSnapshotWithCustomActionsUse, @@ -109,30 +136,67 @@ export type SnapshotRuntimePlan = use: typeof captureSnapshotWithoutActiveAppUse; }>; +/** + * Same two `kind`s the snapshot plan uses for this split — deliberately, so the shared + * admit-then-bind path keeps ONE set of arms rather than growing a parallel dispatch — but + * carrying the selector uses, which add the preferred element read. + */ +export type SelectorCaptureRuntimePlan = + | Readonly<{ + kind: 'selector-active-app'; + operation: 'captureSnapshot'; + use: typeof selectorCaptureUse; + }> + | Readonly<{ + kind: 'selector-without-active-app'; + operation: 'captureSnapshotWithoutActiveApp'; + use: typeof selectorCaptureWithoutActiveAppUse; + }>; + +/** + * The active-app split every selector capture selects from. The selector family exposes no + * `--actions` surface, so custom actions are outside its declaration. + */ +export function resolveSelectorCaptureRuntimePlan( + input: Readonly<{ hasActiveApp: boolean }>, +): SelectorCaptureRuntimePlan { + return input.hasActiveApp + ? Object.freeze({ + kind: 'selector-active-app', + operation: 'captureSnapshot', + use: selectorCaptureUse, + }) + : Object.freeze({ + kind: 'selector-without-active-app', + operation: 'captureSnapshotWithoutActiveApp', + use: selectorCaptureWithoutActiveAppUse, + }); +} + /** Selects one owner-fact-backed capture plan from normalized command/session intent. */ export function resolveSnapshotRuntimePlan(input: { customActions: boolean; hasActiveApp: boolean; }): SnapshotRuntimePlan { - if (input.customActions) { + if (!input.customActions) { return input.hasActiveApp - ? Object.freeze({ - kind: 'custom-actions-active-app', - operation: 'captureSnapshotWithCustomActions', - use: captureSnapshotWithCustomActionsUse, - }) + ? Object.freeze({ kind: 'active-app', operation: 'captureSnapshot', use: captureSnapshotUse }) : Object.freeze({ - kind: 'custom-actions-without-active-app', - operation: 'captureSnapshotWithCustomActions', - use: captureSnapshotWithCustomActionsWithoutActiveAppUse, + kind: 'without-active-app', + operation: 'captureSnapshotWithoutActiveApp', + use: captureSnapshotWithoutActiveAppUse, }); } return input.hasActiveApp - ? Object.freeze({ kind: 'active-app', operation: 'captureSnapshot', use: captureSnapshotUse }) + ? Object.freeze({ + kind: 'custom-actions-active-app', + operation: 'captureSnapshotWithCustomActions', + use: captureSnapshotWithCustomActionsUse, + }) : Object.freeze({ - kind: 'without-active-app', - operation: 'captureSnapshotWithoutActiveApp', - use: captureSnapshotWithoutActiveAppUse, + kind: 'custom-actions-without-active-app', + operation: 'captureSnapshotWithCustomActions', + use: captureSnapshotWithCustomActionsWithoutActiveAppUse, }); } diff --git a/packages/contracts/src/platform-runtime-unavailable.test.ts b/packages/contracts/src/platform-runtime-unavailable.test.ts index e9f15472a..c250bb97b 100644 --- a/packages/contracts/src/platform-runtime-unavailable.test.ts +++ b/packages/contracts/src/platform-runtime-unavailable.test.ts @@ -31,6 +31,7 @@ test('generic unavailable binding preserves exact provider ownership and mode', network: { available: false, reason: 'owner-capability-missing' }, screenshot: { available: false, reason: 'unsupported-device-kind' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, + elementText: { available: false, reason: 'unsupported-provider-mode' }, lifecycle, }); @@ -45,6 +46,10 @@ test('generic unavailable binding preserves exact provider ownership and mode', available: false, reason: 'unsupported-device-kind', }); + assert.deepEqual(binding.facts.operations.readTextAtPoint, { + available: false, + reason: 'unsupported-provider-mode', + }); assert.deepEqual(binding.operations, {}); await binding[Symbol.asyncDispose](); }); diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 4e34870f6..7a995a6a8 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -13,6 +13,7 @@ import type { import { screenshotRuntimeOperationFacts } from './screenshot-runtime.ts'; import { snapshotRuntimeOperationFacts } from './snapshot-runtime.ts'; import { viewportRuntimeOperationFacts } from './viewport-runtime.ts'; +import { elementTextRuntimeOperationFacts } from './element-text-runtime.ts'; /** * A runtime-contract helper for provider ownership gaps. It never assigns lifecycle semantics: @@ -28,25 +29,20 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{ screenshot: RuntimeOperationUnavailability; snapshot?: RuntimeOperationUnavailability; viewport: RuntimeOperationUnavailability; + elementText: RuntimeOperationUnavailability; readiness?: RuntimeOperationUnavailability; shutdown?: RuntimeOperationUnavailability; lifecycle: ApplicationLifecycleOperationFacts; }>; -type FrozenUnavailablePlatformRuntimeFacts = Readonly<{ - appLog: RuntimeOperationUnavailability; - apps: RuntimeOperationUnavailability; - appDeployment: RuntimeOperationUnavailability; - appState: RuntimeOperationUnavailability; - network: RuntimeOperationUnavailability; - screenRecording: RuntimeOperationUnavailability; - screenshot: RuntimeOperationUnavailability; - snapshot: RuntimeOperationUnavailability; - viewport: RuntimeOperationUnavailability; - readiness: RuntimeOperationUnavailability; - shutdown: RuntimeOperationUnavailability; - lifecycle: ApplicationLifecycleOperationFacts; -}>; +/** + * The same cells with every optional one resolved. Derived from the input type rather than + * restated, so a new cell cannot be added to one and forgotten in the other. + */ +type FrozenUnavailablePlatformRuntimeFacts = Readonly< + Required> & + Readonly<{ lifecycle: ApplicationLifecycleOperationFacts }> +>; export function createUnavailablePlatformRuntimeBinding( device: DeviceInfo, @@ -77,6 +73,7 @@ export function createUnavailablePlatformRuntimeFacts( screenshot, snapshot, viewport, + elementText, readiness, shutdown, lifecycle, @@ -109,6 +106,7 @@ export function createUnavailablePlatformRuntimeFacts( withoutActiveApp: snapshot, }), ...viewportRuntimeOperationFacts({ setViewport: viewport }), + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementText }), ensureReady: readiness, bootTarget: readiness, bootTargetHeadless: readiness, @@ -138,6 +136,7 @@ function freezeUnavailableFacts( viewport: Object.freeze({ ...unavailable.viewport }), readiness: orNetwork(unavailable.readiness), shutdown: orNetwork(unavailable.shutdown), + elementText: Object.freeze({ ...unavailable.elementText }), lifecycle: applicationLifecycleOperationFacts(unavailable.lifecycle), }); } diff --git a/packages/contracts/src/platform-runtime.ts b/packages/contracts/src/platform-runtime.ts index da85afb82..5d5f8355f 100644 --- a/packages/contracts/src/platform-runtime.ts +++ b/packages/contracts/src/platform-runtime.ts @@ -10,6 +10,7 @@ import { AppError } from '@agent-device/kernel/errors'; import type { PlatformModuleMetadata } from './platform-module.ts'; import type { PlatformRequestScope } from './platform-runtime-host.ts'; import type { ApplicationLifecycleResourceLifecycle } from './application-lifecycle-runtime.ts'; +import { invalidRuntimeContract } from './runtime-contract-error.ts'; type RuntimeOperation = (...args: never[]) => unknown; @@ -283,10 +284,3 @@ function unsupportedRuntimeOperation(key: string, fact: RuntimeOperationUnavaila hint: fact.hint, }); } - -function invalidRuntimeContract(message: string): AppError { - return new AppError('COMMAND_FAILED', message, { - reason: 'runtime-contract-invalid', - hint: 'This is an agent-device runtime contract bug; report the selected device and command.', - }); -} diff --git a/packages/contracts/src/runtime-contract-error.ts b/packages/contracts/src/runtime-contract-error.ts new file mode 100644 index 000000000..905227bf4 --- /dev/null +++ b/packages/contracts/src/runtime-contract-error.ts @@ -0,0 +1,21 @@ +import { AppError } from '@agent-device/kernel/errors'; + +/** + * A runtime owner whose facts advertised an operation it cannot actually perform. + * + * This is a contract bug, never a legitimate refusal. Consumers must not classify it as an + * unavailable or declined operation: doing so places it inside whatever closed reason set + * licenses a fallback, and the command then answers from stale data precisely because the + * runtime lied about itself (ADR 0019 §2). + * + * Deliberately its own module rather than an export of `platform-runtime.ts`: the façade + * re-exports that module and must stay exhaustive over it, which would make this a public + * symbol with no external consumer — the unused-export defect. Here it stays internal to + * `packages/contracts` and both runtime modules share one construction. + */ +export function invalidRuntimeContract(message: string): AppError { + return new AppError('COMMAND_FAILED', message, { + reason: 'runtime-contract-invalid', + hint: 'This is an agent-device runtime contract bug; report the selected device and command.', + }); +} diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index f34ac21c7..440babae3 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -94,6 +94,8 @@ test.each([ expect(facts.operations.bootTarget).toEqual({ available: true }); expect(facts.operations.bootTargetHeadless.available).toBe(runtimeDevice.kind === 'emulator'); expect(facts.operations.captureSnapshot).toEqual({ available: true }); + // uiautomator reads text at a point over the same adb transport the capture uses. + expect(facts.operations.readTextAtPoint).toEqual({ available: true }); expect(facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(facts.operations.setViewport).toMatchObject({ available: false }); @@ -101,6 +103,7 @@ test.each([ expect(facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); + expect(binding.operations.readTextAtPoint).toBeTypeOf('function'); await expect(binding.operations.ensureReady?.({})).resolves.toMatchObject({ id: runtimeDevice.id, @@ -268,6 +271,7 @@ test.each([ expect(facts.operations.bootTarget).toEqual({ available: true }); expect(facts.operations.bootTargetHeadless.available).toBe(runtimeDevice.kind === 'emulator'); expect(facts.operations.captureSnapshot.available).toBe(runtimeDevice.kind !== 'simulator'); + expect(facts.operations.readTextAtPoint.available).toBe(runtimeDevice.kind !== 'simulator'); expect(binding.operations.captureSnapshot).toBeTypeOf( runtimeDevice.kind === 'simulator' ? 'undefined' : 'function', ); diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 585c2e571..7d11af67b 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -10,7 +10,9 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, + bindElementTextRuntime, bindLocalSnapshotInteractor, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -30,6 +32,10 @@ import { const owner = localRuntimeOwner('android'); const available = Object.freeze({ available: true } as const); +const elementTextKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', +} as const); const headlessUnavailable = Object.freeze({ available: false, reason: 'unsupported-device-kind', @@ -148,6 +154,11 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor capture: device.kind === 'simulator' ? screenshotKindUnavailable : available, }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + // uiautomator reads text at a point through the same adb path the snapshot uses, so the + // synthetic `simulator` row is the only Android kind without a live read. + ...elementTextRuntimeOperationFacts({ + readTextAtPoint: device.kind === 'simulator' ? elementTextKindUnavailable : available, + }), ensureReady: available, bootTarget: available, bootTargetHeadless: device.kind === 'emulator' ? available : headlessUnavailable, @@ -208,6 +219,13 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor resolveInteractor: host.localInteractors.resolve, }) : {}), + ...(facts.operations.readTextAtPoint.available + ? bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ensureReady: async (input: EnsureReadyInput) => await ensureAndroidReady( host, diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index 84431a283..b51e45fad 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -134,6 +134,10 @@ function expectAppleSnapshotAvailability( device.appleOs === 'macos', ); expect(binding.operations.captureSnapshot).toBeTypeOf(available ? 'function' : 'undefined'); + // The live point read needs a driveable Apple UI, so it follows the same watchOS sentinel the + // capture does; every other supported leaf advertises and binds it. + expect(binding.facts.operations.readTextAtPoint.available).toBe(available); + expect(binding.operations.readTextAtPoint).toBeTypeOf(available ? 'function' : 'undefined'); } test.each(['frontmost-app', 'desktop', 'menubar'] as const)( @@ -426,4 +430,5 @@ function expectLegacyLifecycleFactCell( facts.device.appleOs !== 'watchos' && (facts.device.kind === 'simulator' || facts.device.kind === 'device'); expect(facts.operations.captureSnapshot.available).toBe(snapshotAvailable); + expect(facts.operations.readTextAtPoint.available).toBe(snapshotAvailable); } diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 8903e1664..1b94570c3 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -9,6 +9,8 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, + bindElementTextRuntime, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -55,6 +57,15 @@ const headlessUnavailable = Object.freeze({ reason: 'unsupported-provider-mode', hint: 'Headless boot is supported only for local Android emulators.', } as const); +const elementTextLeafUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'watchOS has no XCUITest-driveable UI, so element text comes from the captured tree only.', +} as const); +const elementTextKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', +} as const); const watchOpenTargetUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', @@ -224,6 +235,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR ...appleSnapshotFacts(device), ...screenshotRuntimeOperationFacts({ capture: appleScreenshotFact(device) }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + ...elementTextRuntimeOperationFacts({ readTextAtPoint: appleElementTextFact(device) }), ensureReady: readiness, bootTarget: boot, bootTargetHeadless: headlessUnavailable, @@ -275,6 +287,13 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR resolveInteractor: host.localInteractors.resolve, }) : {}), + ...(facts.operations.readTextAtPoint.available + ? bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ...(facts.operations.ensureReady.available ? { ensureReady: async () => @@ -322,6 +341,18 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR }); } +/** + * The live point read is the XCUITest runner's `readText` for app sessions and the macOS helper + * for desktop/menubar surfaces. Both need a driveable Apple UI, so watchOS and the non + * simulator/device kinds have no read at all. + */ +function appleElementTextFact(device: DeviceInfo) { + if (resolveDeviceAppleOs(device) === 'watchos') return elementTextLeafUnavailable; + return device.kind === 'simulator' || device.kind === 'device' + ? available + : elementTextKindUnavailable; +} + function appleSnapshotFact(device: DeviceInfo) { if (resolveDeviceAppleOs(device) === 'watchos') return snapshotKindUnavailable; return device.kind === 'simulator' || device.kind === 'device' diff --git a/packages/platform-harmonyos/src/runtime.test.ts b/packages/platform-harmonyos/src/runtime.test.ts index 12d40fd98..48519138a 100644 --- a/packages/platform-harmonyos/src/runtime.test.ts +++ b/packages/platform-harmonyos/src/runtime.test.ts @@ -72,6 +72,10 @@ test.each([ expect(binding.operations.setViewport).toBeUndefined(); expect(facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + // HarmonyOS has no point-read tool: `get` answers from the captured tree, which is what the + // legacy dispatch already did once its Apple-runner fall-through failed. + expect(facts.operations.readTextAtPoint).toMatchObject({ available: false }); + expect(binding.operations.readTextAtPoint).toBeUndefined(); await expect(binding.operations.ensureReady?.({})).resolves.toMatchObject({ booted: true }); await expect( binding.operations.listApps?.({ device: runtimeDevice, filter: 'all' }), @@ -188,6 +192,7 @@ test.each([ expect(facts.operations.ensureReady).toMatchObject({ available: true }); expect(facts.operations.bootTarget).toMatchObject({ available: false }); expect(facts.operations.bootTargetHeadless).toMatchObject({ available: false }); + expect(facts.operations.readTextAtPoint.available).toBe(false); expect(facts.operations.captureSnapshot.available).toBe( runtimeDevice.kind === 'emulator' || runtimeDevice.kind === 'device', ); diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index dafbdabaf..f81aebb05 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -9,6 +9,7 @@ import { availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, bindLocalSnapshotInteractor, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -28,6 +29,11 @@ import { } from './deployment/runtime.ts'; const owner = localRuntimeOwner('harmonyos'); +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'HarmonyOS reads element text from the captured tree only.', +} as const); const available = Object.freeze({ available: true } as const); const unavailable = Object.freeze({ available: false, @@ -143,6 +149,9 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor : screenshotKindUnavailable, }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + // HarmonyOS has no point-read tool: `get` answers from the captured tree, which is what + // the legacy dispatch already did after its Apple-runner attempt failed. + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), ensureReady: available, bootTarget: unavailable, bootTargetHeadless: unavailable, diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index 56700ea5b..56ce74812 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -100,6 +100,9 @@ test.each([ expect(binding.facts.operations.appState).toMatchObject({ available: false }); expect(binding.facts.operations.listApps).toMatchObject({ available: false }); expect(binding.facts.operations.captureSnapshot.available).toBe(device.kind === 'device'); + // The Linux read is value-first where the captured tree is label-first, so the desktop row + // genuinely reads differently from its snapshot text and advertises the live read. + expect(binding.facts.operations.readTextAtPoint.available).toBe(device.kind === 'device'); expect(binding.facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(binding.facts.operations.captureSnapshotWithoutActiveApp.available).toBe( device.kind === 'device', diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 10c95f1c9..f8842b4f2 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -11,7 +11,9 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, + bindElementTextRuntime, createUnavailablePlatformRuntimeFacts, + elementTextRuntimeOperationFacts, localRuntimeOwner, sameRuntimeOwner, screenshotRuntimeOperationFacts, @@ -24,6 +26,7 @@ import { bindLinuxApplicationLifecycle } from './lifecycle.ts'; const supported = Object.freeze({ available: true } as const); const linuxOwner = localRuntimeOwner('linux'); const unsupportedPlatformLeaf = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); +const elementTextKindUnavailable = unavailableLinuxRuntimeFact('unsupported-device-kind'); const runtimeHintsUnavailable = unavailableLinuxRuntimeFact( 'unsupported-platform-leaf', 'Runtime hints are supported only for local iOS-family simulators and Android devices.', @@ -93,6 +96,13 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR resolveInteractor: host.localInteractors.resolve, }) : {}), + ...(facts.operations.readTextAtPoint.available + ? bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), }), [Symbol.asyncDispose]: async () => undefined, }) satisfies DeviceBinding; @@ -110,6 +120,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts screenshot: screenshotKindUnavailable, snapshot: snapshotKindUnavailable, viewport: unsupportedPlatformLeaf, + elementText: elementTextKindUnavailable, readiness: unsupportedPlatformLeaf, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: openTarget, @@ -135,6 +146,11 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts ...screenshotRuntimeOperationFacts({ capture: device.kind === 'device' ? supported : screenshotKindUnavailable, }), + // The Linux read is value-first (AXValue/title/description) where the captured tree is + // label-first, so the desktop row genuinely reads differently from its snapshot text. + ...elementTextRuntimeOperationFacts({ + readTextAtPoint: device.kind === 'device' ? supported : elementTextKindUnavailable, + }), }, }); } diff --git a/packages/platform-vega/src/runtime.test.ts b/packages/platform-vega/src/runtime.test.ts index 8859b3a7e..6ab34f852 100644 --- a/packages/platform-vega/src/runtime.test.ts +++ b/packages/platform-vega/src/runtime.test.ts @@ -135,6 +135,8 @@ test.each([ reason: 'unsupported-platform-leaf', }); expect(binding.operations.captureSnapshot).toBeUndefined(); + expect(binding.facts.operations.readTextAtPoint.available).toBe(false); + expect(binding.operations.readTextAtPoint).toBeUndefined(); expect(binding.facts.operations.setViewport).toMatchObject({ available: false }); expect(binding.facts.operations.captureScreenshot).toMatchObject({ available: false, diff --git a/packages/platform-vega/src/runtime.ts b/packages/platform-vega/src/runtime.ts index ebd37d6ce..5c96a1598 100644 --- a/packages/platform-vega/src/runtime.ts +++ b/packages/platform-vega/src/runtime.ts @@ -90,6 +90,7 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts screenshot: screenshotUnavailable, snapshot: unsupportedPlatformLeaf, viewport: unsupportedPlatformLeaf, + elementText: unsupportedPlatformLeaf, readiness: unsupportedPlatformLeaf, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: openTarget, diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index 9b9c4f601..fd226aba8 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -54,6 +54,10 @@ test('preserves a narrow web provider dump including empty successful entries', expect(binding.operations.setViewport).toBeTypeOf('function'); expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + // No point-addressed read on the web backend: `get` answers from the captured DOM tree. The + // legacy `read` dispatch had no web arm at all and threw on every call before falling back. + expect(binding.facts.operations.readTextAtPoint.available).toBe(false); + expect(binding.operations.readTextAtPoint).toBeUndefined(); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); expectLifecycleFacts(binding); }); diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index 5e301ffeb..82390e1c7 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -10,6 +10,7 @@ import { availableApplicationLifecycleOperations, bindLocalScreenshotInteractor, bindLocalSnapshotInteractor, + elementTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -23,6 +24,11 @@ import { bindWebApplicationLifecycle } from './lifecycle.ts'; const owner = localRuntimeOwner('web'); const available = Object.freeze({ available: true } as const); +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'Web targets read element text from the captured tree only.', +} as const); const appLogUnavailable = Object.freeze({ available: false, reason: 'unsupported-platform-leaf', @@ -244,6 +250,9 @@ function webRuntimeFacts( }), ...screenshotRuntimeOperationFacts({ capture: browserDevice }), ...viewportRuntimeOperationFacts({ setViewport: browserDevice }), + // The web backend has no point-addressed read: `get` answers from the captured DOM tree, + // which is what the legacy dispatch already did once its Apple-runner attempt failed. + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), ensureReady: readinessUnavailable, bootTarget: readinessUnavailable, bootTargetHeadless: readinessUnavailable, diff --git a/packages/provider-limrun/src/app-log-runtime.test.ts b/packages/provider-limrun/src/app-log-runtime.test.ts index c372c1c91..c928b7e0c 100644 --- a/packages/provider-limrun/src/app-log-runtime.test.ts +++ b/packages/provider-limrun/src/app-log-runtime.test.ts @@ -284,6 +284,13 @@ test.each([ expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); expect(binding.operations.captureSnapshot).toBeTypeOf('function'); + // Limrun owns the device remotely and exposes no local point-read tool, so the live read is + // unavailable and `get` answers from the captured tree — never by borrowing the local runtime. + expect(binding.facts.operations.readTextAtPoint).toMatchObject({ + available: false, + reason: 'unsupported-provider-mode', + }); + expect(binding.operations.readTextAtPoint).toBeUndefined(); await expect( binding.operations.listApps?.({ device: runtimeDevice, filter: 'all' }), ).resolves.toEqual([]); diff --git a/packages/provider-limrun/src/app-log-runtime.ts b/packages/provider-limrun/src/app-log-runtime.ts index 76d8cb01b..68e4b7841 100644 --- a/packages/provider-limrun/src/app-log-runtime.ts +++ b/packages/provider-limrun/src/app-log-runtime.ts @@ -27,6 +27,7 @@ import { providerRuntimeOwner, sameRuntimeOwner, screenshotRuntimeOperationFacts, + elementTextRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, } from '@agent-device/contracts/platform'; @@ -92,6 +93,16 @@ const viewportUnavailable = Object.freeze({ reason: 'unsupported-provider-mode', hint: 'Limrun does not expose viewport resizing.', } as const); +/** + * A point read needs a local tool (adb uiautomator, the XCUITest runner). Limrun's transport + * carries none of them, so the owner reports no live read and `get` answers from the captured + * tree; provider ownership never borrows the local family read. + */ +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'Limrun-owned devices read element text from the captured tree only.', +} as const); const recordingUnavailable = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -191,6 +202,7 @@ export function createLimrunPlatformRuntimeOwner( network: liveSessionUnavailable, screenshot: liveSessionUnavailable, viewport: liveSessionUnavailable, + elementText: liveSessionUnavailable, readiness: liveSessionUnavailable, shutdown: liveSessionUnavailable, lifecycle: limrunLifecycleFacts(device, false), @@ -442,6 +454,7 @@ function facts( }), ...screenshotRuntimeOperationFacts({ capture: available }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), + ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), ensureReady: available, bootTarget: available, bootTargetHeadless: headlessUnavailable, diff --git a/packages/provider-webdriver/src/platform-runtime.test.ts b/packages/provider-webdriver/src/platform-runtime.test.ts index c14d401e3..d1be56e09 100644 --- a/packages/provider-webdriver/src/platform-runtime.test.ts +++ b/packages/provider-webdriver/src/platform-runtime.test.ts @@ -239,6 +239,13 @@ test('captures through only the active exact WebDriver interactor', async () => expect(binding.operations.setViewport).toBeUndefined(); expect(binding.facts.operations.captureScreenshot).toEqual({ available: true }); expect(binding.operations.captureScreenshot).toBeTypeOf('function'); + // Provider ownership is authoritative and fails closed: a WebDriver owner's transport carries + // no local point-read tool, so it advertises none and never borrows the local family read. + expect(binding.facts.operations.readTextAtPoint).toMatchObject({ + available: false, + reason: 'unsupported-provider-mode', + }); + expect(binding.operations.readTextAtPoint).toBeUndefined(); await expect( binding.operations.captureSnapshot?.({ options: { interactiveOnly: true } }), ).resolves.toEqual({ backend: 'android', nodes: [] }); @@ -273,6 +280,7 @@ test.each([ expect(facts.operations.captureSnapshotWithoutActiveApp.available).toBe(false); expect(facts.operations.setViewport.available).toBe(false); expect(facts.operations.captureScreenshot.available).toBe(false); + expect(facts.operations.readTextAtPoint.available).toBe(false); if (state.isSessionActive()) { const binding = await owner.bind({ device, diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index 813b04ff1..19f36d5ad 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -87,6 +87,18 @@ const viewportUnavailable = Object.freeze({ hint: 'WebDriver provider runtimes do not expose viewport resizing.', } as const); +/** + * A point read is a local-tool operation (adb uiautomator, the XCUITest runner, the macOS + * helper). A WebDriver owner's transport carries none of them, so the owner reports no live + * read and `get` answers from the captured tree; provider ownership never borrows the local + * family read. + */ +const elementTextUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'WebDriver provider runtimes read element text from the captured tree only.', +} as const); + const appStateUnavailable = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -268,6 +280,7 @@ function webDriverFacts( screenRecording: inactiveSession, screenshot: inactiveSession, viewport: inactiveSession, + elementText: inactiveSession, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: inactiveSession, prepareApplicationOpen: inactiveSession, @@ -289,6 +302,7 @@ function webDriverFacts( screenRecording: recordingUnavailable, screenshot: screenshotUnavailable, viewport: viewportUnavailable, + elementText: elementTextUnavailable, lifecycle: webDriverLifecycleFacts(device), }); // Both capture cells need the same reachability: an interactor this provider can drive, on a diff --git a/scripts/layering/check.ts b/scripts/layering/check.ts index 7817c3b01..1795b55f0 100644 --- a/scripts/layering/check.ts +++ b/scripts/layering/check.ts @@ -36,8 +36,8 @@ // composition file; premature implementation loading and forbidden cross-boundary edges fail (R13). // - Over COMMAND-ATOMIC RUNTIME CUTOVERS: one parametrized gate reads the migrated-command // table (appstate R22, shutdown R23, boot R20, apps R21, install/deploy R24-R27, -// lifecycle R28-R31, devices R17, logs R14, network R15, record R16, snapshot R32, diff R33) -// and proves each command keeps +// lifecycle R28-R31, devices R17, logs R14, network R15, record R16, snapshot R32, diff R33, +// viewport R34, get R36, is R37 — R35 reserved for find) and proves each command keeps // exactly one platform-execution path — retired routes, admission, modules, and widened // runtime access cannot coexist with its operation-fact-derived descriptor and handler. // - Over CONTRACTS PRODUCTION SOURCE: contracts owns vocabulary only — host, process, and timer diff --git a/scripts/layering/runtime-command-cutover-model.ts b/scripts/layering/runtime-command-cutover-model.ts index 55519b615..9674860ad 100644 --- a/scripts/layering/runtime-command-cutover-model.ts +++ b/scripts/layering/runtime-command-cutover-model.ts @@ -28,6 +28,17 @@ export type LegacyRetirementClaim = Readonly<{ daemonOnlyProviderMethods?: readonly string[]; /** `PlatformPlugin` facet keys retired with the legacy adapter. */ pluginFacetKeys?: readonly string[]; + /** + * Static platform command sets this command's admission DATA was removed from — the whole + * retirement of a command whose legacy admission was a capability bucket plus set membership, + * with no adapter module, route, or dispatch projection to name. + * + * Every other form above names something that must NOT exist, which a row can satisfy by + * inventing a name that never existed. This one is two-sided and cannot: each named set must + * still EXIST in production source, and must no longer list the command. A fictional set fails + * the first half, a skipped deletion the second. + */ + staticCommandSets?: readonly string[]; }>; /** @@ -157,6 +168,7 @@ const RETIREMENT_FORMS = [ 'daemonOnlyRouteNames', 'daemonOnlyProviderMethods', 'pluginFacetKeys', + 'staticCommandSets', ] as const satisfies readonly (keyof LegacyRetirementClaim)[]; /** diff --git a/scripts/layering/runtime-command-cutover-policy.test.ts b/scripts/layering/runtime-command-cutover-policy.test.ts index 6f5d26a94..f21f6d178 100644 --- a/scripts/layering/runtime-command-cutover-policy.test.ts +++ b/scripts/layering/runtime-command-cutover-policy.test.ts @@ -285,3 +285,57 @@ test('every shipped row states its claims', () => { [], ); }); + +// A data-only admission retirement — a capability bucket plus static-set membership, with no +// module, route, or dispatch projection to name as gone. Both halves are planted, because the +// half that matters is the one an identifier-shaped claim cannot state: a set that never existed. +const DATA_ONLY_ROW: MigratedCommandCutover = { + ...PLANTED_ROW, + legacyRetirement: { staticCommandSets: ['WEB_QUERY_COMMANDS'] }, +}; + +test('a data-only retirement is a stated claim, so a row needs no invented identifier', () => { + assert.deepEqual(cutoverRowDefects(DATA_ONLY_ROW), []); +}); + +test('planted red: a row claiming a static command set that does not exist is rejected', () => { + assert.deepEqual( + summariesFor( + PLANTED_RULE, + [['src/core/capabilities.ts', `const WEB_QUERY_COMMANDS = ['find'];`]], + [ + { + ...DATA_ONLY_ROW, + legacyRetirement: { staticCommandSets: ['WEB_QUERY_COMMANDS_WITH_PLANTED'] }, + }, + ], + ).filter((summary) => summary.includes('static command set')), + [ + "(planted cutover row): claims retired static command set 'WEB_QUERY_COMMANDS_WITH_PLANTED', which no production source declares", + ], + ); +}); + +test('planted red: a claimed static command set that still lists the command is rejected', () => { + assert.deepEqual( + summariesFor( + PLANTED_RULE, + [['src/core/capabilities.ts', `const WEB_QUERY_COMMANDS = ['find', 'planted'];`]], + [DATA_ONLY_ROW], + ).filter((summary) => summary.includes('still admits')), + ['src/core/capabilities.ts: static command set WEB_QUERY_COMMANDS still admits planted'], + ); +}); + +test('a claimed static command set that exists and dropped the command passes', () => { + // Scoped to this column: the planted row's singular-execution claims are unrelated here and + // have their own cases above. + assert.deepEqual( + summariesFor( + PLANTED_RULE, + [['src/core/capabilities.ts', `const WEB_QUERY_COMMANDS = ['find'];`]], + [DATA_ONLY_ROW], + ).filter((summary) => summary.includes('static command set')), + [], + ); +}); diff --git a/scripts/layering/runtime-command-cutover-policy.ts b/scripts/layering/runtime-command-cutover-policy.ts index fe4335413..a48f058ac 100644 --- a/scripts/layering/runtime-command-cutover-policy.ts +++ b/scripts/layering/runtime-command-cutover-policy.ts @@ -79,6 +79,7 @@ function rowViolations( violations.push(...narrowingViolations(row, file, program)); } violations.push(...exactCallViolations(row, files, programs)); + violations.push(...staticCommandSetViolations(row, files, programs)); const sources = new Map(files.map(({ path, source }) => [path, source])); for (const check of rowChecks(row)) violations.push(...check(sources)); return violations; @@ -356,6 +357,59 @@ function isAdmissionMember( ); } +/** + * A data-only admission retirement, proven from both sides. + * + * A command whose legacy admission was a capability bucket plus membership in a static platform + * command set retires no module, route, or dispatch projection — there is no identifier to name + * as gone. Naming an invented one satisfies the non-empty shape check while proving nothing, so + * the row names the sets themselves: each must still be DECLARED in production source, and must + * no longer carry this command. + * + * The existence half is what an identifier-shaped claim cannot express. The membership half + * overlaps the automatic static-set column for `WEB`/`HARMONY`-named sets, deliberately: stating + * it here keeps the declared claim self-sufficient rather than dependent on that regex. + */ +function staticCommandSetViolations( + row: MigratedCommandCutover, + files: readonly ProductionSource[], + programs: ReadonlyMap, +): UnruledViolation[] { + const declared = row.legacyRetirement.staticCommandSets ?? []; + if (declared.length === 0) return []; + const violations: UnruledViolation[] = []; + const seen = new Set(); + for (const file of files) { + const program = programs.get(file.path); + if (!program) continue; + visitAst(program, (node) => { + const name = staticCommandSetName(node, declared); + if (name === undefined) return; + seen.add(name); + if (containsStringLiteral(node['init'], row.command)) { + violations.push(at(file, node, `static command set ${name} still admits ${row.command}`)); + } + }); + } + for (const name of declared) { + if (seen.has(name)) continue; + violations.push({ + file: `(${row.command} cutover row)`, + line: 1, + message: `claims retired static command set '${name}', which no production source declares`, + }); + } + return violations; +} + +function staticCommandSetName(node: AstNode, declared: readonly string[]): string | undefined { + if (node['type'] !== 'VariableDeclarator') return undefined; + const id = node['id'] as AstNode | undefined; + if (id?.['type'] !== 'Identifier') return undefined; + const name = String(id['name']); + return declared.includes(name) ? name : undefined; +} + function containsStringLiteral(node: unknown, expected: string): boolean { let found = false; visitAst(node, (candidate) => { diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index c856a822d..1e5783e3f 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -25,7 +25,8 @@ import { retiredDispatchProjectionViolations } from './runtime-command-cutover-d * A row id is a report heading, so it must be unique across every stack that adds rows here. * `cutoverTableDefects` rejects a duplicate; lifecycle starts at R28 after the accepted * shutdown, install/deploy, and application-lifecycle allocations. Snapshot starts at R32; - * diff follows at R33. + * diff follows at R33, viewport at R34, get at R36, and is at R37. R35 stays reserved for + * find, whose cutover is deferred behind the Wave 5 `focus`/`type` surfaces. */ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ { @@ -506,6 +507,85 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ }, extensions: [diffRetiredDispatchProjectionProof], }, + { + rule: 'R36 get-runtime-cutover', + command: 'get', + subject: 'element read', + tier: 'request-scoped', + execution: 'device-runtime', + // Two retirements. `get`'s own legacy admission was its capability bucket plus the static + // family command sets the matrix augments it with — the row's automatic admission columns + // reject the bucket and the `requireCommandSupported('get', …)` call. And the shared element + // read: both consumers of the selector backend's read (`get text` and read-only + // `find … get text`) now execute the bound `readTextAtPoint`, so the legacy `read` dispatch + // alias retires whole. Deleting its registry entry drops `'read'` from + // `DescriptorDispatchCommandName`, which makes a surviving `DISPATCH_HANDLERS.read` a COMPILE + // error rather than something this row has to police. + legacyRetirement: { + modulePaths: ['src/daemon/handlers/interaction-read-legacy-dispatch.ts'], + importPatterns: [/(?:^|\/)handlers\/interaction-read-legacy-dispatch(?:\.[cm]?[jt]s)?$/], + // `dispatchDirectIosSelectorGet` was `get`'s last route to the platform outside the seam: + // it reached `runAppleRunnerCommand` through a path this row declares no operation for. + // Admitting before a bypass is not executing through the seam, so the bypass is retired + // rather than ordered after admission. `queryDirectIosSelector` itself stays — the Wave 5 + // offscreen-target probe still consumes it and it remains single-copy. + routeNames: ['handleReadCommand', 'dispatchDirectIosSelectorGet'], + }, + runtimeTypeNames: ['ElementTextRuntimeOperations', 'SnapshotRuntimeOperations'], + operations: { + names: ['captureSnapshot', 'captureSnapshotWithoutActiveApp', 'readTextAtPoint'], + }, + singularExecution: { + routes: ['dispatchGetViaRuntime'], + operations: ['captureSnapshot', 'captureSnapshotWithoutActiveApp', 'readTextAtPoint'], + // `get` executes through the shared selector seam, so the capture owners are the SAME + // selectors `snapshot`/`diff` count; only the preferred element read is this unit's own. + // With the direct-iOS bypass retired these names are now the ONLY routes from + // `dispatchGetViaRuntime` to the platform, so the claim states what the code does. + operationOwners: { + captureSnapshot: ['selectActiveAppSnapshot'], + captureSnapshotWithoutActiveApp: ['selectSnapshotWithoutActiveApp'], + readTextAtPoint: ['bindElementRead'], + }, + }, + }, + { + rule: 'R37 is-runtime-cutover', + command: 'is', + subject: 'element predicate', + tier: 'request-scoped', + execution: 'device-runtime', + // `is` retired no module, route, or dispatch projection — it had none. Its whole legacy + // admission was the capability bucket (rejected by this row's automatic descriptor column) + // plus membership in these two static sets, which is a DATA deletion. Naming the sets proves + // it from both sides: each must still be declared in production source and must no longer + // list `is`, so neither an invented name nor a skipped deletion can satisfy it. + legacyRetirement: { + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS', 'WEB_QUERY_COMMANDS'], + }, + runtimeTypeNames: ['SnapshotRuntimeOperations'], + operations: { names: ['captureSnapshot', 'captureSnapshotWithoutActiveApp'] }, + singularExecution: { + routes: ['dispatchIsViaRuntime'], + operations: ['captureSnapshot', 'captureSnapshotWithoutActiveApp'], + // `is` executes through the shared selector seam, so its capture owners are the SAME + // selectors `snapshot`/`diff`/`get` count. It declares no operation of its own: every + // predicate answers from the resolved tree, so `readTextAtPoint` stays R36's alone. + // + // Scope, stated so this is not read as absolute: the claim covers how a predicate is + // EXECUTED. Since the direct-iOS selector shortcut retired, the bound capture is the only + // thing that answers one. It does NOT claim the route makes no other device call — the + // Android foreground-blocker diagnostic still reaches adb through + // `platforms/android/app-lifecycle.ts`, on the FAILURE path only, where it can enrich an + // already-failed response's message but can never produce or change a verdict. That edge + // is pre-existing, co-owned with `wait`, and recorded as Wave 6 denominator work; R22's + // `appState` is its declared replacement. + operationOwners: { + captureSnapshot: ['selectActiveAppSnapshot'], + captureSnapshotWithoutActiveApp: ['selectSnapshotWithoutActiveApp'], + }, + }, + }, { rule: 'R34 viewport-runtime-cutover', command: 'viewport', diff --git a/src/__tests__/cli-exit-paths.test.ts b/src/__tests__/cli-exit-paths.test.ts index 4c97cf77e..73ceb8e52 100644 --- a/src/__tests__/cli-exit-paths.test.ts +++ b/src/__tests__/cli-exit-paths.test.ts @@ -222,3 +222,39 @@ test('a --debug failure caps the daemon-log-tail dump instead of printing it unb 'expected the byte cap to drop the oldest lines, not just the 200-line cap', ); }); + +// The end-to-end half of `is`'s documented contract: "is evaluates UI predicates against a +// selector expression and exits non-zero on failure" (website/docs/docs/commands.md). +// +// This deliberately does NOT know how the daemon decided. It was written when the direct-iOS +// shortcut answered some predicates itself and returned `{ok: true, pass: false}`, which the CLI +// rendered as `Passed: is text` with exit 0 (#1739). The shortcut is retired and every predicate +// now answers from the bound capture, so the guarantee is structural rather than guard-based — +// and this case survives that change untouched, because a failed assertion must exit non-zero +// whatever produced the failure. +test('a failed `is` predicate exits non-zero, whatever answered it', async () => { + const restoreEnv = installIsolatedCliTestEnv(); + const exitSpy = installExitSpy(); + const stderr = captureStderr(); + const sendToDaemon = async (): Promise => ({ + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'is text failed for selector id=greeting: expected="Welcome" actual="Goodbye"', + details: { command: 'is', reason: 'predicate_failed', predicate: 'text' }, + }, + }); + + try { + await runCli(['is', 'text', 'id=greeting', 'Welcome'], { sendToDaemon }); + } finally { + stderr.restore(); + exitSpy.restore(); + restoreEnv(); + } + + assert.deepEqual(exitSpy.calls, [1]); + const output = stderr.read(); + assert.ok(output.includes('COMMAND_FAILED'), 'expected the typed failure on stderr'); + assert.ok(!output.includes('Passed'), 'a failed assertion must never render as passed'); +}); diff --git a/src/__tests__/test-file-size-ratchet.test.ts b/src/__tests__/test-file-size-ratchet.test.ts index 1f3a440e6..d3120b72f 100644 --- a/src/__tests__/test-file-size-ratchet.test.ts +++ b/src/__tests__/test-file-size-ratchet.test.ts @@ -48,7 +48,7 @@ const PINNED_TEST_FILE_LINES: Readonly> = Object.freeze({ 'src/platforms/apple/core/__tests__/runner-command-retry.test.ts': 1327, 'src/__tests__/cli-client-commands.test.ts': 1317, 'src/__tests__/cli-config.test.ts': 1282, - 'src/daemon/handlers/__tests__/find.test.ts': 1223, + 'src/daemon/handlers/__tests__/find.test.ts': 1207, 'src/platforms/apple/core/__tests__/perf.test.ts': 1222, 'src/mcp/__tests__/command-tools.test.ts': 1218, 'src/daemon/handlers/__tests__/session-replay-divergence.test.ts': 1215, diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index 8f2b6f3f5..d7400257d 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -1,6 +1,7 @@ import { applicationLifecycleOperationFacts, screenshotRuntimeOperationFacts, + elementTextRuntimeOperationFacts, snapshotRuntimeOperationFacts, type RuntimeOperationFact, } from '@agent-device/contracts/platform'; @@ -33,6 +34,7 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre ...unavailableShutdownOperationFacts, ...screenshotRuntimeOperationFacts({ capture: unavailable }), setViewport: unavailable, + ...elementTextRuntimeOperationFacts({ readTextAtPoint: unavailable }), }); /** Default facts for tests that are unrelated to application lifecycle commands. */ diff --git a/src/__tests__/test-utils/session-factories.ts b/src/__tests__/test-utils/session-factories.ts index 5c88d362e..f7a6b95fd 100644 --- a/src/__tests__/test-utils/session-factories.ts +++ b/src/__tests__/test-utils/session-factories.ts @@ -51,6 +51,15 @@ export function makeIosSession(name: string, overrides?: Partial): return makeSession(name, { device: IOS_SIMULATOR, ...overrides }); } +/** + * An iOS session with a tracked app — what `open ` produces. The shared snapshot + * runtime exposes capture on an iOS leaf only through the active-app plan row, so a test + * that captures on iOS needs this rather than a bare session. + */ +export function makeIosAppSession(name: string, overrides?: Partial): SessionState { + return makeIosSession(name, { appBundleId: 'com.example.app', ...overrides }); +} + export function makeAndroidSession(name: string, overrides?: Partial): SessionState { return makeSession(name, { device: ANDROID_EMULATOR, ...overrides }); } diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index cf7cb2878..0b4eb5f5c 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -232,7 +232,6 @@ test('macOS supports the Apple runner interaction core but excludes mobile-only 'find', 'focus', 'get', - 'is', 'longpress', 'logs', 'perf', @@ -306,7 +305,6 @@ test('Linux supports desktop interaction commands and blocks mobile/unsupported 'focus', 'get', 'home', - 'is', 'longpress', 'press', 'screenshot', @@ -334,7 +332,6 @@ test('web supports only the initial browser interaction slice', () => { 'find', 'get', 'hover', - 'is', 'press', 'record', 'screenshot', diff --git a/src/core/__tests__/capability-plugin-routing-parity.test.ts b/src/core/__tests__/capability-plugin-routing-parity.test.ts index 1c6983fc0..3abb51df8 100644 --- a/src/core/__tests__/capability-plugin-routing-parity.test.ts +++ b/src/core/__tests__/capability-plugin-routing-parity.test.ts @@ -180,11 +180,9 @@ const HARMONYOS_SUPPORTED_COMMANDS_REF = new Set([ 'fill', 'find', 'focus', - 'get', 'home', 'gesture', 'keyboard', - 'is', 'longpress', 'press', 'screenshot', @@ -270,9 +268,7 @@ test('HarmonyOS static capabilities omit runtime-backed command admissions', () 'find', 'focus', 'gesture', - 'get', 'home', - 'is', 'keyboard', 'longpress', 'perf', diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 438bee70e..2a5e4c2aa 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -44,11 +44,9 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'fill', 'find', 'focus', - 'get', 'home', 'gesture', 'keyboard', - 'is', 'longpress', 'press', 'scroll', @@ -57,7 +55,7 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'type', 'wait', ]); -const WEB_QUERY_COMMANDS = ['audio', 'find', 'get', 'is', 'wait'] as const; +const WEB_QUERY_COMMANDS = ['audio', 'find', 'wait'] as const; const WEB_INTERACTION_COMMANDS = [ 'click', 'fill', diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index f63497733..1f5c32863 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -57,8 +57,10 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.diff, PUBLIC_COMMANDS.doctor, PUBLIC_COMMANDS.events, + PUBLIC_COMMANDS.get, PUBLIC_COMMANDS.install, PUBLIC_COMMANDS.installFromSource, + PUBLIC_COMMANDS.is, PUBLIC_COMMANDS.logs, PUBLIC_COMMANDS.network, PUBLIC_COMMANDS.open, @@ -192,7 +194,11 @@ test('platform dispatch command list is built from descriptor dispatch facets', .sort(); assert.deepEqual(listRegisteredDispatchCommandNames(), dispatchCommands); - assert.ok(dispatchCommands.includes('read'), 'read stays dispatch-only'); + assert.equal( + dispatchCommands.includes('read' as never), + false, + 'the read dispatch alias retired with the selector element-read cutover (#1739)', + ); assert.equal( dispatchCommands.includes(PUBLIC_COMMANDS.gesture), false, diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 24c618fa3..cf79556c7 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -26,6 +26,7 @@ import { readySendPushNotificationUse, openApplicationRuntimePlanUses, closeApplicationRuntimePlanUses, + selectorCaptureRuntimePlanUses, snapshotRuntimePlanUses, prepareAppleRunnerRuntimeUse, runtimeCommandRuntimePlanUses, @@ -1188,21 +1189,9 @@ export const RAW_COMMAND_DESCRIPTORS = [ recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'interaction', refFrameEffect: 'preserve' }, - capability: ALL_DEVICE_COMMAND_CAPABILITY, timeoutPolicy: postActionObservationTimeoutPolicy('get', PRESERVE_DAEMON_TIMEOUT_POLICY), batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, - }, - { - name: 'read', - deviceClaimPolicy: 'require-owner', - ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/handlers/interaction.ts'] as const } : {}), - catalog: { group: 'dispatch-alias' }, - recordsSessionAction: false, - dispatch: {}, - timeoutPolicy: DEFAULT_TIMEOUT_POLICY, - batchable: false, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: selectorCaptureRuntimePlanUses }, }, { name: 'is', @@ -1214,10 +1203,9 @@ export const RAW_COMMAND_DESCRIPTORS = [ recordsSessionAction: true, recordingEffect: 'observes-app', daemon: { route: 'interaction', refFrameEffect: 'preserve' }, - capability: ALL_DEVICE_COMMAND_CAPABILITY, timeoutPolicy: postActionObservationTimeoutPolicy('is', PRESERVE_DAEMON_TIMEOUT_POLICY), batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: selectorCaptureRuntimePlanUses }, }, // -- generic (route: generic) -- diff --git a/src/core/command-descriptor/types.ts b/src/core/command-descriptor/types.ts index 9c8b53e52..1be071340 100644 --- a/src/core/command-descriptor/types.ts +++ b/src/core/command-descriptor/types.ts @@ -103,7 +103,7 @@ export type DeviceClaimPolicy = | 'acquire-session' | 'release-session'; -export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli' | 'dispatch-alias'; +export type CommandCatalogGroup = 'public' | 'internal' | 'local-cli'; /** * Which default tool set a framework adapter (`agent-device/ai-sdk`, the diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 24846aa8a..3a17d6029 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -666,59 +666,6 @@ function parseScrollTarget(input: string): { return { direction: parseScrollDirection(input) }; } -export async function handleReadCommand( - device: DeviceInfo, - positionals: string[], - context: DispatchContext | undefined, -): Promise> { - const { x, y } = readPoint(positionals, 'read requires x y'); - if (device.platform === 'android') { - const { readAndroidTextAtPoint } = await import('../platforms/android/input-actions.ts'); - const text = await readAndroidTextAtPoint(device, x, y); - return { action: 'read', text: text ?? '' }; - } - if (device.platform === 'linux') { - const { readLinuxTextAtPoint } = await import('../platforms/linux/snapshot.ts'); - const text = await readLinuxTextAtPoint(x, y, context?.surface); - return { action: 'read', text }; - } - if (isMacOs(device) && context?.surface && context.surface !== 'app') { - const { runMacOsReadTextAction } = await import('../platforms/apple/os/macos/helper.ts'); - const result = await runMacOsReadTextAction(x, y, { - bundleId: context.appBundleId, - surface: context.surface, - }); - return { action: 'read', text: result.text }; - } - // macOS app sessions run through the XCUITest runner; only desktop/menubar surfaces use the helper. - const { runAppleRunnerCommand } = await import('../platforms/apple/core/runner/runner-client.ts'); - const result = await runAppleRunnerCommand( - device, - { - command: 'readText', - x, - y, - appBundleId: context?.appBundleId, - }, - { - verbose: context?.verbose, - logPath: context?.logPath, - traceLogPath: context?.traceLogPath, - requestId: context?.requestId, - iosXctestrunFile: context?.iosXctestrunFile, - iosXctestDerivedDataPath: context?.iosXctestDerivedDataPath, - iosXctestEnvDir: context?.iosXctestEnvDir, - }, - ); - const text = - typeof result.text === 'string' - ? result.text - : typeof result.message === 'string' - ? result.message - : ''; - return { action: 'read', text }; -} - function findMistargetedTypeRef(positionals: string[]): string | null { return findMistargetedTypeRefToken(positionals[0]); } diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 4d4345e4e..c0232b8e2 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -18,7 +18,6 @@ import { handleHoverCommand, handleLongPressCommand, handlePressCommand, - handleReadCommand, handleScrollCommand, handleTypeCommand, } from './dispatch-interactions.ts'; @@ -180,7 +179,6 @@ const DISPATCH_HANDLERS: Record = { handleTvRemoteCommand(device, interactor, positionals, context), settings: ({ device, interactor, positionals, context }) => handleSettingsCommand(device, interactor, positionals, context), - read: ({ device, positionals, context }) => handleReadCommand(device, positionals, context), }; /** diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 329537222..7358d4419 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -65,6 +65,11 @@ export function createAndroidInteractor( performGesture: (plan) => executeAndroidTouchPlan(device, plan), gestureViewport: () => readAndroidGestureViewport(device), screenshot: (outPath, options) => screenshotAndroid(device, outPath, options), + // uiautomator reads the node covering a point; `undefined` means nothing covers it. + readTextAtPoint: async (point) => { + const { readAndroidTextAtPoint } = await import('../../platforms/android/input-actions.ts'); + return (await readAndroidTextAtPoint(device, point.x, point.y)) ?? undefined; + }, snapshot: async (options) => { const snapshotOptions = options ?? {}; const result = await withDiagnosticTimer( diff --git a/src/core/interactors/linux.ts b/src/core/interactors/linux.ts index 9d57c990d..5d7f21f16 100644 --- a/src/core/interactors/linux.ts +++ b/src/core/interactors/linux.ts @@ -45,6 +45,12 @@ export function createLinuxInteractor(): Interactor { await swipeLinux(start.x, start.y, end.x, end.y, plan.durationMs); }, screenshot: (outPath, options) => screenshotLinux(outPath, options), + // The Linux read is value-first (AXValue/title/description) where the captured tree is + // label-first, so this genuinely reads differently from its snapshot text. + readTextAtPoint: async (point, options) => { + const { readLinuxTextAtPoint } = await import('../../platforms/linux/snapshot.ts'); + return await readLinuxTextAtPoint(point.x, point.y, options?.surface); + }, snapshot: async (options) => { return await withDiagnosticTimer( 'snapshot_capture', diff --git a/src/daemon/__tests__/is-runtime.test.ts b/src/daemon/__tests__/is-runtime.test.ts new file mode 100644 index 000000000..add368a0e --- /dev/null +++ b/src/daemon/__tests__/is-runtime.test.ts @@ -0,0 +1,271 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { SnapshotResult } from '@agent-device/contracts/platform'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import { + makeAndroidSession, + makeIosAppSession, + makeIosSession, +} from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inventory-gateways.ts'; +import type { DaemonRequest } from '../types.ts'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; + +const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn() })); + +vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => { + const actual = + await importOriginal(); + return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; +}); + +import { dispatchIsViaRuntime } from '../selector-runtime.ts'; + +beforeEach(() => { + mockRunAppleRunnerCommand.mockReset(); + mockRunAppleRunnerCommand.mockResolvedValue({}); +}); + +// `is` answers every one of its seven predicates from the resolved capture — `isCommand` never +// reaches `backend.readText`. So its whole platform execution is the request-bound capture, and +// these cases bind at `inspectFacts` / `bindDevice`, never at `core/dispatch.ts`. + +const unavailableCapture = { available: false, reason: 'unsupported-device-kind' } as const; +const activeAppRequired = { available: false, reason: 'owner-capability-missing' } as const; + +/** One resolvable button, so a predicate has something real to answer about. */ +function buttonSnapshot(): SnapshotResult { + return { + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + ], + backend: 'android', + }; +} + +function isRequest(session: string, positionals: readonly string[]): DaemonRequest { + return { token: 't', session, command: 'is', positionals: [...positionals], flags: {} }; +} + +test('an admitted is inspects once, binds once, and answers through the bound capture', async () => { + const fixture = selectorCaptureFixture({ snapshot: () => buttonSnapshot() }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-bound', makeAndroidSession('is-bound', { appBundleId: 'com.example.app' })); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-bound', ['visible', 'id=auth_continue']), + sessionName: 'is-bound', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(true); + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); + expect(fixture.captures.length).toBeGreaterThan(0); +}); + +test('an unavailable capture fact refuses before any bind', async () => { + // The watchOS sentinel shape: capability-supported today, no snapshot backend at the owner. + const fixture = selectorCaptureFixture({ capture: unavailableCapture }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-refused', makeAndroidSession('is-refused', { appBundleId: 'com.a' })); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-refused', ['visible', 'id=auth_continue']), + sessionName: 'is-refused', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(false); + // The inspection is what makes this a typed admission refusal rather than a runtime failure: + // exact owner facts were read once, side-effect-free, and nothing bound or captured after. + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([]); + expect(fixture.captures).toEqual([]); +}); + +// The correctness fix this unit declares. On iOS `appBundleId` is the XCUITest attach target: +// with no tracked app the runner's own process comes to the foreground, DISPLACES the app under +// test, and the capture then answers confidently about the runner's own blank screen. Refusing +// beats displacing-and-lying. Android captures the real launcher in the same state, and the +// platform facts already encode that asymmetry — so `is` asks the facts rather than branching. +test('an iOS session with no tracked app is refused with the open hint, not answered from a displaced capture', async () => { + const fixture = selectorCaptureFixture({ + withoutActiveApp: activeAppRequired, + snapshot: () => buttonSnapshot(), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-no-app', makeIosSession('is-no-app')); + + const response = await withTestDeviceInventory( + {}, + async () => + await dispatchIsViaRuntime({ + req: isRequest('is-no-app', ['visible', 'id=auth_continue']), + sessionName: 'is-no-app', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ); + + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error?.code).toBe('SESSION_NOT_FOUND'); + expect(response.error?.message).toMatch(/requires an active app session/); + } + expect(fixture.binds).toEqual([]); + expect(fixture.captures).toEqual([]); +}); + +test('an iOS session WITH a tracked app still answers, so the refusal is the plan split and not an iOS ban', async () => { + const fixture = selectorCaptureFixture({ + withoutActiveApp: activeAppRequired, + snapshot: () => buttonSnapshot(), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-with-app', makeIosAppSession('is-with-app')); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-with-app', ['visible', 'label=Continue']), + sessionName: 'is-with-app', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(true); + expect(fixture.binds).toEqual([IOS_SIMULATOR]); +}); + +test('an Android session with no tracked app proceeds, because the owner advertises the without-active-app capture', async () => { + const fixture = selectorCaptureFixture({ snapshot: () => buttonSnapshot() }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-android-no-app', makeAndroidSession('is-android-no-app')); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-android-no-app', ['visible', 'id=auth_continue']), + sessionName: 'is-android-no-app', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(true); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); +}); + +// ADR 0019: a refused request reaches the device not at all. This used to guard the direct-iOS +// selector query, which could answer a simple `id=` target without a capture; that shortcut is +// retired, so the runner assertion below now proves the stronger property — on an unavailable +// fact, `is` makes no device call by any route. +test('a refused request reaches the device by no route at all', async () => { + const fixture = selectorCaptureFixture({ capture: unavailableCapture }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-direct-refused', makeIosAppSession('is-direct-refused')); + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + nodes: [ + { + index: 0, + type: 'Button', + label: 'Pickup', + identifier: 'shipping-pickup', + selected: true, + rect: { x: 126, y: 555, width: 75, height: 38 }, + enabled: true, + hittable: true, + }, + ], + }); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-direct-refused', ['selected', 'id="shipping-pickup"']), + sessionName: 'is-direct-refused', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(false); + expect(mockRunAppleRunnerCommand).not.toHaveBeenCalled(); + expect(fixture.binds).toEqual([]); +}); + +// `is` is an assertion: it "exits non-zero on failure" (website/docs/docs/commands.md). A +// direct-iOS shortcut used to answer some predicates itself and reported a failed one as a +// completed command — `is text id=… "Wrong Expected Text"` printed `Passed: is text` and exited 0 +// on device (#1739). That shortcut is retired, so the guarantee is now structural rather than +// guard-based: the bound capture is the only thing that answers a predicate, and `isCommand` +// raises COMMAND_FAILED when one fails. +// +// The CLI half — that such a response actually exits non-zero — lives in +// `src/__tests__/cli-exit-paths.test.ts`, at a layer that does not know how the daemon decided. +test('a failing predicate answers COMMAND_FAILED from the bound capture', async () => { + const fixture = selectorCaptureFixture({ + snapshot: () => ({ + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Apple Account', + identifier: 'account_row', + rect: { x: 10, y: 20, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + ], + backend: 'xctest', + }), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-direct-false', makeIosAppSession('is-direct-false')); + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + text: 'Apple Account', + nodes: [ + { + index: 0, + type: 'Button', + label: 'Apple Account', + identifier: 'account_row', + rect: { x: 10, y: 20, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + ], + }); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-direct-false', ['text', 'id=account_row', 'Wrong Expected Text']), + sessionName: 'is-direct-false', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + // A failed assertion is a failed command on every other path and in the docs; it is one here. + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error?.code).toBe('COMMAND_FAILED'); + expect(response.error?.details?.reason).toBe('predicate_failed'); + } + // The bound capture is what answered it. + expect(fixture.captures.length).toBeGreaterThan(0); +}); diff --git a/src/daemon/__tests__/selector-capture-binding.test.ts b/src/daemon/__tests__/selector-capture-binding.test.ts new file mode 100644 index 000000000..4aa78fb79 --- /dev/null +++ b/src/daemon/__tests__/selector-capture-binding.test.ts @@ -0,0 +1,145 @@ +import { expect, test } from 'vitest'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import { + makeAndroidSession, + makeIosSession, +} from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inventory-gateways.ts'; +import { resolveBoundSelectorCapture } from '../selector-capture-binding.ts'; +import { createBoundSelectorRuntime } from '../selector-runtime-backend.ts'; +import { selectorCaptureFixture } from './selector-capture-fixture.ts'; + +// The seam every selector unit consumes. `find` landed it; `get`, `is`, and `wait` migrate by +// naming their command, so these are the guarantees they inherit rather than re-derive. + +test('an available plan inspects once and binds once, on the admitted device', async () => { + const fixture = selectorCaptureFixture(); + + const bound = await resolveBoundSelectorCapture({ + command: 'find', + device: ANDROID_EMULATOR, + session: makeAndroidSession('selector'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(bound.ok).toBe(true); + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); +}); + +test('repeated captures reuse the one binding the plan was admitted for', async () => { + const fixture = selectorCaptureFixture(); + const bound = await resolveBoundSelectorCapture({ + command: 'wait', + device: ANDROID_EMULATOR, + session: makeAndroidSession('selector'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!bound.ok) throw new Error('expected an admitted capture'); + + await bound.operations.capture({}); + await bound.operations.capture({}); + + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); + expect(fixture.captures).toHaveLength(2); +}); + +test('an unavailable required operation refuses before any bind', async () => { + const fixture = selectorCaptureFixture({ + capture: { available: false, reason: 'unsupported-platform-leaf' }, + }); + + const bound = await resolveBoundSelectorCapture({ + command: 'is', + device: ANDROID_EMULATOR, + session: makeAndroidSession('selector'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(bound).toMatchObject({ ok: false, response: { ok: false } }); + expect(fixture.binds).toEqual([]); +}); + +// The active-app split is the only plan axis selector commands have: no `--actions` surface, +// so `captureSnapshotWithCustomActions` is never required and never admitted for them. +test('a session without a tracked app selects the without-active-app plan', async () => { + const fixture = selectorCaptureFixture({ + withoutActiveApp: { available: false, reason: 'owner-capability-missing' }, + }); + + const withApp = await resolveBoundSelectorCapture({ + command: 'get', + device: IOS_SIMULATOR, + session: makeIosSession('with-app', { appBundleId: 'com.example.app' }), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + // The iOS refusal enriches its hint from device inventory, which request execution owns. + const withoutApp = await withTestDeviceInventory( + {}, + async () => + await resolveBoundSelectorCapture({ + command: 'get', + device: IOS_SIMULATOR, + session: makeIosSession('no-app'), + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + ); + + expect(withApp.ok).toBe(true); + expect(withoutApp.ok).toBe(false); +}); + +// `createBoundSelectorRuntime` is the construction path `get`, `is`, and `wait` switch onto: +// admit, bind once, then build the selector runtime with the bound operations attached. It is +// exercised here rather than from a command route because no selector descriptor has cut over +// yet — `find`'s own cutover is deferred behind the Wave 5 `focus`/`type` surfaces. +test('the bound construction path admits and binds before it builds a runtime', async () => { + const fixture = selectorCaptureFixture(); + const sessionStore = makeSessionStore(); + sessionStore.set('bound', makeAndroidSession('bound')); + + const resolved = await createBoundSelectorRuntime( + { + req: { token: 't', session: 'bound', command: 'get', positionals: [], flags: {} }, + sessionName: 'bound', + logPath: '/tmp/bound.log', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }, + { requireSession: true, command: 'get' }, + ); + + expect(resolved.ok).toBe(true); + expect(fixture.inspections).toEqual([ANDROID_EMULATOR]); + expect(fixture.binds).toEqual([ANDROID_EMULATOR]); +}); + +test('the bound construction path refuses an unavailable operation without building a runtime', async () => { + const fixture = selectorCaptureFixture({ + capture: { available: false, reason: 'unsupported-platform-leaf' }, + }); + const sessionStore = makeSessionStore(); + sessionStore.set('bound', makeAndroidSession('bound')); + + const resolved = await createBoundSelectorRuntime( + { + req: { token: 't', session: 'bound', command: 'is', positionals: [], flags: {} }, + sessionName: 'bound', + logPath: '/tmp/bound.log', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }, + { requireSession: true, command: 'is' }, + ); + + expect(resolved).toMatchObject({ ok: false, response: { ok: false } }); + expect(fixture.binds).toEqual([]); +}); diff --git a/src/daemon/__tests__/selector-capture-fixture.ts b/src/daemon/__tests__/selector-capture-fixture.ts new file mode 100644 index 000000000..2f974d5c3 --- /dev/null +++ b/src/daemon/__tests__/selector-capture-fixture.ts @@ -0,0 +1,97 @@ +import { + localRuntimeOwner, + narrowDeviceBinding, + providerRuntimeOwner, + snapshotRuntimeOperationFacts, + type CaptureSnapshotInput, + type PlatformRuntimeOperations, + type RuntimeFacts, + type RuntimeOperationFact, + type SnapshotResult, +} from '@agent-device/contracts/platform'; +import { deviceShape, type DeviceInfo } from '@agent-device/kernel/device'; +import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import { unavailableDeviceRuntimeGateway } from './test-device-runtime-gateway.ts'; + +const available: RuntimeOperationFact = Object.freeze({ available: true }); + +/** + * The request-bound capture seam every selector command (`find`, `get`, `is`, `wait`) + * consumes, faked at `inspectFacts` / `bindDevice` rather than at the legacy leaf + * dispatch. Records each bind and each capture so a test can assert the ADR 0019 §9 + * shape directly: one inspection, one bind, and every capture through the bound + * operation. + */ +export function selectorCaptureFixture( + params: Readonly<{ + capture?: RuntimeOperationFact; + withoutActiveApp?: RuntimeOperationFact; + snapshot?: (input: CaptureSnapshotInput, index: number) => SnapshotResult; + }> = {}, +): Readonly<{ + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; + inspections: DeviceInfo[]; + binds: DeviceInfo[]; + captures: CaptureSnapshotInput[]; +}> { + const inspections: DeviceInfo[] = []; + const binds: DeviceInfo[] = []; + const captures: CaptureSnapshotInput[] = []; + + const facts = async (device: DeviceInfo): Promise> => { + const base = await unavailableDeviceRuntimeGateway.inspectFacts(device); + return { + device: { + ...deviceShape(device), + providerMode: isActiveProviderDevice(device) ? 'provider-runtime' : 'local', + }, + operations: { + ...base.operations, + ...snapshotRuntimeOperationFacts({ + capture: params.capture ?? available, + customActions: { available: false, reason: 'unsupported-platform-leaf' }, + withoutActiveApp: params.withoutActiveApp ?? params.capture ?? available, + }), + }, + }; + }; + + const captureSnapshot = async (input: CaptureSnapshotInput): Promise => { + const index = captures.length; + captures.push(input); + return params.snapshot?.(input, index) ?? { nodes: [], backend: 'xctest' }; + }; + + return { + inspectFacts: async (device) => { + inspections.push(device); + return await facts(device); + }, + bindDevice: async (device, use) => { + binds.push(device); + const deviceFacts = await facts(device); + return narrowDeviceBinding( + { + device, + owner: + deviceFacts.device.providerMode === 'provider-runtime' + ? providerRuntimeOwner('test', 'selector-capture-fixture') + : localRuntimeOwner(device.platform), + facts: deviceFacts, + operations: { + captureSnapshot, + captureSnapshotWithCustomActions: captureSnapshot, + captureSnapshotWithoutActiveApp: captureSnapshot, + }, + [Symbol.asyncDispose]: async () => {}, + }, + use, + ); + }, + inspections, + binds, + captures, + }; +} diff --git a/src/daemon/handlers/__tests__/find-handler-fixture.ts b/src/daemon/handlers/__tests__/find-handler-fixture.ts new file mode 100644 index 000000000..94ce11baf --- /dev/null +++ b/src/daemon/handlers/__tests__/find-handler-fixture.ts @@ -0,0 +1,29 @@ +import type { SessionStore } from '../../session-store.ts'; +import type { DaemonRequest, DaemonResponse } from '../../types.ts'; +import { handleFindCommands } from '../find.ts'; +import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; + +/** + * One `handleFindCommands` invocation shape. + * + * Read-only `find` constructs a BOUND selector backend — it shares the element read with `get` — + * so every caller needs the request-runtime seams. It lives here rather than in `find.test.ts` + * because that file is over the module-size tripwire and may only shrink. + */ +export function invokeFindHandler(params: { + sessionName: string; + sessionStore: SessionStore; + positionals: string[]; + flags?: DaemonRequest['flags']; + invoke: (req: DaemonRequest) => Promise; +}) { + const { sessionName, sessionStore, positionals, flags } = params; + return handleFindCommands({ + req: { token: 't', session: sessionName, command: 'find', positionals, flags: flags ?? {} }, + sessionName, + logPath: '/tmp/test.log', + sessionStore, + ...getRuntimeBindings(), + invoke: params.invoke, + }); +} diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 146c0cc6a..55c12896e 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -28,9 +28,13 @@ vi.mock('../snapshot-interactor-capture.ts', async () => { import { dispatchCommand } from '../../../core/dispatch.ts'; +import { resetGetRuntimeFixture } from './interaction-get-runtime-fixture.ts'; +import { invokeFindHandler } from './find-handler-fixture.ts'; + const mockDispatch = vi.mocked(dispatchCommand); beforeEach(() => { + resetGetRuntimeFixture(); mockDispatch.mockReset(); mockDispatch.mockImplementation(async (_device: unknown, command: string) => { return command === 'snapshot' ? { nodes: [] } : {}; @@ -63,17 +67,11 @@ async function runFindClickScenario(options: { } const invokeCalls: DaemonRequest[] = []; - const response = await handleFindCommands({ - req: { - token: 't', - session: sessionName, - command: 'find', - positionals: options.positionals, - flags: options.flags ?? {}, - }, + const response = await invokeFindHandler({ sessionName, - logPath: '/tmp/test.log', sessionStore, + positionals: options.positionals, + flags: options.flags, invoke: async (req) => { invokeCalls.push(req); const data = options.invoke ? await options.invoke(req) : {}; @@ -1039,17 +1037,10 @@ test('read-only find while recording is intentionally deferred from target-v1 ev return {}; }); - const response = await handleFindCommands({ - req: { - token: 't', - session: sessionName, - command: 'find', - positionals: ['text', 'Save', 'exists'], - flags: {}, - }, + const response = await invokeFindHandler({ sessionName, - logPath: '/tmp/test.log', sessionStore, + positionals: ['text', 'Save', 'exists'], invoke: async () => ({ ok: true, data: {} }), }); @@ -1134,17 +1125,10 @@ async function runFindThroughLeaf(options: { ); const invokeCalls: DaemonRequest[] = []; - const response = await handleFindCommands({ - req: { - token: 't', - session: sessionName, - command: 'find', - positionals: options.positionals, - flags: {}, - }, + const response = await invokeFindHandler({ sessionName, - logPath: '/tmp/test.log', sessionStore, + positionals: options.positionals, invoke: async (req) => { invokeCalls.push(req); if (options.divergeBeforeDispatch) { diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index 6795cb782..9fbe562ef 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -359,6 +359,7 @@ function sourceRuntimeFacts( }), ...screenshotRuntimeOperationFacts({ capture: unavailable }), setViewport: unavailable, + readTextAtPoint: unavailable, deployApp: unavailable, materializeAppSource: materializationAvailable ? { available: true } : unavailable, deployMaterializedApp: materializationAvailable ? { available: true } : unavailable, diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts new file mode 100644 index 000000000..fd5a944f5 --- /dev/null +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -0,0 +1,129 @@ +import { vi } from 'vitest'; +import { + createUnavailablePlatformRuntimeFacts, + localRuntimeOwner, + narrowDeviceBinding, + applicationLifecycleOperationFacts, + type CaptureSnapshotInput, + type DeviceBinding, + type PlatformRuntimeOperations, + type ElementTextReadOutcome, + type ReadTextAtPointInput, + type RuntimeFacts, +} from '@agent-device/contracts/platform'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { + BindDeviceRuntime, + InspectDeviceRuntimeFacts, +} from '../../request-runtime-binding.ts'; +import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; + +/** + * The request-bound runtime seam `get` consumes, faked at `inspectFacts` / `bindDevice` — never + * at `core/dispatch.ts`. The bound capture still runs the interactor capture the surrounding + * interaction tests already mock, so only the two `get` operations are fixture-owned here. + */ +export const mockReadTextAtPoint = vi.fn( + async (_input: ReadTextAtPointInput): Promise => + Object.freeze({ status: 'unreadable', reason: 'no-text-at-point' } as const), +); + +/** + * Flip to model an exact owner cell: no live element read (web, HarmonyOS, provider), or no + * capture at all (the watchOS sentinel, an inactive provider), which refuses admission outright. + */ +export const elementReadFixtureState = { + readTextAtPointAvailable: true, + captureSnapshotAvailable: true, +}; + +export function resetGetRuntimeFixture(): void { + mockReadTextAtPoint.mockReset(); + mockReadTextAtPoint.mockResolvedValue( + Object.freeze({ status: 'unreadable', reason: 'no-text-at-point' } as const), + ); + elementReadFixtureState.readTextAtPointAvailable = true; + elementReadFixtureState.captureSnapshotAvailable = true; +} + +const available = Object.freeze({ available: true } as const); +const unavailable = Object.freeze({ + available: false, + reason: 'owner-capability-missing', +} as const); + +function elementReadFacts(device: DeviceInfo): RuntimeFacts { + const base = createUnavailablePlatformRuntimeFacts(device, localRuntimeOwner('apple'), { + appLog: unavailable, + network: unavailable, + viewport: unavailable, + elementText: unavailable, + screenshot: unavailable, + lifecycle: applicationLifecycleOperationFacts({ + resolveOpenTarget: unavailable, + prepareApplicationOpen: unavailable, + openApplication: unavailable, + applyRuntimeHints: unavailable, + clearRuntimeHints: unavailable, + closeApplication: unavailable, + finalizeApplicationClose: unavailable, + prepareAppleRunner: unavailable, + configureProviderPortReverse: unavailable, + }), + }); + return Object.freeze({ + device: base.device, + operations: { + ...base.operations, + captureSnapshot: elementReadFixtureState.captureSnapshotAvailable ? available : unavailable, + // The selector plan requires this row too on a session with no tracked app. + captureSnapshotWithoutActiveApp: elementReadFixtureState.captureSnapshotAvailable + ? available + : unavailable, + readTextAtPoint: elementReadFixtureState.readTextAtPointAvailable ? available : unavailable, + }, + }); +} + +const mockInspectElementReadFacts: InspectDeviceRuntimeFacts = vi.fn(async (device: DeviceInfo) => + elementReadFacts(device), +); + +const mockBindElementReadRuntime: BindDeviceRuntime = vi.fn(async (device: DeviceInfo, use) => { + const facts = elementReadFacts(device); + // Delegates to the interactor capture the surrounding suites already mock, so only the two + // selector operations are fixture-owned here. + const capture = async (input: CaptureSnapshotInput) => + await captureSnapshotWithInteractor({ + device, + runnerContext: { ...input.execution, appBundleId: input.options?.appBundleId }, + options: { ...input.options }, + }); + const binding: DeviceBinding = Object.freeze({ + device, + owner: localRuntimeOwner('apple'), + facts, + operations: Object.freeze({ + captureSnapshot: capture, + // The selector plan takes this row on a session with no tracked app, so an owner that + // advertises it must implement it or `narrowDeviceBinding` rejects the contract. + captureSnapshotWithoutActiveApp: capture, + ...(elementReadFixtureState.readTextAtPointAvailable + ? { readTextAtPoint: mockReadTextAtPoint } + : {}), + }), + [Symbol.asyncDispose]: async () => undefined, + }) as DeviceBinding; + return narrowDeviceBinding(binding, use); +}) as BindDeviceRuntime; + +/** + * Spread into a handler's params so a selector command can admit and bind. Consumed by `get` and + * by read-only `find`, which share the bound element read. + */ +export function getRuntimeBindings(): Readonly<{ + inspectFacts: InspectDeviceRuntimeFacts; + bindDevice: BindDeviceRuntime; +}> { + return { inspectFacts: mockInspectElementReadFacts, bindDevice: mockBindElementReadRuntime }; +} diff --git a/src/daemon/handlers/__tests__/interaction-read.test.ts b/src/daemon/handlers/__tests__/interaction-read.test.ts index 6f5414bbd..46f808ea9 100644 --- a/src/daemon/handlers/__tests__/interaction-read.test.ts +++ b/src/daemon/handlers/__tests__/interaction-read.test.ts @@ -1,18 +1,21 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; - -vi.mock('../../../core/dispatch.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - dispatchCommand: vi.fn(async () => ({ text: 'backend-text' })), - }; -}); - -import { dispatchCommand } from '../../../core/dispatch.ts'; +import { + elementTextRead, + type ElementTextReadOutcome, + type ReadTextAtPointInput, +} from '@agent-device/contracts/platform'; import { readTextForNode } from '../interaction-read.ts'; -const mockDispatch = vi.mocked(dispatchCommand); +/** + * Bound at the seam the handler consumes (the runtime's `readTextAtPoint` operation), never at + * `core/dispatch.ts`: `get` is migrated, so the live read reaches this fake through the request + * binding rather than through the legacy dispatcher. + */ +const readTextAtPoint = vi.fn( + async (_input: ReadTextAtPointInput): Promise => + elementTextRead('backend-text'), +); function node(overrides: Partial): SnapshotNode { return { @@ -27,53 +30,102 @@ const baseParams = { device: { platform: 'ios' } as never, flags: undefined, contextFromFlags: () => ({}) as never, + readTextAtPoint, }; describe('readTextForNode', () => { - beforeEach(() => mockDispatch.mockClear()); + beforeEach(() => readTextAtPoint.mockClear()); - it('returns snapshot text without a backend read for non-editable nodes', async () => { + it('returns snapshot text without a live read for non-editable nodes', async () => { const text = await readTextForNode({ ...baseParams, node: node({ type: 'button', label: 'General' }), }); expect(text).toBe('General'); - expect(mockDispatch).not.toHaveBeenCalled(); + expect(readTextAtPoint).not.toHaveBeenCalled(); }); - it('still re-reads via the backend for editable text inputs (live value may exceed snapshot)', async () => { + it('still re-reads live for editable text inputs (live value may exceed snapshot)', async () => { const text = await readTextForNode({ ...baseParams, node: node({ type: 'textfield', value: 'snap' }), }); - expect(mockDispatch).toHaveBeenCalledOnce(); + expect(readTextAtPoint).toHaveBeenCalledOnce(); expect(text).toBe('backend-text'); }); + it('reads at the resolved node center', async () => { + await readTextForNode({ ...baseParams, node: node({ type: 'textfield', value: 'snap' }) }); + expect(readTextAtPoint.mock.calls[0]?.[0].point).toEqual({ x: 50, y: 20 }); + }); + it('re-reads when the snapshot node has no readable text', async () => { await readTextForNode({ ...baseParams, node: node({ type: 'other' }) }); - expect(mockDispatch).toHaveBeenCalledOnce(); + expect(readTextAtPoint).toHaveBeenCalledOnce(); }); - it('returns snapshot text without a backend read when the node has no resolvable center', async () => { + it('returns snapshot text without a live read when the node has no resolvable center', async () => { const text = await readTextForNode({ ...baseParams, node: node({ type: 'button', label: 'General', rect: undefined }), }); expect(text).toBe('General'); - expect(mockDispatch).not.toHaveBeenCalled(); + expect(readTextAtPoint).not.toHaveBeenCalled(); }); - it('does NOT skip the backend read on non-iOS platforms (value-first read semantics differ)', async () => { + it('does NOT skip the live read on non-iOS platforms (value-first read semantics differ)', async () => { for (const platform of ['android', 'macos', 'linux'] as const) { - mockDispatch.mockClear(); + readTextAtPoint.mockClear(); const text = await readTextForNode({ ...baseParams, device: { platform } as never, node: node({ type: 'button', label: 'General' }), }); - expect(mockDispatch).toHaveBeenCalledOnce(); + expect(readTextAtPoint).toHaveBeenCalledOnce(); expect(text).toBe('backend-text'); } }); + + // The preferred-operation absence row: an owner whose facts advertise no live read answers + // entirely from the captured tree. That is the complete required path, not a fallback. + it('answers from the captured tree when the bound owner exposes no live read', async () => { + const text = await readTextForNode({ + ...baseParams, + readTextAtPoint: undefined, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + expect(readTextAtPoint).not.toHaveBeenCalled(); + }); + + // ADR 0019 §2: the ONLY fallbacks are the contract's classified reasons. + it('falls back to the captured tree for the classified reason no-text-at-point', async () => { + readTextAtPoint.mockResolvedValueOnce({ + status: 'unreadable', + reason: 'no-text-at-point', + }); + const text = await readTextForNode({ + ...baseParams, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + }); + + it('classifies a blank live read as no-text-at-point rather than reading blank text', async () => { + readTextAtPoint.mockResolvedValueOnce(elementTextRead(' ')); + const text = await readTextForNode({ + ...baseParams, + node: node({ type: 'textfield', value: 'snap' }), + }); + expect(text).toBe('snap'); + }); + + // The retired generic catch: an unclassified failure must NOT become "this element has no + // text". It propagates, so a runner/helper failure can never be answered from a stale tree. + it('propagates an unexpected live-read failure instead of falling back', async () => { + readTextAtPoint.mockRejectedValueOnce(new Error('runner transport closed')); + await expect( + readTextForNode({ ...baseParams, node: node({ type: 'textfield', value: 'snap' }) }), + ).rejects.toThrow(/runner transport closed/); + }); }); diff --git a/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts b/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts index c42901dc7..6b9f8c1e0 100644 --- a/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts +++ b/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts @@ -47,6 +47,7 @@ vi.mock('../../../platforms/apple/core/runner/runner-client.ts', async (importOr }; }); +import { getRuntimeBindings, resetGetRuntimeFixture } from './interaction-get-runtime-fixture.ts'; import { dispatchCommand } from '../../../core/dispatch.ts'; const mockDispatch = vi.mocked(dispatchCommand); @@ -57,6 +58,7 @@ beforeEach(() => { mockDispatch.mockResolvedValue({}); mockRunAppleRunnerCommand.mockReset(); mockRunAppleRunnerCommand.mockResolvedValue({}); + resetGetRuntimeFixture(); }); const SAVE_BUTTON_NODES: RawSnapshotNode[] = [ @@ -99,6 +101,7 @@ async function runCommand( sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); } diff --git a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts b/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts index a43d03118..1bc59d4be 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts @@ -2,7 +2,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { attachRefs, type SnapshotBackend } from '@agent-device/kernel/snapshot'; import { makeAndroidSession as makeBaseAndroidSession, - makeIosSession, + makeIosAppSession, makeMacOsSession as makeBaseMacOsSession, } from '../../../__tests__/test-utils/session-factories.ts'; import { makeTestScreenRecordingResource } from '../../../__tests__/test-utils/screen-recording-live-handle.ts'; @@ -10,6 +10,7 @@ import { activateCompleteRefFrame } from '../../ref-frame.ts'; import type { SessionStore } from '../../session-store.ts'; import type { SessionState } from '../../types.ts'; import { handleInteractionCommands } from '../interaction.ts'; +import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; import { buildSnapshotState } from '../../snapshot-state.ts'; /** @@ -17,8 +18,12 @@ import { buildSnapshotState } from '../../snapshot-state.ts'; * factories only: each test file installs and resets its own `vi.mock`s. */ +/** + * An iOS session WITH a tracked app: on an iOS leaf the without-active-app capture row is + * unavailable, so a selector command against an app-less session is refused before it captures. + */ export function makeSession(name: string): SessionState { - return makeIosSession(name); + return makeIosAppSession(name); } export function makeAndroidSession(name: string): SessionState { @@ -125,6 +130,7 @@ export async function runInteraction( sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); } diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index cf7402d10..4b30900b9 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -1,5 +1,4 @@ import { test, expect, vi, beforeEach } from 'vitest'; -import { AppError } from '@agent-device/kernel/errors'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { WEB_DESKTOP_DEVICE } from '../../../__tests__/test-utils/device-fixtures.ts'; import { @@ -58,6 +57,13 @@ vi.mock('../../../platforms/apple/core/runner/runner-client.ts', async (importOr }; }); +import { elementTextRead } from '@agent-device/contracts/platform'; +import { + elementReadFixtureState, + getRuntimeBindings, + mockReadTextAtPoint, + resetGetRuntimeFixture, +} from './interaction-get-runtime-fixture.ts'; import { dispatchCommand } from '../../../core/dispatch.ts'; const mockDispatch = vi.mocked(dispatchCommand); import { @@ -75,6 +81,7 @@ beforeEach(() => { mockGetAndroidBlockingDialogFocus.mockResolvedValue(null); mockRunAppleRunnerCommand.mockReset(); mockRunAppleRunnerCommand.mockResolvedValue({}); + resetGetRuntimeFixture(); }); test('get text prefers underlying value for text surfaces and avoids recording giant ref labels', async () => { @@ -111,6 +118,7 @@ test('get text prefers underlying value for text surfaces and avoids recording g sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); @@ -145,10 +153,9 @@ test('get text uses backend read expansion when the resolved node has a rect', a }; sessionStore.set(sessionName, session); - mockDispatch.mockResolvedValue({ - action: 'read', - text: 'package com.example.app\nclass MainActivity {}', - }); + mockReadTextAtPoint.mockResolvedValue( + elementTextRead('package com.example.app\nclass MainActivity {}'), + ); const response = await handleInteractionCommands({ req: { @@ -161,28 +168,123 @@ test('get text uses backend read expansion when the resolved node has a rect', a sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); - expect(mockDispatch).toHaveBeenCalledTimes(1); - expect(mockDispatch.mock.calls[0]?.[1]).toBe('read'); - expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['80', '80']); + // The live read now reaches the bound runtime operation, not the legacy `read` dispatch. + expect(mockDispatch).not.toHaveBeenCalled(); + expect(mockReadTextAtPoint).toHaveBeenCalledTimes(1); + expect(mockReadTextAtPoint.mock.calls[0]?.[0].point).toEqual({ x: 80, y: 80 }); expect(response?.ok).toBe(true); if (response?.ok) { expect(response.data?.text).toBe('package com.example.app\nclass MainActivity {}'); } }); -test('get text simple iOS id selector uses runner query without snapshot', async () => { +test('get text answers from the captured tree when the bound owner advertises no live read', async () => { const sessionStore = makeSessionStore(); - const sessionName = 'get-text-ios-direct-selector'; + const sessionName = 'get-text-no-live-read'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + depth: 0, + type: 'TextView', + label: 'Editor for MainActivity.kt', + value: 'preview only', + rect: { x: 20, y: 40, width: 120, height: 80 }, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + elementReadFixtureState.readTextAtPointAvailable = false; + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'get', + positionals: ['text', '@e1'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + // Preferred-operation absence is not a failure and not a fallback: the required capture path + // answers completely, and nothing reaches the legacy dispatcher. + expect(mockReadTextAtPoint).not.toHaveBeenCalled(); + expect(mockDispatch).not.toHaveBeenCalled(); + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data?.text).toBe('preview only'); + } +}); + +// ADR 0019 regression: `get` declares `device-runtime`, so an ELIGIBLE direct-iOS selector — +// one the fast path would otherwise answer without a tree capture — must not reach the device +// until the request has resolved, admitted, and bound. A refused admission means zero runner +// queries, not a fast-path answer that skipped exact-owner facts entirely. +test('an eligible direct iOS selector cannot operate before admission', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'get-text-direct-before-admission'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + elementReadFixtureState.captureSnapshotAvailable = false; mockRunAppleRunnerCommand.mockResolvedValue({ found: true, text: 'Ada Lovelace', + nodes: [{ index: 0, depth: 0, type: 'StaticText', label: 'Ada Lovelace' }], + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'get', + positionals: ['text', 'id=name'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(false); + if (response && !response.ok) expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); + // The whole point: the fast path never ran. + expect(mockRunAppleRunnerCommand).not.toHaveBeenCalled(); + expect(mockDispatch).not.toHaveBeenCalled(); +}); + +// The direct-iOS shortcut is RETIRED (#1739): `get` declares `device-runtime`, so a simple +// `id=` selector resolves through the bound capture like every other shape rather than through a +// raw runner query R36 declares no operation for. The cost is real and accepted — this selector +// no longer skips the tree capture. +test('get text simple iOS id selector resolves through the bound capture, not a runner query', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'get-text-ios-direct-selector'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + mockDispatch.mockResolvedValue({ + backend: 'xctest', nodes: [ { index: 0, depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 393, height: 852 }, + enabled: true, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, type: 'TextField', label: 'Name', identifier: 'field-name', @@ -205,32 +307,20 @@ test('get text simple iOS id selector uses runner query without snapshot', async sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); - expect(mockRunAppleRunnerCommand).toHaveBeenCalledWith( - expect.anything(), - { - command: 'querySelector', - selectorKey: 'id', - selectorValue: 'field-name', - appBundleId: 'com.example.app', - }, - expect.anything(), - ); - expect(mockDispatch).not.toHaveBeenCalledWith( - expect.anything(), - 'snapshot', - expect.anything(), - expect.anything(), - expect.anything(), - ); if (response?.ok) { expect(response.data?.text).toBe('Ada Lovelace'); expect(response.data?.selector).toBe('id="field-name"'); } - const recorded = sessionStore.get(sessionName)?.actions.at(-1); - expect(recorded?.result?.selectorChain).toEqual(['id="field-name"']); + // No querySelector: the retired bypass was the only caller on this path. + expect(mockRunAppleRunnerCommand).not.toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ command: 'querySelector' }), + expect.anything(), + ); }); test('get text iOS label selector uses snapshot disambiguation instead of runner query', async () => { @@ -292,6 +382,7 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -304,40 +395,6 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner } }); -test('get text simple iOS id selector does not snapshot-fallback on ambiguous runner match', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'get-text-ios-direct-selector-ambiguous'; - sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockRunAppleRunnerCommand.mockRejectedValue( - new AppError('AMBIGUOUS_MATCH', 'selector matched multiple elements'), - ); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'get', - positionals: ['text', 'id="field-name"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - }); - - expect(response?.ok).toBe(false); - if (response?.ok === false) { - expect(response.error.code).toBe('AMBIGUOUS_MATCH'); - } - expect(mockDispatch).not.toHaveBeenCalledWith( - expect.anything(), - 'snapshot', - expect.anything(), - expect.anything(), - expect.anything(), - ); -}); - test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { const sessionStore = makeSessionStore(); const sessionName = 'snapshot-flags'; @@ -382,6 +439,7 @@ test('is visible preserves CLI snapshot flags during runtime snapshot capture', sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -413,6 +471,7 @@ test('is visible reuses fresh cached iOS snapshots with rects', async () => { sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -445,6 +504,7 @@ test('is visible recaptures web snapshots when cached nodes may lack rects', asy sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -453,68 +513,14 @@ test('is visible recaptures web snapshots when cached nodes may lack rects', asy }); }); -test('is selected simple iOS id selector uses runner query without snapshot', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'is-selected-ios-direct-selector'; - sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockRunAppleRunnerCommand.mockResolvedValue({ - found: true, - text: 'Pickup', - nodes: [ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Pickup', - identifier: 'shipping-pickup', - selected: true, - rect: { x: 126, y: 555, width: 75, height: 38 }, - enabled: true, - hittable: true, - }, - ], - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['selected', 'id="shipping-pickup"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - }); - - expect(response?.ok).toBe(true); - expect(mockRunAppleRunnerCommand).toHaveBeenCalledWith( - expect.anything(), - { - command: 'querySelector', - selectorKey: 'id', - selectorValue: 'shipping-pickup', - appBundleId: 'com.example.app', - }, - expect.anything(), - ); - expect(mockDispatch).not.toHaveBeenCalledWith( - expect.anything(), - 'snapshot', - expect.anything(), - expect.anything(), - expect.anything(), - ); - if (response?.ok) { - expect(response.data?.predicate).toBe('selected'); - expect(response.data?.pass).toBe(true); - } - const recorded = sessionStore.get(sessionName)?.actions.at(-1); - expect(recorded?.result?.selectorChain).toEqual(['id="shipping-pickup"']); -}); - -test('is simple iOS selector returns false directly when runner predicate fails', async () => { +// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots +// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero +// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit +// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; +// the shortcut is now retired outright, so the bound capture answers every predicate and this is +// simply what `is` does. The assertion below is unchanged across both edits because it was always +// about the OUTCOME, not about which path produced it. +test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { const sessionStore = makeSessionStore(); const sessionName = 'is-selected-ios-direct-selector-false'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); @@ -544,68 +550,18 @@ test('is simple iOS selector returns false directly when runner predicate fails' sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); - expect(response?.ok).toBe(true); - expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0); - if (response?.ok) { - expect(response.data?.predicate).toBe('selected'); - expect(response.data?.pass).toBe(false); + // The session snapshot has no `id=submit`, so the bound capture reports the typed selector + // failure. Nothing can report a failed assertion as a completed command. + expect(response?.ok).toBe(false); + expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); + if (response?.ok === false) { + expect(response.error?.code).toBe('COMMAND_FAILED'); } }); -test('is simple iOS selector falls back to snapshot while gesture stabilization is pending', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'is-selected-ios-stabilizing'; - const session = makeIosSession(sessionName, { appBundleId: 'com.example.app' }); - session.postGestureStabilization = { action: 'swipe', positionals: [], markedAt: Date.now() }; - sessionStore.set(sessionName, session); - - mockDispatch.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { - index: 0, - depth: 0, - type: 'Window', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Pickup', - identifier: 'shipping-pickup', - selected: true, - rect: { x: 126, y: 555, width: 75, height: 38 }, - enabled: true, - hittable: true, - }, - ], - backend: 'xctest', - }; - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['selected', 'id="shipping-pickup"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - }); - - expect(response?.ok).toBe(true); - expect(mockRunAppleRunnerCommand).not.toHaveBeenCalled(); - expect(mockDispatch.mock.calls.some((call) => call[1] === 'snapshot')).toBe(true); -}); - test('is visible passes for list text that inherits viewport visibility from an ancestor', async () => { const sessionStore = makeSessionStore(); const sessionName = 'visible-list-item'; @@ -646,6 +602,7 @@ test('is visible passes for list text that inherits viewport visibility from an sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); @@ -691,6 +648,7 @@ test('is visible fails for nodes outside the current viewport', async () => { sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); @@ -729,6 +687,7 @@ test('is reports Android permission dialog blocker when app content assertion fa sessionName, sessionStore, contextFromFlags, + ...getRuntimeBindings(), }); expect(response).toBeTruthy(); diff --git a/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts b/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts index 423965078..759a3154b 100644 --- a/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts +++ b/src/daemon/handlers/__tests__/session-capabilities.fixtures.ts @@ -84,6 +84,7 @@ function createAdmissionFacts( deployMaterializedApp: cell(options.sourceAvailable), sendPushNotification: cell(options.pushAvailable), networkDump: cell(options.networkAvailable), + readTextAtPoint: unavailable, screenRecordingStart: unavailable, screenRecordingReattach: unavailable, screenRecordingCleanup: unavailable, diff --git a/src/daemon/handlers/__tests__/session-command-harness.ts b/src/daemon/handlers/__tests__/session-command-harness.ts index b762f1d35..3b8f90b6a 100644 --- a/src/daemon/handlers/__tests__/session-command-harness.ts +++ b/src/daemon/handlers/__tests__/session-command-harness.ts @@ -145,6 +145,7 @@ function readinessFacts(device: DeviceInfo): RuntimeFacts { network: { available: false, reason: 'owner-capability-missing' }, screenshot: { available: false, reason: 'owner-capability-missing' }, viewport: { available: false, reason: 'owner-capability-missing' }, + elementText: { available: false, reason: 'owner-capability-missing' }, readiness: { available: false, reason: 'unsupported-device-kind' }, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: { available: false, reason: 'owner-capability-missing' }, @@ -131,6 +132,7 @@ test('appstate rejects web before Android app-state backend dispatch', async () network: { available: false, reason: 'unsupported-platform-leaf' }, screenshot: { available: false, reason: 'unsupported-platform-leaf' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, + elementText: { available: false, reason: 'unsupported-platform-leaf' }, readiness: { available: false, reason: 'unsupported-platform-leaf' }, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: { available: false, reason: 'unsupported-platform-leaf' }, diff --git a/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts index 4919ef60a..aed6b991c 100644 --- a/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts +++ b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts @@ -1,5 +1,6 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { handleFindCommands } from '../find.ts'; +import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; import { dispatchFindReadOnlyViaRuntime, dispatchWaitViaRuntime } from '../../selector-runtime.ts'; import type { DaemonRequest, DaemonResponse } from '../../types.ts'; import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../core/android-system-surface-disclosure.ts'; @@ -105,6 +106,7 @@ test('read-only find exists on a system-surface capture discloses the occlusion' sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -130,6 +132,7 @@ test('wait timeout for app text hidden behind a system surface discloses the occ sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response.ok).toBe(false); @@ -153,6 +156,7 @@ test('sessionless read-only find still discloses the occluding system surface', sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response?.ok).toBe(true); @@ -213,6 +217,7 @@ test('sessionless wait success on shade content still discloses the occluding sy sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response.ok).toBe(true); @@ -238,6 +243,7 @@ test('sessionless wait timeout still discloses the occluding system surface', as sessionName: 'default', logPath: '/tmp/test.log', sessionStore, + ...getRuntimeBindings(), }); expect(response.ok).toBe(false); diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 89184d80a..82b38d6a0 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -20,6 +20,7 @@ import { recordSessionAction } from './handler-utils.ts'; import { stripInternalInteractionFlags } from '../interaction-outcome-policy.ts'; import { resolveFindMatch } from './find-match-resolution.ts'; import { dispatchFindReadOnlyViaRuntime } from '../selector-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; import { createFindTargetCapture, sparseFindSnapshotResponse } from './find-target-capture.ts'; import { isSparseSnapshotQualityVerdict } from '../../snapshot-quality/verdict.ts'; @@ -57,6 +58,8 @@ export async function handleFindCommands(params: { logPath: string; sessionStore: SessionStore; invoke: DaemonInvokeFn; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; }): Promise { const { req, sessionName, logPath, sessionStore, invoke } = params; const command = req.command; @@ -83,6 +86,8 @@ export async function handleFindCommands(params: { sessionName, logPath, sessionStore, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }); if (runtimeResponse) return runtimeResponse; // Read-only find actions (exists/wait/list/get_text/get_attrs) always return from diff --git a/src/daemon/handlers/interaction-common.ts b/src/daemon/handlers/interaction-common.ts index 104c222c7..8edd16f32 100644 --- a/src/daemon/handlers/interaction-common.ts +++ b/src/daemon/handlers/interaction-common.ts @@ -12,6 +12,7 @@ import { inferFillText } from '../action-utils.ts'; import { recordedInputPlaceholder } from '../../replay/recorded-input.ts'; import { parameterizeRecordedFillPayload } from '../parameterized-recorded-fill.ts'; import { isSessionRecording } from '../session-script-publication-capability.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; export type ContextFromFlags = ( flags: CommandFlags | undefined, @@ -25,6 +26,8 @@ export type InteractionHandlerParams = { logPath?: string; sessionStore: SessionStore; contextFromFlags: ContextFromFlags; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; }; export function finalizeTouchInteraction(params: { diff --git a/src/daemon/handlers/interaction-read.ts b/src/daemon/handlers/interaction-read.ts index 7f69b6945..2294e1beb 100644 --- a/src/daemon/handlers/interaction-read.ts +++ b/src/daemon/handlers/interaction-read.ts @@ -1,6 +1,10 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { + ElementTextRuntimeOperations, + ElementTextUnreadableReason, +} from '@agent-device/contracts/platform'; import { isIosFamily } from '@agent-device/kernel/device'; -import { dispatchCommand } from '../../core/dispatch.ts'; +import { runtimeExecutionFromContext } from '../snapshot-runtime-capture-input.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { SessionState } from '../types.ts'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; @@ -8,6 +12,14 @@ import { extractReadableText, prefersValueForReadableText } from '../../utils/te import type { ContextFromFlags } from './interaction-common.ts'; import { resolveRectCenter } from './interaction-targeting.ts'; +export type ReadElementTextAtPoint = ElementTextRuntimeOperations['readTextAtPoint']; + +/** + * When a selector read consults the owner's live point read, and what it does with the answer. + * This module owns that policy only: the read itself is injected, so nothing here names a + * platform. A migrated command passes its bound `readTextAtPoint` — including passing nothing + * when its selected owner's facts report no live read, which is the complete required path. + */ export async function readTextForNode(params: { device: SessionState['device']; node: SnapshotNode; @@ -15,16 +27,20 @@ export async function readTextForNode(params: { appBundleId?: string; traceOutPath?: string; surface?: SessionState['surface']; + readTextAtPoint?: ReadElementTextAtPoint; contextFromFlags: ContextFromFlags; }): Promise { const { device, node, flags, appBundleId, traceOutPath, surface, contextFromFlags } = params; const fallbackText = extractReadableText(node); + const readTextAtPoint = params.readTextAtPoint; + // No live read on this owner: the captured tree is the whole answer, with nothing to disclose. + if (!readTextAtPoint) return fallbackText; const center = resolveRectCenter(node.rect); if (!center) { return fallbackText; } - // iOS only: the XCUITest backend `read` re-resolves the element at a point by enumerating + // iOS only: the XCUITest backend `readText` re-resolves the element at a point by enumerating // the full element tree (allElementsBoundByIndex), which is ~20x slower than the snapshot we // already captured to resolve this node. That re-read only recovers fuller text for // editable/expandable inputs (textField/searchField/textView/…), where the live value can @@ -36,45 +52,41 @@ export async function readTextForNode(params: { return fallbackText; } - try { - const rawData = await dispatchCommand( - device, - 'read', - [String(center.x), String(center.y)], - undefined, - { - ...contextFromFlags(flags, appBundleId, traceOutPath), - surface, - }, - ); - const data = rawData && typeof rawData === 'object' ? rawData : undefined; - const text = typeof data?.text === 'string' ? data.text : ''; - if (text.trim()) { - return text; + const context = contextFromFlags(flags, appBundleId, traceOutPath); + // No try/catch: an unexpected read failure propagates. Only the outcomes the contract + // classifies fall back to the captured tree (ADR 0019 §2 typed reason), so a runner or + // helper failure can never masquerade as "this element has no text". + const outcome = await readTextAtPoint({ + point: center, + options: { appBundleId, surface }, + execution: runtimeExecutionFromContext(context), + }); + if (outcome.status === 'read') return outcome.text; + emitDiagnostic({ + level: 'warn', + phase: 'interaction_read_fallback', + data: { + reason: classifiedFallbackReason(outcome.reason), + nodeRef: node.ref, + surface, + platform: device.platform, + }, + }); + return fallbackText; +} + +/** + * The typed reason a fallback to the captured tree is allowed, one diagnostic reason per + * classified outcome. The `satisfies never` arm makes a new `ElementTextUnreadableReason` + * a COMPILE error here rather than a silent untyped fallback. + */ +function classifiedFallbackReason(reason: ElementTextUnreadableReason): string { + switch (reason) { + case 'no-text-at-point': + return 'no_text_at_point'; + default: { + const unhandled: never = reason; + return unhandled; } - emitDiagnostic({ - level: 'warn', - phase: 'interaction_read_fallback', - data: { - reason: 'empty_backend_text', - nodeRef: node.ref, - surface, - platform: device.platform, - }, - }); - return fallbackText; - } catch (error) { - emitDiagnostic({ - level: 'warn', - phase: 'interaction_read_fallback', - data: { - reason: 'backend_read_failed', - nodeRef: node.ref, - surface, - platform: device.platform, - error: error instanceof Error ? error.message : String(error), - }, - }); - return fallbackText; } } diff --git a/src/daemon/request-handler-chain.ts b/src/daemon/request-handler-chain.ts index 509696a94..2b757fc8d 100644 --- a/src/daemon/request-handler-chain.ts +++ b/src/daemon/request-handler-chain.ts @@ -224,6 +224,8 @@ async function runFindHandler( logPath: params.logPath, sessionStore: params.sessionStore, invoke: params.invoke, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }), ); } @@ -241,6 +243,8 @@ async function runInteractionHandler( logPath: params.logPath, sessionStore: params.sessionStore, contextFromFlags: params.contextFromFlags, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, }), ); } diff --git a/src/daemon/screenshot-runtime.ts b/src/daemon/screenshot-runtime.ts index 40e63ebb4..9da78cba1 100644 --- a/src/daemon/screenshot-runtime.ts +++ b/src/daemon/screenshot-runtime.ts @@ -18,6 +18,7 @@ import { assertSupportedScreenshotPixelDensity, readScreenshotResultMetadata, } from '../utils/screenshot-density.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; import type { DaemonCommandContext } from './context.ts'; import { captureSnapshotData } from './handlers/snapshot-capture.ts'; import { buildSnapshotState } from './snapshot-state.ts'; @@ -123,20 +124,17 @@ export async function captureScreenshotArtifact( */ type CapturedScreenshot = Readonly<{ path: string; message?: string }>; -/** Runner metadata the capture needs; cancellation comes from the request binding, not from here. */ +/** + * Runner metadata the capture needs; cancellation comes from the request binding, not from here. + * + * `ScreenshotRuntimeExecution` and `SnapshotRuntimeExecution` are the same type — both + * `Readonly>` — so the projection is shared rather + * than restated. Keeping the screenshot-facing name preserves this module's callers. + */ export function screenshotExecutionFromContext( context: DaemonCommandContext, ): ScreenshotRuntimeExecution { - return { - requestId: context.requestId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }; + return runtimeExecutionFromContext(context); } async function executeScreenshot( diff --git a/src/daemon/selector-capture-binding.ts b/src/daemon/selector-capture-binding.ts new file mode 100644 index 000000000..48c259e14 --- /dev/null +++ b/src/daemon/selector-capture-binding.ts @@ -0,0 +1,72 @@ +import { + resolveSelectorCaptureRuntimePlan, + type CaptureSnapshotInput, + type ElementTextRuntimeOperations, + type SnapshotResult, +} from '@agent-device/contracts/platform'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; +import { admitAndBindSnapshotCapture } from './snapshot-runtime-binding.ts'; +import type { DaemonResponse, SessionState } from './types.ts'; + +/** The selector commands that resolve their targets from a request-bound capture. */ +export type SelectorCaptureCommand = 'find' | 'get' | 'is' | 'wait'; + +/** + * One request's bound accessibility capture. Selector commands capture repeatedly under one + * binding (polling, sparse recovery), so the operation is parametrized by intent rather than + * frozen at bind time. + */ +export type BoundSelectorCapture = (input: CaptureSnapshotInput) => Promise; + +/** + * The owner's live element-text read, when its facts advertise one. Optional because it is a + * PREFERRED operation: every selector read's required path answers from the captured tree, so an + * owner without it still executes the command completely (ADR 0019 §2). + */ +export type BoundSelectorRead = ElementTextRuntimeOperations['readTextAtPoint']; + +/** + * The bound operations a selector command's runtime executes through. A record rather than a bare + * capture function so a unit can add its own bound operation without changing any signature on + * this seam — which is how `readText` arrived, and how the next one will. + */ +export type BoundSelectorOperations = Readonly<{ + capture: BoundSelectorCapture; + readText?: BoundSelectorRead; +}>; + +export type ResolvedSelectorCapture = + | Readonly<{ ok: true; operations: BoundSelectorOperations }> + | Readonly<{ ok: false; response: DaemonResponse }>; + +/** + * The selector family's entry to the shared admit-then-bind path: it contributes the + * active-app plan and its command name for the refusal wording, and inherits one inspection, + * refusal-before-bind, and one binding. A sibling unit migrates by naming its command here. + */ +export async function resolveBoundSelectorCapture( + params: Readonly<{ + command: SelectorCaptureCommand; + device: SessionState['device']; + session: SessionState | undefined; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; + }>, +): Promise { + const bound = await admitAndBindSnapshotCapture({ + ...params, + plan: resolveSelectorCaptureRuntimePlan({ + hasActiveApp: params.session?.appBundleId !== undefined, + }), + }); + if (!bound.ok) return bound; + // The read is present only when the admitted owner advertised it; its absence is not a failure + // and not a fallback — every selector read's required path answers from the captured tree. + return { + ok: true, + operations: { + capture: bound.capture, + ...(bound.readTextAtPoint ? { readText: bound.readTextAtPoint } : {}), + }, + }; +} diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index fd71d97fd..6eda22b0f 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -12,10 +12,12 @@ import { captureSnapshot } from './handlers/snapshot-capture.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import { getActiveAndroidSnapshotFreshness } from './android-snapshot-freshness.ts'; import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts'; +import type { BoundSelectorCapture } from './selector-capture-binding.ts'; +import { buildRuntimeCaptureInput } from './snapshot-runtime-capture-input.ts'; const SELECTOR_CAPTURE_CACHE_TTL_MS = 750; -type SelectorCaptureRuntimeParams = { +export type SelectorCaptureRuntimeParams = { device: SessionState['device']; session: SessionState | undefined; sessionStore: SessionStore; @@ -25,6 +27,12 @@ type SelectorCaptureRuntimeParams = { // Sessionless routes have no session record to read the consumed capture back from, so the // capture runtime reports every consumed snapshot here for response-level disclosures. consumedSnapshot?: { state?: SnapshotState }; + /** + * The request-bound capture from `resolveBoundSelectorCapture`: every cache tier, recovery + * re-capture, and poll below reaches the platform through it. Selector commands still on + * legacy admission pass nothing; the last one to migrate makes this required. + */ + capture?: BoundSelectorCapture; }; /** @@ -170,18 +178,35 @@ async function runCapture( snapshotScope: string | undefined, interactiveOnly = request.flags?.snapshotInteractiveOnly, ): Promise { + const flags = { + ...request.flags, + snapshotInteractiveOnly: interactiveOnly, + }; + const boundCapture = params.capture; const capture = await captureSnapshot({ device: params.device, session: params.session, - flags: { - ...request.flags, - snapshotInteractiveOnly: interactiveOnly, - }, + flags, outPath: request.outPath ?? params.req.flags?.out, logPath: params.logPath ?? '', snapshotScope, includeRects: request.includeRects, signal: request.signal, + ...(boundCapture === undefined + ? {} + : { + captureData: async () => + await boundCapture( + buildRuntimeCaptureInput({ + flags, + logPath: params.logPath ?? '', + meta: params.req.meta, + session: params.session, + snapshotScope, + includeRects: request.includeRects, + }), + ), + }), }); return capture.snapshot; } diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 170ed407d..7c0158477 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -2,7 +2,7 @@ import type { AgentDeviceBackend, BackendSnapshotResult } from '../backend.ts'; import { resolveTargetDevice } from '../core/dispatch.ts'; import { createAgentDevice } from '../runtime.ts'; import { isMacOs, isApplePlatform, publicPlatformString } from '@agent-device/kernel/device'; -import { noActiveSessionError, requireCommandSupported } from './handlers/response.ts'; +import { noActiveSessionError } from './handlers/response.ts'; import type { SnapshotState, SnapshotNode } from '@agent-device/kernel/snapshot'; import { findNodeByLabel } from '../core/snapshot-node-lookup.ts'; import { runAppleRunnerCommand } from '../platforms/apple/core/runner/runner-client.ts'; @@ -17,6 +17,12 @@ import type { ContextFromFlags } from './handlers/interaction-common.ts'; import { SessionStore } from './session-store.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; import { createSelectorCaptureRuntime } from './selector-capture-runtime.ts'; +import { + resolveBoundSelectorCapture, + type BoundSelectorOperations, + type SelectorCaptureCommand, +} from './selector-capture-binding.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from './request-runtime-binding.ts'; import { isActiveProviderDevice } from '../provider-device-runtime.ts'; import { getRequestSignal } from '../request/cancel.ts'; import { snapshotOptionsToFlags } from '../backend-snapshot-options.ts'; @@ -31,13 +37,26 @@ export type SelectorRuntimeParams = { // sessionless routes disclose from here because no session record stores the capture. consumedSnapshot?: { state?: SnapshotState }; signal?: AbortSignal; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; }; -type SelectorRuntimeDeviceParams = SelectorRuntimeParams & { +export type SelectorRuntimeDeviceParams = SelectorRuntimeParams & { session: SessionState | undefined; device: SessionState['device']; + /** The request-bound operations this runtime executes through. Absent for selector + * commands still on legacy admission, until their own ADR 0019 unit lands. */ + bound?: BoundSelectorOperations; }; +type ResolvedSelectorRuntime = + | { ok: true; runtime: ReturnType } + | { ok: false; response: DaemonResponse }; + +type ResolvedSelectorDevice = + | { ok: true; session: SessionState | undefined; device: SessionState['device'] } + | { ok: false; response: DaemonResponse }; + type AppleRunnerFindTextTarget = { device: SessionState['device']; appBundleId: string; @@ -62,37 +81,67 @@ export function createSelectorRuntimeForDevice(params: SelectorRuntimeDevicePara }); } -export async function createSelectorRuntime( +/** The session/device a selector command runs against, before any admission decides. */ +async function resolveSelectorRuntimeDevice( params: SelectorRuntimeParams, - options: { requireSession: boolean; capability: 'find' | 'get' | 'is' }, -): Promise< - | { ok: true; runtime: ReturnType } - | { ok: false; response: DaemonResponse } -> { + requireSession: boolean, +): Promise { params.consumedSnapshot ??= {}; const session = params.sessionStore.get(params.sessionName); - if (!session && options.requireSession) { - return { - ok: false, - response: noActiveSessionError(), - }; - } + if (!session && requireSession) return { ok: false, response: noActiveSessionError() }; const device = session?.device ?? (await resolveTargetDevice(params.req.flags ?? {})); if (!session) await ensureDeviceReady(device); - const unsupported = requireCommandSupported(options.capability, device); - if (unsupported) return { ok: false, response: unsupported }; + return { ok: true, session, device }; +} + +/** + * THE selector runtime: facts-first admission, exactly one binding, and a backend whose every + * capture goes through the bound operation. Since `is` (R37) there is no other one — the legacy + * capability-admitted `createSelectorRuntime` and its `requireCommandSupported` call were its + * last consumer and retired with it, so a selector command cannot reach the device on a + * capability bucket even by mistake. + * + * ADR 0019 §6: a `device-runtime` command reaches the device only after resolve -> admit -> + * bind, so THIS CALL COMES FIRST in its route — ahead of every shortcut, including the + * direct-iOS selector query that answers some targets without a capture. That query is a fast + * path *within* an admitted request, never a way around exact-owner facts or the one-binding + * invariant. `get` (R36) and `is` (R37) both order it this way. + */ +export async function createBoundSelectorRuntime( + params: SelectorRuntimeParams, + options: { requireSession: boolean; command: SelectorCaptureCommand }, +): Promise { + const resolved = await resolveSelectorRuntimeDevice(params, options.requireSession); + if (!resolved.ok) return resolved; + const bound = await resolveBoundSelectorCapture({ + command: options.command, + device: resolved.device, + session: resolved.session, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!bound.ok) return { ok: false, response: bound.response }; return { ok: true, runtime: createSelectorRuntimeForDevice({ ...params, - session, - device, + session: resolved.session, + device: resolved.device, + bound: bound.operations, }), }; } function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDeviceBackend { + // The bound operation is the ONLY element read. Both consumers of the shared backend read — + // `get text` and read-only `find … get text` — construct a bound backend, so there is no second + // read path to choose between and nothing reaches the retired `read` dispatch. const { req, session, device, logPath, sessionName, sessionStore } = params; + const resolveContextFromFlags: ContextFromFlags = + params.contextFromFlags ?? + ((flags, appBundleId, traceLogPath) => + contextFromFlags(logPath ?? '', flags, appBundleId, traceLogPath)); + const readTextAtPoint = params.bound?.readText; const captureRuntime = createSelectorCaptureRuntime({ device, session, @@ -101,6 +150,7 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice req, consumedSnapshot: params.consumedSnapshot, logPath, + capture: params.bound?.capture, }); return { platform: publicPlatformString(device), @@ -129,16 +179,14 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice }, readText: async (_context, node: SnapshotNode) => ({ text: await readTextForNode({ + readTextAtPoint, device, node, flags: req.flags, appBundleId: session?.appBundleId, traceOutPath: session?.trace?.outPath, surface: session?.surface, - contextFromFlags: - params.contextFromFlags ?? - ((flags, appBundleId, traceLogPath) => - contextFromFlags(logPath ?? '', flags, appBundleId, traceLogPath)), + contextFromFlags: resolveContextFromFlags, }), }), findText: async (context, text) => ({ diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 9626763e5..1f255ecb2 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -17,9 +17,7 @@ import { checkIsArgs, checkWaitText, checkFindArgs, - evaluateIsPredicate, isReadOnlyFindAction, - type IsPredicate, } from '@agent-device/selectors'; import { refSnapshotFlagGuardResponse } from './handlers/interaction-flags.ts'; import { parseVersionedRefPositional } from './handlers/interaction-touch-targets.ts'; @@ -48,7 +46,7 @@ import { } from './direct-ios-selector.ts'; import { isSessionRecording } from './session-script-publication-capability.ts'; import { - createSelectorRuntime, + createBoundSelectorRuntime, createSelectorRuntimeForDevice, type SelectorRuntimeParams, } from './selector-runtime-backend.ts'; @@ -66,15 +64,6 @@ type DirectIosSelectorFallbackResult = | DirectIosSelectorErrorResult | null; -type ResolvedDirectIosSelectorQuery = - | { - session: SessionState; - selector: DirectIosSelectorTarget; - result: DirectIosSelectorQueryResult; - } - | DirectIosSelectorErrorResult - | null; - export async function dispatchFindReadOnlyViaRuntime( params: SelectorRuntimeParams, ): Promise { @@ -87,9 +76,13 @@ export async function dispatchFindReadOnlyViaRuntime( const action = parsed.action; if (!isReadOnlyFindAction(action)) return null; - const resolvedRuntime = await createSelectorRuntime(params, { + // Read-only `find` shares the element read with `get`, so it constructs a BOUND backend and + // the two consume one bound operation instead of one binding it and the other dispatching the + // legacy `read`. This moves find's READ LEG only: find's descriptor stays + // `LEGACY_PLATFORM_EXECUTION`, it claims no cutover row, and its mutating actions are untouched. + const resolvedRuntime = await createBoundSelectorRuntime(params, { requireSession: false, - capability: 'find', + command: 'find', }); if (!resolvedRuntime.ok) return resolvedRuntime.response; @@ -165,17 +158,20 @@ export async function dispatchGetViaRuntime( // ADR 0012 step 4: a guarded replay dispatch must resolve through the // snapshot path so the post-resolution identity guard runs. const replayTargetGuard = req.internal?.replayTargetGuard; - if (target.target.kind === 'selector' && !replayTargetGuard) { - const directResponse = await dispatchDirectIosSelectorGet(params, sub, target.target.selector); - if (directResponse) return directResponse; - } - const resolvedRuntime = await createSelectorRuntime(params, { + // ADR 0019: `get` declares `device-runtime`, so its request path reaches the device ONLY + // through operations R36 declares. Every target shape — including the simple iOS `id=` selector + // a direct runner query used to answer without a capture — resolves through the bound capture. + // Admission before a bypass is not the same as executing through the seam, so the bypass is + // gone rather than merely ordered after admission. + const resolvedRuntime = await createBoundSelectorRuntime(params, { requireSession: true, - capability: 'get', + command: 'get', }); if (!resolvedRuntime.ok) return resolvedRuntime.response; + const runtime = resolvedRuntime.runtime; + // #1076 + ADR 0014: a get @ref binds against the retained ref-frame evidence, // so it never silently retargets to a newer positional tree. Its warning is // frame-derived: once the ref frame has expired any ref gets the frame-derived @@ -190,7 +186,7 @@ export async function dispatchGetViaRuntime( }) : undefined; const response = await toDaemonResponse(async () => { - const result = await resolvedRuntime.runtime.selectors.get({ + const result = await runtime.selectors.get({ session: params.sessionName, requestId: req.meta?.requestId, property: sub, @@ -226,26 +222,19 @@ export async function dispatchIsViaRuntime( checked.hint ? { hint: checked.hint } : undefined, ); } - const { predicate, expectedText } = checked; - const split = { selectorExpression: checked.selectorExpression }; - // ADR 0012 decision 3 / #1349: recording and a guarded replay dispatch both - // require the snapshot path — evidence and the post-resolution identity - // guard are computed from the resolution tree. + const { predicate, selectorExpression, expectedText } = checked; + // ADR 0012 decision 3 / #1349: a guarded replay dispatch resolves through the snapshot path so + // the post-resolution identity guard runs against the resolution tree. const replayTargetGuard = req.internal?.replayTargetGuard; - const recordingSession = isSessionRecording(params.sessionStore.get(params.sessionName)); - if (!replayTargetGuard && !recordingSession) { - const directResponse = await dispatchDirectIosSelectorIs( - params, - predicate as IsPredicate, - split.selectorExpression, - expectedText, - ); - if (directResponse) return directResponse; - } - const resolvedRuntime = await createSelectorRuntime(params, { + // ADR 0019: `is` declares `device-runtime`, so its request path reaches the device ONLY through + // the operations R37 declares. Every predicate — including the simple iOS `id=` selector a + // direct runner query used to answer without a capture — resolves through the bound capture. + // Admission before a bypass is not the same as executing through the seam, so the bypass is + // gone rather than merely ordered after admission. + const resolvedRuntime = await createBoundSelectorRuntime(params, { requireSession: true, - capability: 'is', + command: 'is', }); if (!resolvedRuntime.ok) return resolvedRuntime.response; @@ -253,8 +242,8 @@ export async function dispatchIsViaRuntime( const result = await resolvedRuntime.runtime.selectors.is({ session: params.sessionName, requestId: req.meta?.requestId, - predicate: predicate as IsPredicate, - selector: split.selectorExpression, + predicate, + selector: selectorExpression, expectedText, expectedResolvedTarget: replayTargetGuard, }); @@ -314,7 +303,7 @@ export async function dispatchWaitViaRuntime( mintedGeneration: versionedRef.generation, }); } - // Wait builds its runtime directly (no createSelectorRuntime), so the consumed-snapshot slot + // Wait builds its runtime directly (no createBoundSelectorRuntime), so the consumed-snapshot slot // must be initialized here too or sessionless waits have nowhere to report the capture from. params.consumedSnapshot ??= {}; const execute = async () => { @@ -368,76 +357,6 @@ function readRecordedResolutionTarget( return { node: node as SnapshotNode, preActionNodes: preActionNodes as SnapshotNode[] }; } -function readDirectIosGetSelector( - session: SessionState | undefined, - property: 'text' | 'attrs', - selectorExpression: string, -): DirectIosSelectorTarget | null { - // ADR 0012 decision 3: recording requires the snapshot path so target - // evidence can be computed from the resolution tree. - if (!session || isSessionRecording(session)) return null; - const selector = readSimpleIosSelectorTarget({ session, selectorExpression }); - // get text intentionally disambiguates label/text/value triplets from snapshots; the runner - // direct query rejects those ambiguous matches before the shared selector resolver can rank them. - if (property === 'text' && selector?.key !== 'id') return null; - return selector; -} - -async function dispatchDirectIosSelectorGet( - params: SelectorRuntimeParams, - property: 'text' | 'attrs', - selectorExpression: string, -): Promise { - const session = params.sessionStore.get(params.sessionName); - const selector = readDirectIosGetSelector(session, property, selectorExpression); - if (!session || !selector) return null; - - const result = await queryDirectIosSelectorOrFallback(params, session, selector); - if (isDirectIosSelectorErrorResult(result)) return result.response; - if (!result) return null; - const payload = buildDirectIosGetResult(property, selector.raw, result); - if (!payload) return null; - recordIfSession( - params.sessionStore, - params.sessionName, - params.req, - buildGetRecordResult(payload, property), - ); - return { ok: true, data: toDaemonGetData(payload) }; -} - -async function dispatchDirectIosSelectorIs( - params: SelectorRuntimeParams, - predicate: IsPredicate, - selectorExpression: string, - expectedText: string, -): Promise { - if (predicate === 'hidden') return null; - const directQuery = await resolveDirectIosSelectorQuery(params, selectorExpression); - if (isDirectIosSelectorErrorResult(directQuery)) return directQuery.response; - if (!directQuery?.result.found || !directQuery.result.node) return null; - - const payload = - predicate === 'exists' - ? { - predicate, - pass: true, - selector: directQuery.selector.raw, - matches: 1, - selectorChain: [directQuery.selector.raw], - } - : buildDirectIosIsResult( - predicate, - expectedText, - directQuery.selector.raw, - directQuery.session, - directQuery.result.node, - ); - if (!payload) return null; - recordIfSession(params.sessionStore, params.sessionName, params.req, payload); - return { ok: true, data: stripSelectorChain(payload) }; -} - async function dispatchDirectIosSelectorWait( params: SelectorRuntimeParams & { session: SessionState | undefined; @@ -469,19 +388,6 @@ async function dispatchDirectIosSelectorWait( ); } -async function resolveDirectIosSelectorQuery( - params: SelectorRuntimeParams, - selectorExpression: string, -): Promise { - const session = params.sessionStore.get(params.sessionName); - const selector = readSimpleIosSelectorTarget({ session, selectorExpression }); - if (!session || !selector) return null; - const result = await queryDirectIosSelectorOrFallback(params, session, selector); - if (isDirectIosSelectorErrorResult(result)) return result; - if (!result) return null; - return { session, selector, result }; -} - /** * The single querySelector client for the local XCTest runner: a live, * tree-independent read (and its found/text/node shape) for exactly one @@ -536,50 +442,11 @@ async function queryDirectIosSelectorOrFallback( } function isDirectIosSelectorErrorResult( - result: DirectIosSelectorFallbackResult | ResolvedDirectIosSelectorQuery, + result: DirectIosSelectorFallbackResult, ): result is DirectIosSelectorErrorResult { return result !== null && 'kind' in result && result.kind === 'error'; } -function buildDirectIosGetResult( - property: 'text' | 'attrs', - selector: string, - result: DirectIosSelectorQueryResult, -) { - if (!result.found || !result.node) return null; - const base = { - target: { kind: 'selector' as const, selector }, - node: result.node, - selectorChain: [selector], - }; - if (property === 'attrs') return { kind: 'attrs' as const, ...base }; - if (typeof result.text !== 'string') return null; - return { kind: 'text' as const, ...base, text: result.text }; -} - -function buildDirectIosIsResult( - predicate: Exclude, - expectedText: string, - selector: string, - session: SessionState, - node: SnapshotNode, -): Record | null { - const result = evaluateIsPredicate({ - predicate, - node, - nodes: [node], - expectedText, - platform: session.device.platform, - }); - return { - predicate, - pass: result.pass, - selector, - ...(predicate === 'text' ? { text: result.actualText } : {}), - selectorChain: [selector], - }; -} - function readDirectIosSelectorNode(data: Record): SnapshotNode | undefined { const nodes = data.nodes; if (!Array.isArray(nodes)) return undefined; diff --git a/src/daemon/snapshot-runtime-binding.ts b/src/daemon/snapshot-runtime-binding.ts index 5c135e9d2..c8e1e8410 100644 --- a/src/daemon/snapshot-runtime-binding.ts +++ b/src/daemon/snapshot-runtime-binding.ts @@ -1,7 +1,10 @@ import { resolveSnapshotRuntimePlan, type CaptureSnapshotInput, + type ElementTextRuntimeOperations, + type ReadTextAtPointInput, type RuntimeOperationFact, + type SelectorCaptureRuntimePlan, type SnapshotResult, type SnapshotRuntimeOperations, type SnapshotRuntimePlan, @@ -43,10 +46,22 @@ type ResolvedSnapshotCaptureRuntime = /** A capture operation parametrized by intent, so a polling caller can capture repeatedly * under the one binding it was admitted for. */ -type BoundSnapshotCapture = (input: CaptureSnapshotInput) => Promise; +export type BoundSnapshotCapture = (input: CaptureSnapshotInput) => Promise; -type AdmittedSnapshotCapture = - | Readonly<{ ok: true; capture: BoundSnapshotCapture }> +/** The owner's live element read, when its facts advertise one. */ +export type BoundElementRead = ElementTextRuntimeOperations['readTextAtPoint']; + +export type AdmittedSnapshotCapture = + | Readonly<{ + ok: true; + capture: BoundSnapshotCapture; + /** + * The owner's live element read, present only when the caller's plan declared it PREFERRED + * and the admitted owner advertised it. `snapshot`/`diff` plans declare no read, so this is + * simply absent for them — the member is additive and they are unchanged. + */ + readTextAtPoint?: BoundElementRead; + }> | Readonly<{ ok: false; response: DaemonResponse }>; /** @@ -55,12 +70,12 @@ type AdmittedSnapshotCapture = * the admitted device. `snapshot`/`diff` supply the four-way custom-actions plan and the * selector family the active-app plan; a new consumer supplies a plan and a command name. */ -async function admitAndBindSnapshotCapture( +export async function admitAndBindSnapshotCapture( params: Readonly<{ command: string; device: SessionState['device']; session: SessionState | undefined; - plan: SnapshotRuntimePlan; + plan: SnapshotRuntimePlan | SelectorCaptureRuntimePlan; inspectFacts?: InspectDeviceRuntimeFacts; bindDevice?: BindDeviceRuntime; }>, @@ -79,10 +94,11 @@ async function admitAndBindSnapshotCapture( }), }; } - const runtime = await bindSnapshotCaptureRuntime(params.bindDevice, admission); + const bound = await bindSnapshotCaptureRuntime(params.bindDevice, admission); return Object.freeze({ ok: true, - capture: async (input: CaptureSnapshotInput) => await runtime.captureSnapshot(input), + capture: async (input: CaptureSnapshotInput) => await bound.captureSnapshot(input), + ...(bound.readTextAtPoint ? { readTextAtPoint: bound.readTextAtPoint } : {}), }); } @@ -132,15 +148,28 @@ export async function resolveBoundSnapshotCaptureRuntime( */ async function bindSnapshotCaptureRuntime( bindDevice: BindDeviceRuntime | undefined, - admission: AdmittedRuntimePlan, -): Promise }>> { + admission: AdmittedRuntimePlan, +): Promise< + Readonly<{ + captureSnapshot(input: CaptureSnapshotInput): Promise; + readTextAtPoint?: BoundElementRead; + }> +> { const bind = requireRuntimeBinding(bindDevice); const { device, plan } = unwrapAdmittedRuntimePlan(admission); + // One switch, one set of operation selectors. The selector arms reuse the SAME + // `selectActiveAppSnapshot` / `selectSnapshotWithoutActiveApp` the snapshot arms use and only + // add the preferred element read; the discriminants differ solely so the compiler can narrow + // `plan.use` per family. No parallel plan-to-operation dispatch is introduced. switch (plan.kind) { case 'active-app': { const runtime = await bind(device, plan.use); return selectActiveAppSnapshot(runtime); } + case 'selector-active-app': { + const runtime = await bind(device, plan.use); + return { ...selectActiveAppSnapshot(runtime), ...selectElementRead(runtime) }; + } case 'custom-actions-active-app': { const runtime = await bind(device, plan.use); return selectCustomActionsSnapshot(runtime); @@ -149,6 +178,10 @@ async function bindSnapshotCaptureRuntime( const runtime = await bind(device, plan.use); return selectSnapshotWithoutActiveApp(runtime); } + case 'selector-without-active-app': { + const runtime = await bind(device, plan.use); + return { ...selectSnapshotWithoutActiveApp(runtime), ...selectElementRead(runtime) }; + } case 'custom-actions-without-active-app': { const runtime = await bind(device, plan.use); return selectCustomActionsSnapshot(runtime); @@ -156,6 +189,28 @@ async function bindSnapshotCaptureRuntime( } } +/** + * Projects the preferred element read when the admitted owner advertised it. A projection whose + * use never declared it simply has no such member, so this yields `{}` for `snapshot`/`diff`. + */ +function selectElementRead( + runtime: Readonly<{ operations: Readonly<{ readTextAtPoint?: BoundElementRead }> }>, +): Readonly<{ readTextAtPoint?: BoundElementRead }> { + const readTextAtPoint = runtime.operations.readTextAtPoint; + // Narrowed by CONSTRUCTION rather than by assertion: the projection below is only buildable + // from a non-undefined local, so presence is carried by the value that captured it. + return readTextAtPoint + ? { readTextAtPoint: bindElementRead({ operations: { readTextAtPoint } }) } + : {}; +} + +/** The one lexical owner of the narrowed `readTextAtPoint` call. */ +function bindElementRead( + runtime: Readonly<{ operations: Readonly<{ readTextAtPoint: BoundElementRead }> }>, +): BoundElementRead { + return async (input: ReadTextAtPointInput) => await runtime.operations.readTextAtPoint(input); +} + type BoundSnapshotOperation = Readonly<{ operations: Readonly>; }>; diff --git a/src/daemon/snapshot-runtime-capture-input.ts b/src/daemon/snapshot-runtime-capture-input.ts index d89688b96..3e8dd51f0 100644 --- a/src/daemon/snapshot-runtime-capture-input.ts +++ b/src/daemon/snapshot-runtime-capture-input.ts @@ -1,5 +1,8 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import type { CaptureSnapshotInput } from '@agent-device/contracts/platform'; +import type { + CaptureSnapshotInput, + SnapshotRuntimeExecution, +} from '@agent-device/contracts/platform'; import { contextFromFlags } from './context.ts'; import type { DaemonRequest, SessionState } from './types.ts'; @@ -15,6 +18,11 @@ export function buildRuntimeCaptureInput( meta?: DaemonRequest['meta']; session: SessionState | undefined; snapshotScope: string | undefined; + /** + * Web rect captures request bounds explicitly. Lands here with the selector capture path, + * its first consumer; `snapshot`/`diff` pass nothing and are unaffected. + */ + includeRects?: boolean; }>, ): CaptureSnapshotInput { const { flags, logPath, meta, session, snapshotScope } = params; @@ -37,17 +45,41 @@ export function buildRuntimeCaptureInput( raw: flags?.snapshotRaw, customActions: flags?.snapshotCustomActions, includeHiddenContentHints: flags?.snapshotIncludeHiddenContentHints, + includeRects: params.includeRects, surface, }, - execution: { - requestId: context.requestId, - verbose: context.verbose, - logPath: context.logPath, - traceLogPath: context.traceLogPath, - iosXctestrunFile: context.iosXctestrunFile, - iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, - iosXctestEnvDir: context.iosXctestEnvDir, - runnerLeaseContext: context.runnerLeaseContext, - }, + execution: runtimeExecutionFromContext(context), + }; +} + +/** + * Projects the runner execution metadata a platform operation needs out of a resolved command + * context. Every request-bound operation — capture and element read alike — must forward the + * SAME set: dropping a field silently strips request id, log/trace paths, XCUITest overrides, or + * runner lease context, so the operation still answers but runs unconfigured and its diagnostics + * land nowhere. One projection means a new field reaches every operation at once and cannot be + * forgotten at one call site. + */ +export function runtimeExecutionFromContext( + context: Readonly<{ + requestId?: string; + verbose?: boolean; + logPath?: string; + traceLogPath?: string; + iosXctestrunFile?: string; + iosXctestDerivedDataPath?: string; + iosXctestEnvDir?: string; + runnerLeaseContext?: SnapshotRuntimeExecution['runnerLeaseContext']; + }>, +): SnapshotRuntimeExecution { + return { + requestId: context.requestId, + verbose: context.verbose, + logPath: context.logPath, + traceLogPath: context.traceLogPath, + iosXctestrunFile: context.iosXctestrunFile, + iosXctestDerivedDataPath: context.iosXctestDerivedDataPath, + iosXctestEnvDir: context.iosXctestEnvDir, + runnerLeaseContext: context.runnerLeaseContext, }; } diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index a5c37b86f..ba2e386a0 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -48,6 +48,7 @@ describe('composed platform runtime gateway', () => { appLog: unavailable, network: unavailable, screenshot: unavailable, + elementText: unavailable, viewport: unavailable, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: unavailable, @@ -125,6 +126,7 @@ describe('composed platform runtime gateway', () => { network: unavailable, screenshot: unavailable, viewport: unavailable, + elementText: unavailable, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: unavailable, prepareApplicationOpen: unavailable, diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index 990d93f29..74975f085 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -305,6 +305,7 @@ function unavailableProviderBinding( network: unavailable, screenshot: unavailable, viewport: unavailable, + elementText: unavailable, lifecycle: unavailableProviderLifecycleFacts(unavailable), }); } @@ -323,6 +324,7 @@ function unavailableProviderFacts(runtime: ProviderDeviceRuntime, device: Device network: unavailable, screenshot: unavailable, viewport: unavailable, + elementText: unavailable, readiness: unavailable, lifecycle: unavailableProviderLifecycleFacts(unavailable), }, diff --git a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts index 0c9d07a75..dc222fdd7 100644 --- a/src/platforms/apple/__tests__/interactor-runner-provider.test.ts +++ b/src/platforms/apple/__tests__/interactor-runner-provider.test.ts @@ -41,6 +41,12 @@ const RUNNER_TRANSPORT_METHODS: Record< }, gestureViewport: { invoke: (i) => i.gestureViewport!(), runnerCommand: 'gestureViewport' }, snapshot: { invoke: (i) => i.snapshot(), runnerCommand: 'snapshot' }, + // Runner-routed for every provider-backed device: the macOS-helper branch is reachable only + // for a local desktop/menubar surface, which a provider-owned mobile device never carries. + readTextAtPoint: { + invoke: (i) => i.readTextAtPoint!({ x: 10, y: 20 }), + runnerCommand: 'readText', + }, back: { invoke: (i) => i.back(), runnerCommand: 'backInApp' }, home: { invoke: (i) => i.home(), runnerCommand: 'home' }, setOrientation: { invoke: (i) => i.setOrientation('portrait'), runnerCommand: 'rotate' }, diff --git a/src/platforms/apple/interactor.ts b/src/platforms/apple/interactor.ts index 060e67a02..8fc8a9bcb 100644 --- a/src/platforms/apple/interactor.ts +++ b/src/platforms/apple/interactor.ts @@ -18,13 +18,14 @@ import { type AppleRunnerProvider, } from './core/runner/runner-provider.ts'; import { toAppleTvRemoteButton } from '@agent-device/contracts/interaction'; +import type { SessionSurface } from '@agent-device/contracts/session'; import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { isMacOs, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { withMethodScope } from '../../utils/method-scope.ts'; -import type { RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; +import type { Point, RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; import type { Interactor, RunnerCallOptions, @@ -65,6 +66,12 @@ export function createAppleInteractor( close: (app) => closeIosApp(device, app, runnerOpts), screenshot: (outPath, options) => runAppleScreenshot(device, outPath, options, runnerOpts), snapshot: async (options) => await captureAppleSnapshot(device, options, runnerOpts), + // The live text at a point: helper for macOS desktop/menubar surfaces, XCTest runner for + // every other Apple leaf including a macOS app session. + readTextAtPoint: async (point, options) => + usesMacOsHelperSurface(device, options?.surface) + ? await readMacOsSurfaceTextAtPoint(point, options) + : await readRunnerTextAtPoint(device, point, options, runnerOpts), back: async (mode) => { if (isTvOsDevice(device)) { // tvOS focus-only navigation: the Menu button pops focus, not a coordinate tap. @@ -330,3 +337,36 @@ function readAppleSnapshotResult(result: Record): { : undefined, }; } + +/** Only non-app macOS surfaces are helper-read; an app session is runner-read like any leaf. */ +function usesMacOsHelperSurface(device: DeviceInfo, surface: SessionSurface | undefined): boolean { + return isMacOs(device) && surface !== undefined && surface !== 'app'; +} + +async function readMacOsSurfaceTextAtPoint( + point: Point, + options?: { appBundleId?: string; surface?: SessionSurface }, +): Promise { + const { runMacOsReadTextAction } = await import('./os/macos/helper.ts'); + const result = await runMacOsReadTextAction(point.x, point.y, { + bundleId: options?.appBundleId, + surface: options?.surface, + }); + return result.text; +} + +async function readRunnerTextAtPoint( + device: DeviceInfo, + point: Point, + options: { appBundleId?: string; signal?: AbortSignal } | undefined, + runnerOpts: RunnerCallOptions, +): Promise { + const result = await runAppleRunnerCommand( + device, + { command: 'readText', x: point.x, y: point.y, appBundleId: options?.appBundleId }, + options?.signal ? { ...runnerOpts, signal: options.signal } : runnerOpts, + ); + if (typeof result.text === 'string') return result.text; + // The runner answers `message` when it reached the element but rendered no readable text. + return typeof result.message === 'string' ? result.message : undefined; +} diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index 5705e9411..5b3cc2328 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -142,31 +142,12 @@ export async function createIosSettingsWorld(): Promise { }, result: { transformed: true }, }, - { - command: 'ios.runner.querySelector', - deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, - platform: 'apple', - request: { - command: 'querySelector', - selectorKey: 'label', - selectorValue: 'General', - appBundleId: 'com.apple.Preferences', - }, - result: { - found: true, - nodes: [ - { - index: 0, - type: 'XCUIElementTypeCell', - label: 'General', - identifier: 'General', - rect: { x: 16, y: 100, width: 360, height: 44 }, - enabled: true, - hittable: true, - }, - ], - }, - }, + // `is visible label=General` answered from a direct `querySelector` here until R37 retired + // that shortcut; it now resolves through the bound capture like every other predicate, so it + // consumes a snapshot and issues no runner query at all. The second snapshot is + // `find attrs by label`. This transcript is the scripted proof that the bypass is gone: an + // unexpected `querySelector` would fail the scenario rather than pass unnoticed. + runnerSnapshot(), runnerSnapshot(), { command: 'ios.runner.findText',