diff --git a/packages/contracts/src/apple-multitouch-support.ts b/packages/contracts/src/apple-multitouch-support.ts index d35f6b160..81b6c5fc9 100644 --- a/packages/contracts/src/apple-multitouch-support.ts +++ b/packages/contracts/src/apple-multitouch-support.ts @@ -4,19 +4,11 @@ import { type AppleOS, type DeviceInfo, } from '@agent-device/kernel/device'; +import { APPLE_OS_DISPLAY_NAMES } from './apple-os-display-names.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { GesturePlan } from './gesture-plan-types.ts'; -const APPLE_OS_DISPLAY_NAMES: Record = { - ios: 'iOS', - ipados: 'iPadOS', - tvos: 'tvOS', - watchos: 'watchOS', - visionos: 'visionOS', - macos: 'macOS', -}; - -const APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS: Partial> = { +export const APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS: Partial> = { visionos: 'visionOS uses spatial input and does not support two-finger touch synthesis.', tvos: 'tvOS has no touch input — this gesture is supported on Android and the iOS simulator only.', macos: diff --git a/packages/contracts/src/apple-os-display-names.ts b/packages/contracts/src/apple-os-display-names.ts new file mode 100644 index 000000000..eb680b018 --- /dev/null +++ b/packages/contracts/src/apple-os-display-names.ts @@ -0,0 +1,18 @@ +import type { AppleOS } from '@agent-device/kernel/device'; + +/** + * How each Apple OS names itself in agent-facing prose. + * + * Its own module because two callers need it — the defensive adapter check in + * `apple-multitouch-support.ts` and the gesture refusal subject in `gesture-admission.ts` — and + * neither the display table nor the wording it produces is a public contracts surface. Keeping it + * out of a façade-re-exported module is what lets both callers share ONE copy of the wording. + */ +export const APPLE_OS_DISPLAY_NAMES: Record = { + ios: 'iOS', + ipados: 'iPadOS', + tvos: 'tvOS', + watchos: 'watchOS', + visionos: 'visionOS', + macos: 'macOS', +}; diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index c3893da25..118c5a5f6 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -72,6 +72,7 @@ export { providerRuntimeOwner, runtimeOwnerKey, sameRuntimeOwner, + whenAdmitted, } from '../platform-runtime.ts'; export type { BoundDeviceRuntime, @@ -223,11 +224,17 @@ export { waitSelectorCaptureRuntimePlanUses, findRuntimePlanUses, focusRuntimeUse, + gestureRuntimePlanUses, + resolveGestureRuntimePlan, + resolveScrollRuntimePlan, + scrollRuntimePlanUses, typeTextRuntimeUse, viewportRuntimeUse, } from '../platform-runtime-operations.ts'; export type { + GestureRuntimePlan, ScreenshotRuntimePlan, + ScrollRuntimePlan, SelectorCaptureRuntimeIntent, SelectorCaptureRuntimePlan, SnapshotRuntimePlan, @@ -315,6 +322,38 @@ export type { TypeTextRuntimeOperationFacts, TypeTextRuntimeOperations, } from '../type-text-runtime.ts'; +export { + bindLocalGestureInteractor, + bindProviderGestureInteractor, + gestureRuntimeOperationFacts, +} from '../gesture-runtime.ts'; +export type { + GesturePlanInput, + GestureRuntimeOperationFacts, + GestureRuntimeOperations, + GestureViewportInput, + LocalGestureInteractorResolver, + ProviderGestureInteractorResolver, +} from '../gesture-runtime.ts'; +export { + ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, + gestureRefusalMessage, + PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} from '../gesture-admission.ts'; +export { APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS } from '../apple-multitouch-support.ts'; +export { + bindLocalScrollInteractor, + bindProviderScrollInteractor, + scrollRuntimeOperationFacts, +} from '../scroll-runtime.ts'; +export type { + LocalScrollInteractorResolver, + ProviderScrollInteractorResolver, + ScrollDirectionInput, + ScrollRuntimeOperationFacts, + ScrollRuntimeOperations, +} from '../scroll-runtime.ts'; export { viewportRuntimeOperationFacts } from '../viewport-runtime.ts'; export type { SetViewportInput, diff --git a/packages/contracts/src/gesture-admission.ts b/packages/contracts/src/gesture-admission.ts new file mode 100644 index 000000000..884368745 --- /dev/null +++ b/packages/contracts/src/gesture-admission.ts @@ -0,0 +1,65 @@ +import { + isApplePlatform, + resolveDeviceAppleOs, + type DeviceInfo, +} from '@agent-device/kernel/device'; +import { APPLE_OS_DISPLAY_NAMES } from './apple-os-display-names.ts'; +import type { GestureCommandInput } from './gesture-plan-types.ts'; +import type { GestureRuntimeTier } from './gesture-tier.ts'; + +/** The hint an owner states when it cannot preserve a target-authored drag's timing. */ +export const TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT = + 'Target-authored drag requires an adapter that preserves source hold, timed movement, and destination hold; it is supported on Android touch devices and iOS/iPadOS.'; + +/** The hint the Android owner states for a TV target, which has no touch input at all. */ +export const ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT = + 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.'; + +/** The hint the Apple owner states for a physical iOS/iPadOS device. */ +export const PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT = + 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.'; + +/** + * How a refused cell names itself, reproducing every subject the retired admission produced. + * + * Four owner-specific subjects, then the plain platform name. The special cases resolve the + * Apple OS the way `assertAppleMultiTouchSupported` does; the default reads `appleOs` raw, the way + * the retired `gesturePlatformMessage` did — an Apple device with no declared OS therefore still + * reports `apple`, exactly as before. + */ +function gestureRefusalSubject(device: DeviceInfo, tier: GestureRuntimeTier): string { + const owned = + tier === 'multi-touch' + ? multiTouchRefusalSubject(device) + : tier === 'directional-fling' && device.platform === 'linux' + ? 'Linux' + : undefined; + return owned ?? device.appleOs ?? device.platform; +} + +/** + * The three owner-specific subjects two-contact synthesis produced, or `undefined` where the + * retired admission fell through to the plain platform name. + */ +function multiTouchRefusalSubject(device: DeviceInfo): string | undefined { + if (device.platform === 'android') return device.target === 'tv' ? 'Android TV' : undefined; + if (!isApplePlatform(device.platform)) return undefined; + const appleOs = resolveDeviceAppleOs(device); + if (appleOs === 'ios' || appleOs === 'ipados') return 'physical iOS devices'; + if (appleOs === 'macos' || appleOs === 'tvos' || appleOs === 'visionos') { + return APPLE_OS_DISPLAY_NAMES[appleOs]; + } + return undefined; +} + +/** + * The refusal one unavailable gesture cell reports. `gesture fling` on Linux keeps its bare intent + * wording because its subject is the platform's display name, so no special-casing is needed here. + */ +export function gestureRefusalMessage( + device: DeviceInfo, + tier: GestureRuntimeTier, + intent: GestureCommandInput['intent'], +): string { + return `gesture ${intent} is not supported on ${gestureRefusalSubject(device, tier)}`; +} diff --git a/packages/contracts/src/gesture-runtime.test.ts b/packages/contracts/src/gesture-runtime.test.ts new file mode 100644 index 000000000..edec2d87e --- /dev/null +++ b/packages/contracts/src/gesture-runtime.test.ts @@ -0,0 +1,215 @@ +import { expect, test, vi } from 'vitest'; +import { + bindLocalGestureInteractor, + bindProviderGestureInteractor, + gestureRuntimeOperationFacts, +} from './gesture-runtime.ts'; +import type { GesturePlan } from './gesture-plan-types.ts'; +import type { Interactor } from './interactor-types.ts'; + +const device = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +} as const; + +const available = { available: true } as const; +const unavailable = { available: false, reason: 'unsupported-platform-leaf' } as const; + +const allAvailable = gestureRuntimeOperationFacts({ + plan: available, + directionalFling: available, + multiTouch: available, + targetAuthoredDrag: available, + viewport: available, +}); + +const plan: GesturePlan = { + topology: 'single', + intent: 'pan', + executionProfile: 'timed-pan', + durationMs: 300, + viewport: { x: 0, y: 0, width: 400, height: 800 }, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point: { x: 10, y: 20 } }, + { offsetMs: 300, point: { x: 10, y: 220 } }, + ], + }, + ], +}; + +test('builds the exact gesture operation fact catalog', () => { + expect( + gestureRuntimeOperationFacts({ + plan: available, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: available, + viewport: unavailable, + }), + ).toEqual({ + performGesturePlan: available, + performDirectionalFlingPlan: unavailable, + performMultiTouchGesturePlan: unavailable, + performTargetAuthoredDrag: available, + gestureViewport: unavailable, + }); +}); + +test('a local binding executes the plan through the owner interactor', async () => { + const performGesture = vi.fn(async () => ({ backend: 'adb' })); + const resolveInteractor = vi.fn(async () => ({ performGesture }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindLocalGestureInteractor({ + device, + signal, + facts: allAvailable, + resolveInteractor, + }); + await operations.performGesturePlan?.({ + plan, + options: { appBundleId: 'com.example.app' }, + execution: { logPath: '/tmp/daemon.log', requestId: 'gesture-1' }, + }); + + expect(resolveInteractor).toHaveBeenCalledWith(device, { + logPath: '/tmp/daemon.log', + requestId: 'gesture-1', + appBundleId: 'com.example.app', + signal, + }); + // The plan reaches the seam whole and unmodified — this is the sole argument, so an executor + // that dropped or rebuilt it shows up here. + expect(performGesture).toHaveBeenCalledWith(plan); +}); + +test('every admitted tier reaches the same single plan executor', async () => { + const performGesture = vi.fn(async () => ({})); + const operations = bindLocalGestureInteractor({ + device, + signal: new AbortController().signal, + facts: allAvailable, + resolveInteractor: async () => ({ performGesture }) as unknown as Interactor, + }); + + await operations.performDirectionalFlingPlan?.({ plan }); + await operations.performMultiTouchGesturePlan?.({ plan }); + await operations.performTargetAuthoredDrag?.({ plan }); + + expect(performGesture).toHaveBeenCalledTimes(3); +}); + +test('a binding exposes only the tiers its owner facts admitted', () => { + const operations = bindLocalGestureInteractor({ + device, + signal: new AbortController().signal, + facts: gestureRuntimeOperationFacts({ + plan: available, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: unavailable, + viewport: unavailable, + }), + resolveInteractor: async () => ({ performGesture: async () => ({}) }) as unknown as Interactor, + }); + + expect(operations.performGesturePlan).toBeTypeOf('function'); + expect(operations.performDirectionalFlingPlan).toBeUndefined(); + expect(operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(operations.performTargetAuthoredDrag).toBeUndefined(); + expect(operations.gestureViewport).toBeUndefined(); +}); + +test('a local binding reads the owner frame through its interactor', async () => { + const gestureViewport = vi.fn(async () => ({ x: 0, y: 0, width: 393, height: 852 })); + const resolveInteractor = vi.fn(async () => ({ gestureViewport }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindLocalGestureInteractor({ + device, + signal, + facts: allAvailable, + resolveInteractor, + }); + + await expect( + operations.gestureViewport?.({ execution: { requestId: 'gesture-2' } }), + ).resolves.toEqual({ x: 0, y: 0, width: 393, height: 852 }); + expect(resolveInteractor).toHaveBeenCalledWith(device, { + requestId: 'gesture-2', + appBundleId: undefined, + signal, + }); +}); + +test('a provider binding executes through its own resolved interactor', async () => { + const performGesture = vi.fn(async () => ({})); + const resolveInteractor = vi.fn(() => ({ performGesture }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindProviderGestureInteractor({ + device, + signal, + facts: allAvailable, + resolveInteractor, + }); + await operations.performGesturePlan?.({ plan, execution: { requestId: 'gesture-3' } }); + + expect(resolveInteractor).toHaveBeenCalledWith({ + requestId: 'gesture-3', + appBundleId: undefined, + signal, + }); + expect(performGesture).toHaveBeenCalledWith(plan); +}); + +test('a provider binding fails closed when its exact owner exposes no interactor', async () => { + const operations = bindProviderGestureInteractor({ + device, + signal: new AbortController().signal, + facts: allAvailable, + resolveInteractor: () => undefined, + }); + + await expect(operations.performGesturePlan?.({ plan })).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'provider-runtime-interactor-missing', deviceId: device.id }, + }); +}); + +test('an advertised tier whose interactor cannot execute is a contract bug, not a refusal', async () => { + const operations = bindLocalGestureInteractor({ + device, + signal: new AbortController().signal, + facts: allAvailable, + resolveInteractor: async () => ({}) as unknown as Interactor, + }); + + await expect(operations.performGesturePlan?.({ plan })).rejects.toMatchObject({ + message: expect.stringContaining('advertised gesture execution'), + }); +}); + +test('an already-cancelled request never resolves an interactor', async () => { + const controller = new AbortController(); + controller.abort(); + const performGesture = vi.fn(async () => ({})); + const resolveInteractor = vi.fn(async () => ({ performGesture }) as unknown as Interactor); + + const operations = bindLocalGestureInteractor({ + device, + signal: controller.signal, + facts: allAvailable, + resolveInteractor, + }); + + await expect(operations.performGesturePlan?.({ plan })).rejects.toThrow(); + expect(resolveInteractor).not.toHaveBeenCalled(); + expect(performGesture).not.toHaveBeenCalled(); +}); diff --git a/packages/contracts/src/gesture-runtime.ts b/packages/contracts/src/gesture-runtime.ts new file mode 100644 index 000000000..ddba866f1 --- /dev/null +++ b/packages/contracts/src/gesture-runtime.ts @@ -0,0 +1,187 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import type { GesturePlan } from './gesture-plan-types.ts'; +import { + localInteractorSource, + providerInteractorSource, + type LocalInteractorOperationResolver, + type ProviderInteractorOperationResolver, +} from './interactor-operation-binding.ts'; +import type { Interactor, RunnerContext } from './interactor-types.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; +import { invalidRuntimeContract } from './runtime-contract-error.ts'; +import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; + +/** + * Neutral intent for executing one typed gesture plan (ADR 0013). The plan is already built — + * from the coordinates, preset, or resolved drag targets a caller normalized — so the operation + * names no command, request, session, or CLI flag. + */ +export type GesturePlanInput = Readonly<{ + plan: GesturePlan; + options?: Readonly<{ appBundleId?: string }>; + /** Same runner metadata a capture needs; reuses that type rather than restating it. */ + execution?: SnapshotRuntimeExecution; +}>; + +/** Reading the gesture coordinate frame needs no plan — only the owner's authority. */ +export type GestureViewportInput = Readonly<{ + options?: Readonly<{ appBundleId?: string }>; + execution?: SnapshotRuntimeExecution; +}>; + +/** + * The gesture family's execution surface. + * + * The four plan operations run the SAME mechanics — one `Interactor.performGesture` call — and + * are separate keys because their **cells** differ, not their implementations. That is the shape + * `SnapshotRuntimeOperations` already uses for its three capture keys: an owner declares each + * requirement it can actually meet, and a command requires exactly the tier its input selected, + * so a device is refused where the retired `requireGestureSupported` refused it instead of + * failing mid-execution. + * + * The tiers, and the retired admission each one restates: + * - `performGesturePlan` — one-contact fling/pan, and every `swipe`. Refused where the legacy + * check refused a plain gesture (web, watchOS, visionOS). + * - `performDirectionalFlingPlan` — `gesture fling --direction`, whose speed semantics Linux + * cannot honor even though it executes coordinate flings through its drag primitive. + * - `performMultiTouchGesturePlan` — pinch/rotate/transform and two-pointer pan. On Apple this is + * the two-finger XCTest synthesis, which is iOS/iPadOS **simulator** only. + * - `performTargetAuthoredDrag` — `gesture drag`, which needs an adapter preserving source hold, + * timed movement, and destination hold. + */ +export type GestureRuntimeOperations = Readonly<{ + performGesturePlan(input: GesturePlanInput): Promise | void>; + performDirectionalFlingPlan(input: GesturePlanInput): Promise | void>; + performMultiTouchGesturePlan(input: GesturePlanInput): Promise | void>; + performTargetAuthoredDrag(input: GesturePlanInput): Promise | void>; + /** + * The owner's own gesture coordinate frame. Declared `preferred`, never `required`: a caller + * without it derives the frame from a capture instead (`resolveGestureViewport`), which is how + * Linux executes gestures today, so requiring it would refuse a cell that works. + */ + gestureViewport(input: GestureViewportInput): Promise; +}>; + +export type GestureRuntimeOperationFacts = Readonly<{ + performGesturePlan: RuntimeOperationFact; + performDirectionalFlingPlan: RuntimeOperationFact; + performMultiTouchGesturePlan: RuntimeOperationFact; + performTargetAuthoredDrag: RuntimeOperationFact; + gestureViewport: RuntimeOperationFact; +}>; + +/** Builds the exhaustive owner claims for the five gesture requirements. */ +export function gestureRuntimeOperationFacts( + input: Readonly<{ + plan: RuntimeOperationFact; + directionalFling: RuntimeOperationFact; + multiTouch: RuntimeOperationFact; + targetAuthoredDrag: RuntimeOperationFact; + viewport: RuntimeOperationFact; + }>, +): GestureRuntimeOperationFacts { + return Object.freeze({ + performGesturePlan: input.plan, + performDirectionalFlingPlan: input.directionalFling, + performMultiTouchGesturePlan: input.multiTouch, + performTargetAuthoredDrag: input.targetAuthoredDrag, + gestureViewport: input.viewport, + }); +} + +/** + * Captures one selected owner's interactor authority for the lifetime of a request binding, and + * exposes only the tiers that owner's own facts admitted. + * + * The per-tier gating lives HERE rather than in each of the eight runtime owners: the tiers share + * one executor, so eight copies of the same five-branch spread would be duplication of mechanism + * — and the next tier added would cost eight more edits. + */ +function bindGestureOperations( + facts: GestureRuntimeOperationFacts, + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, +): Partial { + const performPlan = async (input: GesturePlanInput) => { + const interactor = await resolveGestureInteractor(signal, resolveInteractor, input); + // Facts advertised gesture execution but the owner's interactor cannot perform it. That is a + // contract violation, not a refusal (ADR 0019 §2): degrading here would execute nothing and + // report success. + if (typeof interactor.performGesture !== 'function') { + throw invalidRuntimeContract( + 'Runtime owner advertised gesture execution without an interactor implementation', + ); + } + return await interactor.performGesture(input.plan); + }; + return Object.freeze({ + ...(facts.performGesturePlan.available ? { performGesturePlan: performPlan } : {}), + ...(facts.performDirectionalFlingPlan.available + ? { performDirectionalFlingPlan: performPlan } + : {}), + ...(facts.performMultiTouchGesturePlan.available + ? { performMultiTouchGesturePlan: performPlan } + : {}), + ...(facts.performTargetAuthoredDrag.available + ? { performTargetAuthoredDrag: performPlan } + : {}), + ...(facts.gestureViewport.available + ? { + gestureViewport: async (input: GestureViewportInput) => { + const interactor = await resolveGestureInteractor(signal, resolveInteractor, input); + if (typeof interactor.gestureViewport !== 'function') { + throw invalidRuntimeContract( + 'Runtime owner advertised gestureViewport without an interactor implementation', + ); + } + return await interactor.gestureViewport(); + }, + } + : {}), + }); +} + +async function resolveGestureInteractor( + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, + input: GesturePlanInput | GestureViewportInput, +): Promise { + signal.throwIfAborted(); + return await resolveInteractor({ + ...input.execution, + appBundleId: input.options?.appBundleId, + signal, + }); +} + +export type LocalGestureInteractorResolver = LocalInteractorOperationResolver; + +export function bindLocalGestureInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + facts: GestureRuntimeOperationFacts; + resolveInteractor: LocalGestureInteractorResolver; + }>, +): Partial { + return bindGestureOperations(params.facts, params.signal, localInteractorSource(params)); +} + +export type ProviderGestureInteractorResolver = ProviderInteractorOperationResolver; + +/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */ +export function bindProviderGestureInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + facts: GestureRuntimeOperationFacts; + resolveInteractor: ProviderGestureInteractorResolver; + }>, +): Partial { + return bindGestureOperations( + params.facts, + params.signal, + providerInteractorSource({ ...params, operation: 'gesture' }), + ); +} diff --git a/packages/contracts/src/gesture-tier.ts b/packages/contracts/src/gesture-tier.ts new file mode 100644 index 000000000..e276215c6 --- /dev/null +++ b/packages/contracts/src/gesture-tier.ts @@ -0,0 +1,31 @@ +import type { GestureCommandInput, GestureSemanticInput } from './gesture-plan-types.ts'; + +/** + * Which execution tier one gesture input needs. The tier comes from the input alone — never from + * the device — because a tier's *availability* is what varies by owner, and mixing the two is how + * the retired `requireGestureSupported` ended up owning a platform table inside the daemon. + * + * Its own module because the classifier is contracts-internal: the runtime-use catalog turns a + * tier into a declared use, and callers outside this package select a use, never a tier. + */ +export type GestureRuntimeTier = + | 'plan' + | 'directional-fling' + | 'multi-touch' + | 'target-authored-drag'; + +/** Two contacts are what pinch, rotate, transform, and an explicit two-pointer pan all need. */ +function isMultiTouchGesture(input: GestureSemanticInput): boolean { + if (input.intent === 'pan') return ('pointerCount' in input ? input.pointerCount : 1) === 2; + return input.intent === 'pinch' || input.intent === 'rotate' || input.intent === 'transform'; +} + +/** Selects the one tier a gesture input needs, so its handler binds exactly once (ADR 0019 §9). */ +export function gestureRuntimeTier(input: GestureCommandInput): GestureRuntimeTier { + if (input.intent === 'drag') return 'target-authored-drag'; + if (isMultiTouchGesture(input)) return 'multi-touch'; + // A direction-authored fling carries speed semantics a coordinate fling does not, which is the + // one thing the Linux drag primitive cannot reproduce. + if (input.intent === 'fling' && 'direction' in input) return 'directional-fling'; + return 'plan'; +} diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 2c93a6181..f86dab4a3 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -17,6 +17,10 @@ import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot- import type { SelectorObservationRuntimeOperations } from './selector-observation-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; import type { FocusRuntimeOperations } from './focus-runtime.ts'; +import type { GestureCommandInput } from './gesture-plan-types.ts'; +import { gestureRuntimeTier, type GestureRuntimeTier } from './gesture-tier.ts'; +import type { GestureRuntimeOperations } from './gesture-runtime.ts'; +import type { ScrollRuntimeOperations } from './scroll-runtime.ts'; import type { TypeTextRuntimeOperations } from './type-text-runtime.ts'; import type { ElementTextRuntimeOperations } from './element-text-runtime.ts'; import type { @@ -53,6 +57,8 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & SelectorObservationRuntimeOperations & ViewportRuntimeOperations & FocusRuntimeOperations & + GestureRuntimeOperations & + ScrollRuntimeOperations & TypeTextRuntimeOperations & ElementTextRuntimeOperations & DeviceReadinessRuntimeOperations & @@ -76,6 +82,108 @@ export const captureSnapshotUse = defineUse({ required: ['captureSnapshot'] }); export const viewportRuntimeUse = defineUse({ required: ['setViewport'] }); export const focusRuntimeUse = defineUse({ required: ['focusPoint'] }); export const typeTextRuntimeUse = defineUse({ required: ['typeText'] }); + +/** + * The gesture family's action-selected uses (ADR 0019 §9: one bind per handler). A gesture input + * needs exactly one execution tier, so each tier carries the complete requirement set and the + * handler binds once — for the whole `swipe --count N` series as much as for one `gesture pinch`. + * + * `gestureViewport` is `preferred` on every tier, and that is parity rather than taste: a caller + * without it derives the coordinate frame from a capture instead, which is how Linux executes + * gestures today. Requiring it would refuse a cell that currently works. + */ +const gesturePlanUse = defineUse({ + required: ['performGesturePlan'], + preferred: ['gestureViewport'], +}); +const gestureDirectionalFlingUse = defineUse({ + required: ['performDirectionalFlingPlan'], + preferred: ['gestureViewport'], +}); +const gestureMultiTouchUse = defineUse({ + required: ['performMultiTouchGesturePlan'], + preferred: ['gestureViewport'], +}); +const gestureTargetAuthoredDragUse = defineUse({ + required: ['performTargetAuthoredDrag'], + preferred: ['gestureViewport'], +}); + +/** `scroll ` executes one pass and needs nothing else. */ +const scrollDirectionUse = defineUse({ required: ['scrollDirection'] }); +/** + * `scroll top` / `scroll bottom` verify hidden content between passes, so the capture is part of + * the tier's requirement rather than something discovered mid-run — the retired leaf's + * "requires snapshot support to verify hidden content before scrolling" refusal, moved to + * admission. + */ +const scrollEdgeUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] }); + +const gestureUsesByTier = Object.freeze({ + plan: gesturePlanUse, + 'directional-fling': gestureDirectionalFlingUse, + 'multi-touch': gestureMultiTouchUse, + 'target-authored-drag': gestureTargetAuthoredDragUse, +} as const); + +/** Every use `gesture` and `swipe` can select between; the descriptor declares the whole set. */ +export const gestureRuntimePlanUses = Object.freeze([ + gesturePlanUse, + gestureDirectionalFlingUse, + gestureMultiTouchUse, + gestureTargetAuthoredDragUse, +] as const); + +/** Every use `scroll` can select between. */ +export const scrollRuntimePlanUses = Object.freeze([scrollDirectionUse, scrollEdgeUse] as const); + +type GesturePlanFor = Readonly<{ + tier: Tier; + operation: GestureTierOperation; + use: (typeof gestureUsesByTier)[Tier]; +}>; + +type GestureTierOperation = + (typeof gestureUsesByTier)[Tier]['required'][0]; + +export type GestureRuntimePlan = { + [Tier in GestureRuntimeTier]: GesturePlanFor; +}[GestureRuntimeTier]; + +/** Selects the one owner-fact-backed gesture plan a normalized gesture input needs. */ +export function resolveGestureRuntimePlan(input: GestureCommandInput): GestureRuntimePlan { + const tier = gestureRuntimeTier(input); + switch (tier) { + case 'plan': + return gesturePlan(tier); + case 'directional-fling': + return gesturePlan(tier); + case 'multi-touch': + return gesturePlan(tier); + case 'target-authored-drag': + return gesturePlan(tier); + } +} + +function gesturePlan(tier: Tier): GesturePlanFor { + const use = gestureUsesByTier[tier]; + return Object.freeze({ + tier, + operation: use.required[0] as GestureTierOperation, + use, + }); +} + +export type ScrollRuntimePlan = + | Readonly<{ kind: 'direction'; use: typeof scrollDirectionUse }> + | Readonly<{ kind: 'edge'; use: typeof scrollEdgeUse }>; + +/** `scroll top`/`scroll bottom` verify between passes; every other scroll executes one pass. */ +export function resolveScrollRuntimePlan(input: Readonly<{ edge: boolean }>): ScrollRuntimePlan { + return input.edge + ? Object.freeze({ kind: 'edge', use: scrollEdgeUse } as const) + : Object.freeze({ kind: 'direction', use: scrollDirectionUse } as const); +} const captureSnapshotWithCustomActionsUse = defineUse({ required: ['captureSnapshot', 'captureSnapshotWithCustomActions'], }); diff --git a/packages/contracts/src/platform-runtime-unavailable.test.ts b/packages/contracts/src/platform-runtime-unavailable.test.ts index 9bf2abdb7..5bfa5a91a 100644 --- a/packages/contracts/src/platform-runtime-unavailable.test.ts +++ b/packages/contracts/src/platform-runtime-unavailable.test.ts @@ -32,6 +32,8 @@ test('generic unavailable binding preserves exact provider ownership and mode', screenshot: { available: false, reason: 'unsupported-device-kind' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, focus: { available: false, reason: 'unsupported-provider-mode' }, + gesture: { available: false, reason: 'unsupported-provider-mode' }, + scroll: { available: false, reason: 'unsupported-provider-mode' }, typeText: { available: false, reason: 'unsupported-provider-mode' }, elementText: { available: false, reason: 'unsupported-provider-mode' }, lifecycle, diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 55c94a807..b098a56f1 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -15,6 +15,8 @@ import { snapshotRuntimeOperationFacts } from './snapshot-runtime.ts'; import { selectorObservationRuntimeOperationFacts } from './selector-observation-runtime.ts'; import { viewportRuntimeOperationFacts } from './viewport-runtime.ts'; import { focusRuntimeOperationFacts } from './focus-runtime.ts'; +import { gestureRuntimeOperationFacts } from './gesture-runtime.ts'; +import { scrollRuntimeOperationFacts } from './scroll-runtime.ts'; import { typeTextRuntimeOperationFacts } from './type-text-runtime.ts'; import { elementTextRuntimeOperationFacts } from './element-text-runtime.ts'; @@ -33,6 +35,8 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{ snapshot?: RuntimeOperationUnavailability; viewport: RuntimeOperationUnavailability; focus: RuntimeOperationUnavailability; + gesture: RuntimeOperationUnavailability; + scroll: RuntimeOperationUnavailability; typeText: RuntimeOperationUnavailability; elementText: RuntimeOperationUnavailability; readiness?: RuntimeOperationUnavailability; @@ -79,6 +83,8 @@ export function createUnavailablePlatformRuntimeFacts( snapshot, viewport, focus, + gesture, + scroll, typeText, elementText, readiness, @@ -121,6 +127,14 @@ export function createUnavailablePlatformRuntimeFacts( }), ...viewportRuntimeOperationFacts({ setViewport: viewport }), ...focusRuntimeOperationFacts({ focus }), + ...gestureRuntimeOperationFacts({ + plan: gesture, + directionalFling: gesture, + multiTouch: gesture, + targetAuthoredDrag: gesture, + viewport: gesture, + }), + ...scrollRuntimeOperationFacts({ scroll }), ...typeTextRuntimeOperationFacts({ type: typeText }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementText }), ensureReady: readiness, @@ -153,6 +167,8 @@ function freezeUnavailableFacts( // Interaction cells are stated by their owner: a family that can drive touch says so for its // exact kinds, and one that cannot must say why rather than inherit a transport gap. focus: Object.freeze({ ...unavailable.focus }), + gesture: Object.freeze({ ...unavailable.gesture }), + scroll: Object.freeze({ ...unavailable.scroll }), typeText: Object.freeze({ ...unavailable.typeText }), readiness: orNetwork(unavailable.readiness), shutdown: orNetwork(unavailable.shutdown), diff --git a/packages/contracts/src/platform-runtime.ts b/packages/contracts/src/platform-runtime.ts index 3891f846a..549a82d10 100644 --- a/packages/contracts/src/platform-runtime.ts +++ b/packages/contracts/src/platform-runtime.ts @@ -110,6 +110,18 @@ export type RuntimeOperationUnavailability = Readonly<{ export type RuntimeOperationFact = Readonly<{ available: true }> | RuntimeOperationUnavailability; +/** + * An operation is present on a binding only when the owner's own facts admitted it. One helper so + * every owner's `bind` reads as a list of admitted operations rather than a chain of branches — + * and so the next operation added there costs no additional branch. + */ +export function whenAdmitted( + fact: RuntimeOperationFact, + build: () => T, +): T | Record { + return fact.available ? build() : {}; +} + export type RuntimeFacts = Readonly<{ device: RuntimeDeviceShape; operations: Readonly<{ diff --git a/packages/contracts/src/scroll-runtime.test.ts b/packages/contracts/src/scroll-runtime.test.ts new file mode 100644 index 000000000..4b2898388 --- /dev/null +++ b/packages/contracts/src/scroll-runtime.test.ts @@ -0,0 +1,103 @@ +import { expect, test, vi } from 'vitest'; +import { + bindLocalScrollInteractor, + bindProviderScrollInteractor, + scrollRuntimeOperationFacts, +} from './scroll-runtime.ts'; +import type { Interactor } from './interactor-types.ts'; + +const device = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +} as const; + +const options = { + amount: 0.55, + pixels: undefined, + durationMs: undefined, + releaseBehavior: 'controlled', +} as const; + +test('builds the exact scroll operation fact catalog', () => { + const scroll = { available: true } as const; + expect(scrollRuntimeOperationFacts({ scroll })).toEqual({ scrollDirection: scroll }); +}); + +test('a local binding scrolls the owner in the requested direction', async () => { + const scroll = vi.fn(async () => ({ pixels: 240 })); + const resolveInteractor = vi.fn(async () => ({ scroll }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindLocalScrollInteractor({ device, signal, resolveInteractor }); + await expect( + operations.scrollDirection({ + direction: 'down', + options, + target: { appBundleId: 'com.example.app' }, + execution: { logPath: '/tmp/daemon.log', requestId: 'scroll-1' }, + }), + ).resolves.toEqual({ pixels: 240 }); + + expect(resolveInteractor).toHaveBeenCalledWith(device, { + logPath: '/tmp/daemon.log', + requestId: 'scroll-1', + appBundleId: 'com.example.app', + signal, + }); + // Positional (direction, options): the `Interactor` seam takes them in that order, and + // transposing them is the one swap an object-shaped assertion would not catch. + expect(scroll).toHaveBeenCalledWith('down', options); +}); + +test('a provider binding scrolls through its own resolved interactor', async () => { + const scroll = vi.fn(async () => undefined); + const resolveInteractor = vi.fn(() => ({ scroll }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindProviderScrollInteractor({ device, signal, resolveInteractor }); + await operations.scrollDirection({ + direction: 'up', + options, + execution: { requestId: 'scroll-2' }, + }); + + expect(resolveInteractor).toHaveBeenCalledWith({ + requestId: 'scroll-2', + appBundleId: undefined, + signal, + }); + expect(scroll).toHaveBeenCalledWith('up', options); +}); + +test('a provider binding fails closed when its exact owner exposes no interactor', async () => { + const operations = bindProviderScrollInteractor({ + device, + signal: new AbortController().signal, + resolveInteractor: () => undefined, + }); + + await expect(operations.scrollDirection({ direction: 'down', options })).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'provider-runtime-interactor-missing', deviceId: device.id }, + }); +}); + +test('an already-cancelled request never resolves an interactor', async () => { + const controller = new AbortController(); + controller.abort(); + const scroll = vi.fn(async () => undefined); + const resolveInteractor = vi.fn(async () => ({ scroll }) as unknown as Interactor); + + const operations = bindLocalScrollInteractor({ + device, + signal: controller.signal, + resolveInteractor, + }); + + await expect(operations.scrollDirection({ direction: 'down', options })).rejects.toThrow(); + expect(resolveInteractor).not.toHaveBeenCalled(); + expect(scroll).not.toHaveBeenCalled(); +}); diff --git a/packages/contracts/src/scroll-runtime.ts b/packages/contracts/src/scroll-runtime.ts new file mode 100644 index 000000000..43a363a09 --- /dev/null +++ b/packages/contracts/src/scroll-runtime.ts @@ -0,0 +1,94 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + localInteractorSource, + providerInteractorSource, + type LocalInteractorOperationResolver, + type ProviderInteractorOperationResolver, +} from './interactor-operation-binding.ts'; +import type { Interactor, RunnerContext } from './interactor-types.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; +import type { ResolvedScrollExecutionOptions } from './scroll-command.ts'; +import type { ScrollDirection } from './scroll-gesture.ts'; +import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; + +/** + * Neutral intent for one directional scroll. Distance, timing, and release behavior are already + * resolved by the caller (`resolveScrollExecutionOptions`), so the owner receives them as data + * and the operation names no command, request, session, or CLI flag. + */ +export type ScrollDirectionInput = Readonly<{ + direction: ScrollDirection; + options: ResolvedScrollExecutionOptions; + target?: Readonly<{ appBundleId?: string }>; + /** Same runner metadata a capture needs; reuses that type rather than restating it. */ + execution?: SnapshotRuntimeExecution; +}>; + +/** + * One scroll pass. `scroll top` / `scroll bottom` run several, verifying between passes with the + * capture operation they additionally require — the platform-visible unit is still a single pass, + * so edge repetition stays caller-side policy rather than a second operation. + */ +export type ScrollRuntimeOperations = Readonly<{ + scrollDirection(input: ScrollDirectionInput): Promise | void>; +}>; + +export type ScrollRuntimeOperationFacts = Readonly<{ + scrollDirection: RuntimeOperationFact; +}>; + +export function scrollRuntimeOperationFacts( + input: Readonly<{ scroll: RuntimeOperationFact }>, +): ScrollRuntimeOperationFacts { + return Object.freeze({ scrollDirection: input.scroll }); +} + +/** + * Captures one selected owner's interactor authority for the lifetime of a request binding. The + * owner is already chosen by the time a binder is called, so each entry point supplies its own + * resolution and this holds only what both share: the runner context and the scroll itself. + */ +function bindScrollDirection( + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, +): ScrollRuntimeOperations { + return Object.freeze({ + scrollDirection: async (input: ScrollDirectionInput) => { + signal.throwIfAborted(); + const interactor = await resolveInteractor({ + ...input.execution, + appBundleId: input.target?.appBundleId, + signal, + }); + return await interactor.scroll(input.direction, input.options); + }, + }); +} + +export type LocalScrollInteractorResolver = LocalInteractorOperationResolver; + +export function bindLocalScrollInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: LocalScrollInteractorResolver; + }>, +): ScrollRuntimeOperations { + return bindScrollDirection(params.signal, localInteractorSource(params)); +} + +export type ProviderScrollInteractorResolver = ProviderInteractorOperationResolver; + +/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */ +export function bindProviderScrollInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: ProviderScrollInteractorResolver; + }>, +): ScrollRuntimeOperations { + return bindScrollDirection( + params.signal, + providerInteractorSource({ ...params, operation: 'scroll' }), + ); +} diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 6fc3f5c78..1b25779a7 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -317,3 +317,83 @@ function expectLifecycleFacts( } } } + +// R42/R43: the Android gesture-tier and scroll cells. The one gate the retired +// `requireGestureSupported` carried on Android was the TV target, which it applied to two-contact +// synthesis and to target-authored drag but never to a plain one-contact fling or pan. +test.each([ + // name, device, plan, multiTouch, drag, scroll + ['emulator', device, true, true, true, true], + ['physical device', { ...device, kind: 'device' as const }, true, true, true, true], + ['unknown kind', unknownKindDevice, true, true, true, true], + ['TV target', { ...device, target: 'tv' as const }, true, false, false, true], + [ + 'synthetic simulator row', + { ...device, kind: 'simulator' as const }, + false, + false, + false, + false, + ], +])( + 'declares the Android %s gesture and scroll cells', + async (_name, runtimeDevice, plan, multiTouch, drag, scroll) => { + const facts = await createAndroidPlatformRuntime(gestureHost()).inspectFacts(runtimeDevice); + expect(facts.operations.performGesturePlan.available).toBe(plan); + // Android honors a direction-authored fling's speed semantics, so it shares the plan cell. + expect(facts.operations.performDirectionalFlingPlan.available).toBe(plan); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(multiTouch); + expect(facts.operations.performTargetAuthoredDrag.available).toBe(drag); + expect(facts.operations.gestureViewport.available).toBe(plan); + expect(facts.operations.scrollDirection.available).toBe(scroll); + }, +); + +test('carries the retired Android TV hints verbatim', async () => { + const facts = await createAndroidPlatformRuntime(gestureHost()).inspectFacts({ + ...device, + target: 'tv', + }); + expect(facts.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.', + }); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); +}); + +test('binds only the Android gesture tiers the target admitted', async () => { + const bind = async (runtimeDevice: DeviceInfo) => + await createAndroidPlatformRuntime(gestureHost()).bind({ + device: runtimeDevice, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const phone = await bind(device); + expect(phone.operations.performMultiTouchGesturePlan).toBeTypeOf('function'); + expect(phone.operations.scrollDirection).toBeTypeOf('function'); + const tv = await bind({ ...device, target: 'tv' }); + expect(tv.operations.performGesturePlan).toBeTypeOf('function'); + expect(tv.operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(tv.operations.performTargetAuthoredDrag).toBeUndefined(); +}); + +function gestureHost(): PlatformRuntimeHost { + return { + processTransports: { resolve: async () => ({ mode: 'local' as const }) }, + appInventory: { + apple: { listApps: async () => [] }, + android: { listApps: async () => [] }, + harmonyos: { listApps: async () => [] }, + }, + localInteractors: { resolve: async () => ({}) }, + screenRecording: { android: { resolve: async () => ({ mode: 'local' as const }) } }, + } as unknown as PlatformRuntimeHost; +} diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 1f326a0fa..8fc325945 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -2,6 +2,7 @@ import type { DeviceBinding, NetworkDumpInput, PlatformRuntimeHost, + RuntimeOperationFact, PlatformRuntimeOperations, PlatformRuntimeOwner, EnsureReadyInput, @@ -15,9 +16,16 @@ import { bindElementTextRuntime, bindLocalSnapshotInteractor, elementTextRuntimeOperationFacts, + ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, + bindLocalGestureInteractor, + bindLocalScrollInteractor, focusRuntimeOperationFacts, + gestureRuntimeOperationFacts, + scrollRuntimeOperationFacts, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, typeTextRuntimeOperationFacts, localRuntimeOwner, + whenAdmitted, screenshotRuntimeOperationFacts, selectorObservationRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -145,6 +153,36 @@ function androidRuntimeHintsFact(device: DeviceInfo) { } /** adb drives every interaction cell the same way; only the synthetic `simulator` row lacks a device. */ +const gestureKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: 'Gestures are supported on Android emulators and physical devices.', +} as const); +const androidTvMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); +const androidTvDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); + +/** + * A TV target has no touch input at all, which is the one Android gate the retired admission + * carried — it applied to two-contact synthesis and to target-authored drag, never to a plain + * one-contact fling or pan (the D-pad adapter executes those). + */ +function androidTouchTargetFact(device: DeviceInfo, refusal: RuntimeOperationFact) { + if (device.kind === 'simulator') return gestureKindUnavailable; + return device.target === 'tv' ? refusal : available; +} + +function androidGestureFact(device: DeviceInfo) { + return device.kind === 'simulator' ? gestureKindUnavailable : available; +} + function androidTouchFact(device: DeviceInfo) { return device.kind === 'simulator' ? focusKindUnavailable : available; } @@ -179,6 +217,16 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ...focusRuntimeOperationFacts({ focus: androidTouchFact(device) }), + ...gestureRuntimeOperationFacts({ + plan: androidGestureFact(device), + directionalFling: androidGestureFact(device), + multiTouch: androidTouchTargetFact(device, androidTvMultiTouchUnavailable), + targetAuthoredDrag: androidTouchTargetFact(device, androidTvDragUnavailable), + viewport: androidGestureFact(device), + }), + // `scroll` had no admission beyond its capability bucket, so its cell is that bucket + // verbatim: every Android kind but the synthetic `simulator` row. + ...scrollRuntimeOperationFacts({ scroll: androidGestureFact(device) }), // Text entry shares focus's cell: adb drives both, and only the synthetic `simulator` // row has no device behind it (parity with the retired `type` bucket). ...typeTextRuntimeOperationFacts({ type: androidTouchFact(device) }), @@ -233,41 +281,54 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor networkDump: async (input: NetworkDumpInput) => await dumpAndroidNetworkTraffic(host, request.device, input, request.scope.signal), ...recording, - ...(facts.operations.captureSnapshot.available - ? bindLocalSnapshotInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.captureScreenshot.available - ? bindLocalScreenshotInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.focusPoint.available - ? bindLocalFocusInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.typeText.available - ? bindLocalTypeTextInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.readTextAtPoint.available - ? bindElementTextRuntime({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), + ...whenAdmitted(facts.operations.captureSnapshot, () => + bindLocalSnapshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...whenAdmitted(facts.operations.captureScreenshot, () => + bindLocalScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...whenAdmitted(facts.operations.focusPoint, () => + bindLocalFocusInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...bindLocalGestureInteractor({ + device: request.device, + signal: request.scope.signal, + facts: facts.operations, + resolveInteractor: host.localInteractors.resolve, + }), + ...whenAdmitted(facts.operations.scrollDirection, () => + bindLocalScrollInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...whenAdmitted(facts.operations.typeText, () => + bindLocalTypeTextInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...whenAdmitted(facts.operations.readTextAtPoint, () => + bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), ensureReady: async (input: EnsureReadyInput) => await ensureAndroidReady( host, diff --git a/packages/platform-apple/src/gesture-facts.test.ts b/packages/platform-apple/src/gesture-facts.test.ts new file mode 100644 index 000000000..9f9e36c17 --- /dev/null +++ b/packages/platform-apple/src/gesture-facts.test.ts @@ -0,0 +1,108 @@ +import { expect, test } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createApplePlatformRuntime } from './runtime.ts'; +import { platformRuntimeHostFixture } from './runtime.fixtures.ts'; + +function appleDevice(overrides: Partial = {}): DeviceInfo { + return { + platform: 'apple', + appleOs: 'ios', + id: 'apple-fact', + name: 'Apple', + kind: 'simulator', + target: 'mobile', + booted: true, + ...overrides, + }; +} + +const leaves = { + ios: appleDevice(), + ipados: appleDevice({ appleOs: 'ipados' }), + tvos: appleDevice({ appleOs: 'tvos', target: 'tv' }), + macos: appleDevice({ appleOs: 'macos', kind: 'device', target: 'desktop' }), + visionos: appleDevice({ appleOs: 'visionos' }), + watchos: appleDevice({ appleOs: 'watchos' }), +}; + +// R42/R43: the gesture-tier and scroll cells the retired `requireGestureSupported` used to +// decide inside the daemon. Each row is one Apple leaf's complete gesture table, so a cell that +// silently widens (or narrows) fails here rather than on a device. +test.each([ + // leaf, plan, directionalFling, multiTouch, drag, viewport, scroll + ['iOS simulator', leaves.ios, true, true, true, true, true, true], + [ + 'iOS physical', + appleDevice({ kind: 'device', iosPhysicalDeviceBackend: 'coredevice' }), + true, + true, + false, + true, + true, + true, + ], + ['iPadOS simulator', leaves.ipados, true, true, true, true, true, true], + ['tvOS simulator', leaves.tvos, true, true, false, false, true, true], + ['macOS host', leaves.macos, true, true, false, false, true, true], + ['visionOS simulator', leaves.visionos, false, false, false, false, true, true], + ['watchOS sentinel', leaves.watchos, false, false, false, false, false, true], +])( + 'declares the %s gesture and scroll cells', + async (_name, device, plan, directionalFling, multiTouch, drag, viewport, scroll) => { + const facts = await createApplePlatformRuntime(platformRuntimeHostFixture()).inspectFacts( + device, + ); + expect(facts.operations.performGesturePlan.available).toBe(plan); + expect(facts.operations.performDirectionalFlingPlan.available).toBe(directionalFling); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(multiTouch); + expect(facts.operations.performTargetAuthoredDrag.available).toBe(drag); + expect(facts.operations.gestureViewport.available).toBe(viewport); + expect(facts.operations.scrollDirection.available).toBe(scroll); + }, +); + +test('carries the retired multi-touch hints verbatim on every Apple leaf that refused', async () => { + const runtime = createApplePlatformRuntime(platformRuntimeHostFixture()); + const physical = await runtime.inspectFacts(appleDevice({ kind: 'device' })); + expect(physical.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-device-kind', + hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.', + }); + const macos = await runtime.inspectFacts(leaves.macos); + expect(macos.operations.performMultiTouchGesturePlan).toMatchObject({ + available: false, + hint: expect.stringContaining('macOS automation has no multi-touch input'), + }); + expect(macos.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + // watchOS was caught by the retired admission's FIRST branch, before the policy that carries + // the per-OS hints ever ran, so it refuses without one. + const watchos = await runtime.inspectFacts(leaves.watchos); + expect(watchos.operations.performGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); +}); + +test('binds only the gesture tiers the leaf admitted', async () => { + const bind = async (device: DeviceInfo) => + await createApplePlatformRuntime(platformRuntimeHostFixture()).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const simulator = await bind(leaves.ios); + expect(simulator.operations.performGesturePlan).toBeTypeOf('function'); + expect(simulator.operations.performMultiTouchGesturePlan).toBeTypeOf('function'); + expect(simulator.operations.scrollDirection).toBeTypeOf('function'); + const physical = await bind(appleDevice({ kind: 'device' })); + expect(physical.operations.performGesturePlan).toBeTypeOf('function'); + expect(physical.operations.performMultiTouchGesturePlan).toBeUndefined(); +}); diff --git a/packages/platform-apple/src/gesture-facts.ts b/packages/platform-apple/src/gesture-facts.ts new file mode 100644 index 000000000..5c3fa2493 --- /dev/null +++ b/packages/platform-apple/src/gesture-facts.ts @@ -0,0 +1,118 @@ +import { + APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS, + gestureRuntimeOperationFacts, + PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, + scrollRuntimeOperationFacts, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, + type RuntimeOperationFact, +} from '@agent-device/contracts/platform'; +import { resolveDeviceAppleOs, type DeviceInfo } from '@agent-device/kernel/device'; + +/** + * The Apple owner's gesture-family cell table (R42/R43). + * + * This is admission the daemon used to own: `requireGestureSupported` decided Apple's gesture + * tiers from inside `core/capabilities.ts`. Every refusal below reproduces the exact cell — and + * the exact hint — that function produced, which is why the wording constants are imported rather + * than restated. `runtime.ts` composes these facts; it does not decide them. + */ +const available = Object.freeze({ available: true } as const); +const gestureLeafUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', +} as const); +const gestureKindUnavailable = unsupportedAppleDeviceKind( + 'Gestures are supported only for Apple simulators and devices.', +); +const physicalIosMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); +const targetAuthoredDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); +const scrollKindUnavailable = unsupportedAppleDeviceKind( + 'scroll is supported only for Apple simulators and devices.', +); + +function unsupportedAppleDeviceKind(hint: string) { + return Object.freeze({ available: false, reason: 'unsupported-device-kind', hint } as const); +} + +/** The gesture and scroll cells one Apple leaf declares, ready to spread into its fact catalog. */ +export function appleGestureAndScrollFacts(device: DeviceInfo) { + return { + ...gestureRuntimeOperationFacts({ + plan: appleGesturePlanFact(device), + directionalFling: appleGesturePlanFact(device), + multiTouch: appleMultiTouchGestureFact(device), + targetAuthoredDrag: appleTargetAuthoredDragFact(device), + viewport: appleGestureViewportFact(device), + }), + ...scrollRuntimeOperationFacts({ scroll: appleScrollFact(device) }), + }; +} + +/** + * One-contact gesture execution. The retired admission refused watchOS with every other + * `platform === 'web'` case and refused visionOS just after the multi-touch branch, both by + * reading `device.appleOs` RAW — an Apple device that declares no OS was admitted, so this reads + * it raw too rather than resolving a default that would newly refuse. + */ +function appleGesturePlanFact(device: DeviceInfo): RuntimeOperationFact { + if (device.appleOs === 'watchos' || device.appleOs === 'visionos') return gestureLeafUnavailable; + return appleTouchKind(device) ? available : gestureKindUnavailable; +} + +/** + * Two-contact synthesis, which on Apple is the iOS-simulator-only XCTest two-finger model. watchOS + * is refused with no hint because the retired admission caught it in its first branch, before the + * multi-touch policy that carries the per-OS hints ever ran. + */ +function appleMultiTouchGestureFact(device: DeviceInfo): RuntimeOperationFact { + if (device.appleOs === 'watchos') return gestureLeafUnavailable; + if (!appleTouchKind(device)) return gestureKindUnavailable; + const appleOs = resolveDeviceAppleOs(device); + if (appleOs !== 'ios' && appleOs !== 'ipados') { + const hint = APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS[appleOs]; + return Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + ...(hint === undefined ? {} : { hint }), + } as const); + } + return device.kind === 'simulator' ? available : physicalIosMultiTouchUnavailable; +} + +/** Target-authored drag needs source hold, timed movement, and destination hold preserved. */ +function appleTargetAuthoredDragFact(device: DeviceInfo): RuntimeOperationFact { + if (!appleTouchKind(device)) return gestureKindUnavailable; + const supported = + device.appleOs === undefined + ? device.target !== 'desktop' && device.target !== 'tv' + : device.appleOs === 'ios' || device.appleOs === 'ipados'; + return supported ? available : targetAuthoredDragUnavailable; +} + +/** The runner reads the frame for every Apple leaf that has one; watchOS has no runner at all. */ +function appleGestureViewportFact(device: DeviceInfo): RuntimeOperationFact { + if (device.appleOs === 'watchos') return gestureLeafUnavailable; + return appleTouchKind(device) ? available : gestureKindUnavailable; +} + +/** + * `scroll` had no admission beyond its capability bucket — no plugin closure, no gesture policy — + * so its cell is the bucket verbatim, watchOS included. A watchOS scroll still fails where it + * fails today: when the Apple interactor refuses to construct, not at admission. + */ +function appleScrollFact(device: DeviceInfo): RuntimeOperationFact { + return appleTouchKind(device) ? available : scrollKindUnavailable; +} + +/** The two kinds the Apple capability bucket ever admitted. */ +function appleTouchKind(device: DeviceInfo): boolean { + return device.kind === 'simulator' || device.kind === 'device'; +} diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 47f71cbcd..70beef08a 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -10,6 +10,8 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalFocusInteractor, + bindLocalGestureInteractor, + bindLocalScrollInteractor, bindLocalScreenshotInteractor, bindLocalTypeTextInteractor, bindElementTextRuntime, @@ -18,6 +20,7 @@ import { typeTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, + whenAdmitted, selectorObservationRuntimeOperationFacts, snapshotRuntimeOperationFacts, viewportRuntimeOperationFacts, @@ -28,6 +31,7 @@ import { resolveDeviceAppleOs, type DeviceInfo, } from '@agent-device/kernel/device'; +import { appleGestureAndScrollFacts } from './gesture-facts.ts'; import { createAppleAppLogRuntime } from './logs/runtime.ts'; import { dumpAppleNetworkTraffic } from './network/runtime.ts'; import { @@ -269,6 +273,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ...focusRuntimeOperationFacts({ focus: appleFocusFact(device) }), + ...appleGestureAndScrollFacts(device), // Text entry rides the same interactor authority the point focus does, so it shares the // exact kind cell (parity with the retired `type` bucket, `{ simulator, device }`). ...typeTextRuntimeOperationFacts({ type: appleFocusFact(device) }), @@ -331,6 +336,19 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR resolveInteractor: host.localInteractors.resolve, }), ), + ...bindLocalGestureInteractor({ + device: request.device, + signal: request.scope.signal, + facts: facts.operations, + resolveInteractor: host.localInteractors.resolve, + }), + ...(facts.operations.scrollDirection.available + ? bindLocalScrollInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ...whenAdmitted(facts.operations.typeText, () => bindLocalTypeTextInteractor({ device: request.device, @@ -439,15 +457,3 @@ function appleSnapshotFacts(device: DeviceInfo) { withoutActiveApp: isIosFamily(device) ? snapshotActiveAppRequired : capture, }); } - -/** - * An operation is present on a binding only when the owner's own facts admitted it. One helper so - * the binding below reads as a list of admitted operations rather than a chain of branches — and - * so the next operation added here costs no additional complexity. - */ -function whenAdmitted( - fact: RuntimeOperationFact, - build: () => T, -): T | Record { - return fact.available ? build() : {}; -} diff --git a/packages/platform-harmonyos/src/runtime.test.ts b/packages/platform-harmonyos/src/runtime.test.ts index 10d43198a..ad9913d0e 100644 --- a/packages/platform-harmonyos/src/runtime.test.ts +++ b/packages/platform-harmonyos/src/runtime.test.ts @@ -227,3 +227,52 @@ function expectLegacyLifecycleCell( } } } + +// R42/R43: hdc synthesizes one contact, so HarmonyOS admits the one-contact tiers on the same +// kind cell its focus/type overlay admitted, and refuses the two tiers it cannot reproduce. +test.each([ + ['device', device], + ['emulator', { ...device, kind: 'emulator' as const }], +])('declares the HarmonyOS %s gesture and scroll cells', async (_name, runtimeDevice) => { + const facts = await createHarmonyPlatformRuntime(gestureHost()).inspectFacts(runtimeDevice); + expect(facts.operations.performGesturePlan).toEqual({ available: true }); + expect(facts.operations.performDirectionalFlingPlan).toEqual({ available: true }); + expect(facts.operations.gestureViewport).toEqual({ available: true }); + expect(facts.operations.scrollDirection).toEqual({ available: true }); + // The retired admission refused two-contact synthesis on every platform that is neither + // Android nor Apple, with no hint — that is this cell, verbatim. + expect(facts.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); +}); + +test('binds the HarmonyOS gesture tiers it admitted and omits the rest', async () => { + const binding = await createHarmonyPlatformRuntime(gestureHost()).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + expect(binding.operations.performGesturePlan).toBeTypeOf('function'); + expect(binding.operations.gestureViewport).toBeTypeOf('function'); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); + expect(binding.operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(binding.operations.performTargetAuthoredDrag).toBeUndefined(); +}); + +function gestureHost(): PlatformRuntimeHost { + return { + processTransports: { resolve: async () => ({ mode: 'local' as const }) }, + appInventory: { harmonyos: { listApps: async () => [] } }, + appState: { harmonyos: { run: async () => ({ stdout: '' }) } }, + localInteractors: { resolve: async () => ({}) }, + } as unknown as PlatformRuntimeHost; +} diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index 6b0b0a3fb..0a57149b6 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -9,11 +9,16 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalFocusInteractor, + bindLocalGestureInteractor, + bindLocalScrollInteractor, bindLocalScreenshotInteractor, bindLocalTypeTextInteractor, bindLocalSnapshotInteractor, elementTextRuntimeOperationFacts, focusRuntimeOperationFacts, + gestureRuntimeOperationFacts, + scrollRuntimeOperationFacts, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, typeTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, @@ -130,10 +135,35 @@ function harmonyCloseTargetFact(device: DeviceInfo) { : closeTargetKindUnavailable; } +const gestureKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: 'Gestures are supported on HarmonyOS emulators and physical devices.', +} as const); +/** + * hdc synthesizes one contact. The retired admission refused two-contact synthesis on every + * platform that is neither Android nor Apple, with no hint — that is this cell. + */ +const multiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', +} as const); +const targetAuthoredDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); + function harmonyFocusFact(device: DeviceInfo): RuntimeOperationFact { return device.kind === 'emulator' || device.kind === 'device' ? available : focusKindUnavailable; } +function harmonyGestureFact(device: DeviceInfo): RuntimeOperationFact { + return device.kind === 'emulator' || device.kind === 'device' + ? available + : gestureKindUnavailable; +} + export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): PlatformRuntimeOwner { const appLogs = createHarmonyAppLogRuntime(host); const inspectFacts = async (device: Parameters[0]) => { @@ -174,6 +204,16 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ...focusRuntimeOperationFacts({ focus: harmonyFocusFact(device) }), + // Gestures share focus's kind cell (the overlay admitted `{emulator, device}`); only the + // two tiers hdc cannot synthesize are refused. + ...gestureRuntimeOperationFacts({ + plan: harmonyGestureFact(device), + directionalFling: harmonyGestureFact(device), + multiTouch: multiTouchUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: harmonyGestureFact(device), + }), + ...scrollRuntimeOperationFacts({ scroll: harmonyGestureFact(device) }), // Text entry shares focus's cell: hdc drives both on the same two kinds. ...typeTextRuntimeOperationFacts({ type: harmonyFocusFact(device) }), // HarmonyOS has no point-read tool: `get` answers from the captured tree, which is what @@ -254,6 +294,19 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor resolveInteractor: host.localInteractors.resolve, }) : {}), + ...bindLocalGestureInteractor({ + device: request.device, + signal: request.scope.signal, + facts: facts.operations, + resolveInteractor: host.localInteractors.resolve, + }), + ...(facts.operations.scrollDirection.available + ? bindLocalScrollInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), listApps: async (input: { device: DeviceInfo; filter: 'all' | 'user-installed' }) => await host.appInventory.harmonyos.listApps( input.device, diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index 96938457c..7a0a93295 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -216,3 +216,57 @@ test('the Linux surface capture composes the per-capture signal with the request scope.abort(); expect(passedSecond.aborted).toBe(true); }); + +// R42/R43: Linux is the one owner whose gesture tiers genuinely split. Its drag primitive +// preserves a coordinate fling's endpoints but not a direction-authored fling's speed semantics +// (`gesture fling is not supported on Linux`), it synthesizes one contact, and it has no frame +// read of its own — which is why `gestureViewport` is PREFERRED rather than required. +test.each([ + ['desktop device', 'device' as const, true], + ['non-desktop kind', 'emulator' as const, false], +])('declares the Linux %s gesture and scroll cells', async (_name, kind, supported) => { + const facts = await createLinuxPlatformRuntime(lifecycleHost()).inspectFacts({ + platform: 'linux', + id: 'linux', + name: 'Linux', + kind, + target: 'desktop', + booted: true, + }); + expect(facts.operations.performGesturePlan.available).toBe(supported); + expect(facts.operations.scrollDirection.available).toBe(supported); + expect(facts.operations.performDirectionalFlingPlan.available).toBe(false); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(false); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + expect(facts.operations.gestureViewport.available).toBe(false); +}); + +test('binds the Linux coordinate-fling tier without a frame read', async () => { + const binding = await createLinuxPlatformRuntime(lifecycleHost()).bind({ + device: { + platform: 'linux', + id: 'linux', + name: 'Linux', + kind: 'device', + target: 'desktop', + booted: true, + }, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + expect(binding.operations.performGesturePlan).toBeTypeOf('function'); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); + expect(binding.operations.performDirectionalFlingPlan).toBeUndefined(); + expect(binding.operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(binding.operations.performTargetAuthoredDrag).toBeUndefined(); + // Absent, not broken: the caller derives the coordinate frame from a capture instead, which is + // exactly how a Linux gesture resolved its viewport before this migration. + expect(binding.operations.gestureViewport).toBeUndefined(); +}); diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 01e6558b5..9abdb28c5 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -11,6 +11,8 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindLocalFocusInteractor, + bindLocalGestureInteractor, + bindLocalScrollInteractor, bindLocalScreenshotInteractor, bindLocalTypeTextInteractor, bindElementTextRuntime, @@ -18,8 +20,12 @@ import { createUnavailablePlatformRuntimeFacts, elementTextRuntimeOperationFacts, focusRuntimeOperationFacts, + gestureRuntimeOperationFacts, + scrollRuntimeOperationFacts, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, typeTextRuntimeOperationFacts, localRuntimeOwner, + whenAdmitted, sameRuntimeOwner, screenshotRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -40,6 +46,27 @@ const typeKindUnavailable = unavailableLinuxRuntimeFact( 'unsupported-device-kind', 'type is supported only for the Linux desktop device.', ); +const gestureKindUnavailable = unavailableLinuxRuntimeFact( + 'unsupported-device-kind', + 'Gestures are supported only for the Linux desktop device.', +); +const scrollKindUnavailable = unavailableLinuxRuntimeFact( + 'unsupported-device-kind', + 'scroll is supported only for the Linux desktop device.', +); +/** + * The drag primitive preserves a coordinate fling's endpoints but not a direction-authored + * fling's speed semantics, which is the one gesture the retired admission refused BY PLATFORM + * rather than by leaf or kind. + */ +const directionalFlingUnavailable = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); +const multiTouchUnavailable = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); +const targetAuthoredDragUnavailable = unavailableLinuxRuntimeFact( + 'unsupported-platform-leaf', + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +); +/** No frame read of its own: a Linux gesture derives its viewport from a capture, as it does today. */ +const gestureViewportUnavailable = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); const runtimeHintsUnavailable = unavailableLinuxRuntimeFact( 'unsupported-platform-leaf', 'Runtime hints are supported only for local iOS-family simulators and Android devices.', @@ -102,34 +129,47 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR ...(facts.operations.captureSnapshot.available ? linuxSnapshotOperations(host, request) : {}), - ...(facts.operations.captureScreenshot.available - ? bindLocalScreenshotInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.focusPoint.available - ? bindLocalFocusInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.typeText.available - ? bindLocalTypeTextInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.readTextAtPoint.available - ? bindElementTextRuntime({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), + ...whenAdmitted(facts.operations.captureScreenshot, () => + bindLocalScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...whenAdmitted(facts.operations.focusPoint, () => + bindLocalFocusInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...bindLocalGestureInteractor({ + device: request.device, + signal: request.scope.signal, + facts: facts.operations, + resolveInteractor: host.localInteractors.resolve, + }), + ...whenAdmitted(facts.operations.scrollDirection, () => + bindLocalScrollInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...whenAdmitted(facts.operations.typeText, () => + bindLocalTypeTextInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), + ...whenAdmitted(facts.operations.readTextAtPoint, () => + bindElementTextRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), }), [Symbol.asyncDispose]: async () => undefined, }) satisfies DeviceBinding; @@ -148,6 +188,8 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts snapshot: snapshotKindUnavailable, viewport: unsupportedPlatformLeaf, focus: focusKindUnavailable, + gesture: gestureKindUnavailable, + scroll: scrollKindUnavailable, typeText: typeKindUnavailable, elementText: elementTextKindUnavailable, readiness: unsupportedPlatformLeaf, @@ -180,6 +222,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts ...focusRuntimeOperationFacts({ focus: device.kind === 'device' ? supported : focusKindUnavailable, }), + ...linuxGestureFacts(device), // Text entry shares focus's cell: ydotool drives both on the desktop device only. ...typeTextRuntimeOperationFacts({ type: device.kind === 'device' ? supported : typeKindUnavailable, @@ -193,6 +236,25 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts }); } +/** + * Linux's gesture-family cells. The desktop device is the only Linux cell with a pointer to + * drive, and three tiers are refused on every cell: a direction-authored fling's speed semantics, + * two-contact synthesis, and target-authored drag timing. + */ +function linuxGestureFacts(device: DeviceInfo) { + const pointer = device.kind === 'device'; + return { + ...gestureRuntimeOperationFacts({ + plan: pointer ? supported : gestureKindUnavailable, + directionalFling: directionalFlingUnavailable, + multiTouch: multiTouchUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: gestureViewportUnavailable, + }), + ...scrollRuntimeOperationFacts({ scroll: pointer ? supported : scrollKindUnavailable }), + }; +} + function linuxSnapshotOperations( host: PlatformRuntimeHost, request: Parameters[0], diff --git a/packages/platform-vega/src/runtime.test.ts b/packages/platform-vega/src/runtime.test.ts index a58f27f18..5eddeef8d 100644 --- a/packages/platform-vega/src/runtime.test.ts +++ b/packages/platform-vega/src/runtime.test.ts @@ -186,3 +186,41 @@ function expectLifecycleFacts( } } } + +// R42/R43: `gesture`, `scroll` and `swipe` never carried a vega capability bucket, so no cell of +// the gesture family was ever admitted on this owner. +test('declares every Vega gesture and scroll cell unavailable', async () => { + const facts = await createVegaPlatformRuntime(lifecycleHost()).inspectFacts({ + platform: 'vega', + id: 'vega-vvd', + name: 'Vega VVD', + kind: 'emulator', + target: 'tv', + booted: true, + }); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'gestureViewport', + ] as const) { + expect(facts.operations[operation]).toMatchObject({ + available: false, + hint: expect.stringContaining('remote navigation only'), + }); + } + // Two tiers keep the retired closures' own wording instead of this owner's: two-contact + // synthesis refused with no hint at all on a non-Android, non-Apple platform, and + // target-authored drag refused by naming the phases an adapter must preserve. + expect(facts.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + expect(facts.operations.scrollDirection).toMatchObject({ + available: false, + hint: expect.stringContaining('remote navigation only'), + }); +}); diff --git a/packages/platform-vega/src/runtime.ts b/packages/platform-vega/src/runtime.ts index c071e08f3..8e48f5e73 100644 --- a/packages/platform-vega/src/runtime.ts +++ b/packages/platform-vega/src/runtime.ts @@ -10,6 +10,8 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, createUnavailablePlatformRuntimeFacts, + gestureRuntimeOperationFacts, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, localRuntimeOwner, sameRuntimeOwner, } from '@agent-device/contracts/platform'; @@ -88,12 +90,30 @@ const typeUnavailable = vegaUnavailable( 'unsupported-platform-leaf', 'type is not supported on Vega OS: the Vega runtime exposes remote navigation only.', ); +const gestureUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + 'Gestures are not supported on Vega OS: the Vega runtime exposes remote navigation only.', +); +/** + * The two tiers the retired admission refused BY NAME on a non-Android, non-Apple platform. Their + * wording is the retired closures' own — two-contact synthesis refused with no hint at all, and + * target-authored drag refused by naming the phases an adapter has to preserve. + */ +const multiTouchUnavailable = vegaUnavailable('unsupported-platform-leaf'); +const targetAuthoredDragUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +); +const scrollUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + 'scroll is not supported on Vega OS: the Vega runtime exposes remote navigation only.', +); function vegaFacts(device: DeviceInfo): RuntimeFacts { const supported = device.kind === 'emulator' && device.target === 'tv'; const openTarget = supported ? lifecycleAvailable : openTargetUnavailable; const closeTarget = supported ? lifecycleAvailable : closeTargetUnavailable; - return createUnavailablePlatformRuntimeFacts(device, vegaOwner, { + const unavailable = createUnavailablePlatformRuntimeFacts(device, vegaOwner, { appLog: unsupportedPlatformLeaf, network: unsupportedPlatformLeaf, screenshot: screenshotUnavailable, @@ -101,6 +121,10 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts viewport: unsupportedPlatformLeaf, // Vega exposes remote navigation only; it never carried a `focus` capability bucket. focus: focusUnavailable, + // Vega exposes remote navigation only; `gesture`, `scroll` and `swipe` never carried a vega + // capability bucket, so no gesture-family cell was ever admitted here. + gesture: gestureUnavailable, + scroll: scrollUnavailable, typeText: typeUnavailable, elementText: unsupportedPlatformLeaf, readiness: unsupportedPlatformLeaf, @@ -116,6 +140,25 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts configureProviderPortReverse: providerPortReverseUnavailable, }), }); + return Object.freeze({ + device: unavailable.device, + operations: { + ...unavailable.operations, + // Two tiers keep the retired closures' own wording rather than this owner's generic one: + // the retired admission refused two-contact synthesis on every non-Android, non-Apple + // platform with no hint, and refused target-authored drag by naming the phases an adapter + // must preserve. The remaining tiers had no retired closure at all — they failed when the + // Vega interactor turned out to have no gesture execution — so they carry this owner's own + // hint and now refuse at admission instead. + ...gestureRuntimeOperationFacts({ + plan: gestureUnavailable, + directionalFling: gestureUnavailable, + multiTouch: multiTouchUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: gestureUnavailable, + }), + }, + }); } function vegaUnavailable( diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index 4d1324138..4c1350c92 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -353,3 +353,45 @@ function host( }, } as unknown as PlatformRuntimeHost; } + +// R42/R43: `scroll` is the one gesture-family command the web overlay ever admitted +// (`WEB_INTERACTION_COMMANDS`); `gesture` and `swipe` carried no web bucket at all, and the +// retired admission refused `platform === 'web'` outright. +test('admits web scrolling and refuses every gesture tier', async () => { + const binding = await createWebPlatformRuntime(host({ mode: 'local' })).bind({ + device, + intent: { kind: 'ordinary' }, + scope: scope(), + }); + expect(binding.facts.operations.scrollDirection).toEqual({ available: true }); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'gestureViewport', + ] as const) { + expect(binding.facts.operations[operation]).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); + expect(binding.operations[operation]).toBeUndefined(); + } + // The retired admission checked drag FIRST, before its `platform === 'web'` branch, so this one + // tier keeps the target-authored-drag wording rather than the bare platform refusal. + expect(binding.facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + expect(binding.operations.performTargetAuthoredDrag).toBeUndefined(); +}); + +test('refuses web scrolling on a non-browser web cell', async () => { + const binding = await createWebPlatformRuntime(host({ mode: 'local' })).bind({ + device: { ...device, kind: 'emulator' }, + intent: { kind: 'ordinary' }, + scope: scope(), + }); + expect(binding.facts.operations.scrollDirection.available).toBe(false); + expect(binding.operations.scrollDirection).toBeUndefined(); +}); diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index c14e9086f..38d2b9b46 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -13,7 +13,11 @@ import { bindLocalTypeTextInteractor, bindLocalSnapshotInteractor, elementTextRuntimeOperationFacts, + bindLocalScrollInteractor, focusRuntimeOperationFacts, + gestureRuntimeOperationFacts, + scrollRuntimeOperationFacts, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, typeTextRuntimeOperationFacts, localRuntimeOwner, screenshotRuntimeOperationFacts, @@ -67,6 +71,16 @@ const prepareUnavailable = Object.freeze({ hint: 'Apple runner preparation is supported only for Apple targets.', } as const); +/** `gesture` and `swipe` never carried a web capability bucket; the browser drives no synthesis. */ +const gestureUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', +} as const); +const targetAuthoredDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); const openTargetKindUnavailable = Object.freeze({ available: false, reason: 'unsupported-device-kind', @@ -202,6 +216,13 @@ function bindWebRuntime( resolveInteractor: host.localInteractors.resolve, }) : {}), + ...(facts.operations.scrollDirection.available + ? bindLocalScrollInteractor({ + device, + signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ...(facts.operations.setViewport.available ? { setViewport: async (input) => { @@ -280,6 +301,21 @@ function webRuntimeFacts( // Text entry shares focus's cell: the browser device is the only web cell with an // interactor to drive (parity with the retired `type` overlay membership). ...typeTextRuntimeOperationFacts({ type: browserDevice }), + // `scroll` is the one gesture-family command the web overlay admitted + // (`WEB_INTERACTION_COMMANDS`), so it shares focus's `{ device: true }` cell. `gesture` and + // `swipe` never carried a web bucket, and the retired admission refused `platform === 'web'` + // outright — every gesture tier stays unavailable here. + ...scrollRuntimeOperationFacts({ scroll: browserDevice }), + // The retired admission refused `platform === 'web'` with no hint for every tier except + // target-authored drag, which it refused earlier by naming the phases an adapter has to + // preserve — so that one tier keeps its own wording. + ...gestureRuntimeOperationFacts({ + plan: gestureUnavailable, + directionalFling: gestureUnavailable, + multiTouch: gestureUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: gestureUnavailable, + }), ...viewportRuntimeOperationFacts({ setViewport: browserDevice }), // The web backend has no point-addressed read: `get` answers from the captured DOM tree, // which is what the legacy dispatch already did once its Apple-runner attempt failed. diff --git a/packages/provider-limrun/src/app-log-runtime.test.ts b/packages/provider-limrun/src/app-log-runtime.test.ts index 7ec574758..01d996373 100644 --- a/packages/provider-limrun/src/app-log-runtime.test.ts +++ b/packages/provider-limrun/src/app-log-runtime.test.ts @@ -364,3 +364,78 @@ test('fails closed for a stale Android identity before exposing facts or binding }); expect(getAppState).not.toHaveBeenCalled(); }); + +// R42/R43: Limrun's two session kinds run different interactors. The Android emulator session +// runs the ordinary Android interactor (every tier); the iOS direct session's own +// `performGesture` refuses, so stating that as a fact refuses at admission instead of +// mid-execution — with the interactor's wording preserved. +const limrunAndroid = { + platform: 'android' as const, + id: 'limrun:android:lease-a', + name: 'Limrun Android', + kind: 'emulator' as const, + target: 'mobile' as const, + booted: true, +}; + +test('admits every gesture tier on a Limrun Android session', async () => { + const owner = createLimrunPlatformRuntimeOwner( + limrunOwnerOptions({ getInteractor: () => ({}) as never }), + ); + const binding = await owner.bind({ + device: limrunAndroid, + intent: { kind: 'ordinary' }, + scope, + }); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + 'scrollDirection', + ] as const) { + expect(binding.facts.operations[operation]).toEqual({ available: true }); + expect(binding.operations[operation]).toBeTypeOf('function'); + } +}); + +test('refuses Limrun iOS gestures at admission while keeping its scroll', async () => { + const owner = createLimrunPlatformRuntimeOwner( + limrunOwnerOptions({ getInteractor: () => ({}) as never }), + ); + const binding = await owner.bind({ device, intent: { kind: 'ordinary' }, scope }); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + ] as const) { + expect(binding.facts.operations[operation]).toEqual({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'Limrun iOS direct sessions do not expose portable gesture execution yet.', + }); + expect(binding.operations[operation]).toBeUndefined(); + } + // The iOS direct session exposes scrolling directly — no gesture synthesis involved. + expect(binding.facts.operations.scrollDirection).toEqual({ available: true }); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); +}); + +test('closes every Limrun gesture and scroll cell without a live session', async () => { + const owner = createLimrunPlatformRuntimeOwner( + limrunOwnerOptions({ getInteractor: () => ({}) as never, hasLiveSession: () => false }), + ); + const facts = await owner.inspectFacts(limrunAndroid); + for (const operation of [ + 'performGesturePlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + 'scrollDirection', + ] as const) { + expect(facts.operations[operation].available).toBe(false); + } +}); diff --git a/packages/provider-limrun/src/app-log-runtime.ts b/packages/provider-limrun/src/app-log-runtime.ts index e6a78c47d..74a41ffe1 100644 --- a/packages/provider-limrun/src/app-log-runtime.ts +++ b/packages/provider-limrun/src/app-log-runtime.ts @@ -206,6 +206,8 @@ export function createLimrunPlatformRuntimeOwner( screenshot: liveSessionUnavailable, viewport: liveSessionUnavailable, focus: liveSessionUnavailable, + gesture: liveSessionUnavailable, + scroll: liveSessionUnavailable, typeText: liveSessionUnavailable, elementText: liveSessionUnavailable, readiness: liveSessionUnavailable, @@ -458,7 +460,7 @@ function facts( ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), // Focus rides the same provider interactor the captures do, and a live-session Limrun // device always has one, so it is available wherever a capture is. - ...limrunInteractionOperationFacts(), + ...limrunInteractionOperationFacts(device), ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), ensureReady: available, bootTarget: available, @@ -504,7 +506,7 @@ function recoveryFacts( findSelector: liveSessionUnavailable, }), ...viewportRuntimeOperationFacts({ setViewport: liveSessionUnavailable }), - ...limrunInteractionOperationFacts(liveSessionUnavailable), + ...limrunInteractionOperationFacts(device, liveSessionUnavailable), ensureReady: liveSessionUnavailable, bootTarget: liveSessionUnavailable, bootTargetHeadless: liveSessionUnavailable, diff --git a/packages/provider-limrun/src/interaction-operations.ts b/packages/provider-limrun/src/interaction-operations.ts index 5f9247ffc..3fe6f478a 100644 --- a/packages/provider-limrun/src/interaction-operations.ts +++ b/packages/provider-limrun/src/interaction-operations.ts @@ -1,16 +1,80 @@ import { + ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, bindProviderFocusInteractor, + bindProviderGestureInteractor, bindProviderScreenshotInteractor, + bindProviderScrollInteractor, bindProviderSnapshotInteractor, bindProviderTypeTextInteractor, focusRuntimeOperationFacts, + gestureRuntimeOperationFacts, + scrollRuntimeOperationFacts, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, typeTextRuntimeOperationFacts, + type GestureRuntimeOperationFacts, + type RuntimeOperationUnavailability, } from '@agent-device/contracts/platform'; import type { Interactor, RunnerContext } from '@agent-device/contracts/interaction'; -import type { RuntimeOperationUnavailability } from '@agent-device/contracts/platform'; import type { DeviceInfo } from '@agent-device/kernel/device'; const available = Object.freeze({ available: true } as const); +/** + * Limrun's iOS direct session drives text and touch but exposes no portable gesture execution — + * its interactor's own `performGesture` refuses with this wording. Stating it as a fact refuses + * at admission instead of mid-execution (ADR 0019 §6), keeping the agent-facing hint identical. + */ +const iosGestureUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'Limrun iOS direct sessions do not expose portable gesture execution yet.', +} as const); +const androidTvMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); +const androidTvDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); + +/** + * Gesture cells split by the interactor behind the session: an Android emulator session runs the + * ordinary Android interactor (every tier, minus the TV gates it always carried), while the iOS + * direct session has no gesture execution at all. + */ +function limrunGestureFacts( + device: DeviceInfo, + cell: RuntimeOperationUnavailability | typeof available, +): GestureRuntimeOperationFacts { + if (cell !== available) { + return gestureRuntimeOperationFacts({ + plan: cell, + directionalFling: cell, + multiTouch: cell, + targetAuthoredDrag: cell, + viewport: cell, + }); + } + if (device.platform !== 'android') { + return gestureRuntimeOperationFacts({ + plan: iosGestureUnavailable, + directionalFling: iosGestureUnavailable, + multiTouch: iosGestureUnavailable, + targetAuthoredDrag: iosGestureUnavailable, + viewport: iosGestureUnavailable, + }); + } + const touchTarget = device.target === 'tv'; + return gestureRuntimeOperationFacts({ + plan: available, + directionalFling: available, + multiTouch: touchTarget ? androidTvMultiTouchUnavailable : available, + targetAuthoredDrag: touchTarget ? androidTvDragUnavailable : available, + viewport: available, + }); +} /** * The interactor-backed interaction cells a live Limrun session serves: everything here rides @@ -19,16 +83,24 @@ const available = Object.freeze({ available: true } as const); * behavior — the owner module composes this, it does not define it. */ export function limrunInteractionOperationFacts( + device: DeviceInfo, liveSessionUnavailable?: RuntimeOperationUnavailability, ) { const cell = liveSessionUnavailable ?? available; return Object.freeze({ ...focusRuntimeOperationFacts({ focus: cell }), ...typeTextRuntimeOperationFacts({ type: cell }), + ...limrunGestureFacts(device, cell), + // `scroll` needs no gesture synthesis: both session kinds expose it directly. + ...scrollRuntimeOperationFacts({ scroll: cell }), }); } -/** Binds the interactor-backed operations (snapshot, screenshot, focus, type) for one session. */ +/** + * Binds the interactor-backed operations (snapshot, screenshot, focus, type, gestures, scroll) + * for one session. Only a live session reaches here, so the gesture tiers are gated by the same + * device split their facts use rather than by liveness. + */ export function bindLimrunInteractionOperations( params: Readonly<{ device: DeviceInfo; @@ -43,5 +115,12 @@ export function bindLimrunInteractionOperations( ...bindProviderFocusInteractor({ device, signal, resolveInteractor }), ...bindProviderTypeTextInteractor({ device, signal, resolveInteractor }), ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderGestureInteractor({ + device, + signal, + facts: limrunGestureFacts(device, available), + resolveInteractor, + }), + ...bindProviderScrollInteractor({ device, signal, resolveInteractor }), }); } diff --git a/packages/provider-webdriver/src/platform-runtime.test.ts b/packages/provider-webdriver/src/platform-runtime.test.ts index 1d9d26f01..980f6c17f 100644 --- a/packages/provider-webdriver/src/platform-runtime.test.ts +++ b/packages/provider-webdriver/src/platform-runtime.test.ts @@ -421,3 +421,53 @@ function host(run: PlatformRuntimeHost['commands']['run']): PlatformRuntimeHost }, } as unknown as PlatformRuntimeHost; } + +// R42/R43: gestures and scrolling ride the same reachability gate the captures do. The one extra +// gate is the retired multi-touch policy — this provider owns physical devices only, and +// two-finger synthesis on a physical iOS device was refused before this migration too. +test.each([ + ['Android physical', device, true], + ['iOS physical', { ...device, platform: 'apple' as const, appleOs: 'ios' as const }, false], +])('declares the WebDriver %s gesture and scroll cells', async (_name, owned, multiTouch) => { + const owner = createWebDriverPlatformRuntimeOwner({ + host: host(async () => ({ stdout: '', stderr: '', exitCode: 0 })), + owner: providerRuntimeOwner('browserstack', 'android'), + ownsDevice: () => true, + getInteractor: () => ({}) as unknown as Interactor, + }); + const facts = await owner.inspectFacts(owned); + expect(facts.operations.performGesturePlan).toEqual({ available: true }); + expect(facts.operations.performDirectionalFlingPlan).toEqual({ available: true }); + expect(facts.operations.performTargetAuthoredDrag).toEqual({ available: true }); + expect(facts.operations.gestureViewport).toEqual({ available: true }); + expect(facts.operations.scrollDirection).toEqual({ available: true }); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(multiTouch); + if (!multiTouch) { + expect(facts.operations.performMultiTouchGesturePlan).toMatchObject({ + hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.', + }); + } +}); + +test('closes every WebDriver gesture and scroll cell when the interactor is unreachable', async () => { + const owner = createWebDriverPlatformRuntimeOwner({ + host: host(async () => ({ stdout: '', stderr: '', exitCode: 0 })), + owner: providerRuntimeOwner('browserstack', 'android'), + ownsDevice: () => true, + getInteractor: undefined, + }); + const facts = await owner.inspectFacts(device); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + 'scrollDirection', + ] as const) { + expect(facts.operations[operation]).toMatchObject({ + available: false, + reason: 'unsupported-provider-mode', + }); + } +}); diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index dfdcf7659..f600d62c5 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -2,12 +2,17 @@ import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, bindProviderFocusInteractor, + bindProviderGestureInteractor, bindProviderScreenshotInteractor, + bindProviderScrollInteractor, bindProviderTypeTextInteractor, bindProviderSnapshotInteractor, createUnavailablePlatformRuntimeFacts, sameRuntimeOwner, focusRuntimeOperationFacts, + gestureRuntimeOperationFacts, + PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, + scrollRuntimeOperationFacts, screenshotRuntimeOperationFacts, typeTextRuntimeOperationFacts, snapshotRuntimeOperationFacts, @@ -109,6 +114,21 @@ const typeUnavailable = Object.freeze({ reason: 'unsupported-provider-mode', hint: 'This WebDriver provider runtime does not expose text entry for this device.', } as const); +const gestureUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'This WebDriver provider runtime does not expose gestures for this device.', +} as const); +const scrollUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'This WebDriver provider runtime does not expose scrolling for this device.', +} as const); +const physicalIosMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); const elementTextUnavailable = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -258,6 +278,19 @@ function bindWebDriverPlatformRuntime( resolveInteractor: (runner) => options.getInteractor?.(device, runner), }) : {}), + ...bindProviderGestureInteractor({ + device, + signal, + facts: facts.operations, + resolveInteractor: (runner) => options.getInteractor?.(device, runner), + }), + ...(facts.operations.scrollDirection.available + ? bindProviderScrollInteractor({ + device, + signal, + resolveInteractor: (runner) => options.getInteractor?.(device, runner), + }) + : {}), networkDump: async (input) => { const recent = await options.host.appLogs.readRecent(input.sessionId, input.maxScanLines); const dump = readRecentNetworkTrafficFromText(recent.text, { @@ -319,6 +352,8 @@ function webDriverFacts( screenshot: inactiveSession, viewport: inactiveSession, focus: inactiveSession, + gesture: inactiveSession, + scroll: inactiveSession, typeText: inactiveSession, elementText: inactiveSession, lifecycle: applicationLifecycleOperationFacts({ @@ -343,6 +378,8 @@ function webDriverFacts( screenshot: screenshotUnavailable, viewport: viewportUnavailable, focus: focusUnavailable, + gesture: gestureUnavailable, + scroll: scrollUnavailable, typeText: typeUnavailable, elementText: elementTextUnavailable, lifecycle: webDriverLifecycleFacts(device), @@ -374,6 +411,18 @@ function webDriverFacts( // reachability and nothing more: this provider drives touch wherever it can drive a capture. ...focusRuntimeOperationFacts({ focus: interactorCell(reachable, focusUnavailable) }), ...typeTextRuntimeOperationFacts({ type: interactorCell(reachable, typeUnavailable) }), + // Gestures and scrolling ride the same provider interactor the captures do, so they need + // the same reachability. The one extra gate is the retired multi-touch policy: this + // provider only ever owns physical devices, and two-finger synthesis on a physical + // iOS device was refused before this migration exactly as it is refused here. + ...gestureRuntimeOperationFacts({ + plan: interactorCell(reachable, gestureUnavailable), + directionalFling: interactorCell(reachable, gestureUnavailable), + multiTouch: webDriverMultiTouchCell(device, reachable), + targetAuthoredDrag: interactorCell(reachable, gestureUnavailable), + viewport: interactorCell(reachable, gestureUnavailable), + }), + ...scrollRuntimeOperationFacts({ scroll: interactorCell(reachable, scrollUnavailable) }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ensureReady: available, bootTarget: available, @@ -388,6 +437,12 @@ function webDriverFacts( }); } +/** Two-finger synthesis is iOS-simulator only, and this provider owns no simulators. */ +function webDriverMultiTouchCell(device: DeviceInfo, reachable: boolean): RuntimeOperationFact { + if (!reachable) return gestureUnavailable; + return device.platform === 'apple' ? physicalIosMultiTouchUnavailable : available; +} + /** The device shapes this provider can reach at all through its own WebDriver interactor. */ function webDriverInteractorDevice(device: DeviceInfo): boolean { return ( diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index 3ea4849bd..e58edf466 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -26,7 +26,8 @@ import { retiredDispatchProjectionViolations } from './runtime-command-cutover-d * `cutoverTableDefects` rejects a duplicate; lifecycle starts at R28 after the accepted * shutdown, install/deploy, and application-lifecycle allocations. Snapshot starts at R32; * diff follows at R33, find at R35, get at R36, is at R37, screenshot at R39, wait at R38, - * focus at R40, and type at R41 — the Wave 4 observation family is complete. + * focus at R40, and type at R41 — the Wave 4 observation family is complete. Wave 5's gesture + * cluster follows: gesture at R42, scroll at R43, swipe at R44. */ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ { @@ -663,6 +664,105 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ operationOwners: { typeText: ['executeBoundTypeText'] }, }, }, + { + rule: 'R42 gesture-runtime-cutover', + command: 'gesture', + subject: 'gesture execution', + tier: 'request-scoped', + execution: 'device-runtime', + legacyRetirement: { + // The whole admission and the plan dispatcher. `requireGestureSupported` was the last + // daemon-owned platform table in this family: its intent-dependent tiers are now owner + // facts, so the function and its five private helpers go with it. + routeNames: ['requireGestureSupported', 'dispatchGesturePlan'], + // `gesture` also leaves the hand-maintained overlay that granted it a capability bucket on + // a family the descriptor never listed. `dispatchGestureViewport` is deliberately NOT + // claimed here: the Maestro replay port still consumes it, and ADR 0019 §6 keeps a shared + // mechanic in place until its last consumer can move. + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS'], + }, + runtimeTypeNames: ['GestureRuntimeOperations'], + operations: { + names: [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + ], + }, + singularExecution: { + routes: ['handleInteractionCommands'], + operations: [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + ], + // One lexical owner per tier, all four inside the single binder `swipe` shares (R44). + // Four keys rather than one because their CELLS differ, not their mechanics — the tier a + // gesture input selects is what admission proves and what that branch then calls, on a + // binding narrow enough that the operation needs no cast or non-null repair. + operationOwners: { + performGesturePlan: ['bindGestureTier'], + performDirectionalFlingPlan: ['bindGestureTier'], + performMultiTouchGesturePlan: ['bindGestureTier'], + performTargetAuthoredDrag: ['bindGestureTier'], + gestureViewport: ['selectGestureViewport'], + }, + }, + }, + { + rule: 'R43 scroll-runtime-cutover', + command: 'scroll', + subject: 'directional scrolling', + tier: 'request-scoped', + execution: 'device-runtime', + legacyRetirement: { + // The interactor leaf and its dispatch-table arm. Deleting the descriptor's `dispatch` + // leaf drops `'scroll'` from `DescriptorDispatchCommandName`, which makes a surviving + // `DISPATCH_HANDLERS.scroll` a COMPILE error rather than something this row has to police. + routeNames: ['handleScrollCommand'], + // Both overlays: `scroll` was the one gesture-family command the web overlay admitted. + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS', 'WEB_INTERACTION_COMMANDS'], + }, + runtimeTypeNames: ['ScrollRuntimeOperations'], + operations: { names: ['scrollDirection'] }, + singularExecution: { + routes: ['dispatchGenericCommand'], + operations: ['scrollDirection'], + // The edge verification consumes the SHARED `captureSnapshot` the selector family owns, so + // this row claims only the scroll pass itself. + operationOwners: { scrollDirection: ['executeBoundScroll'] }, + }, + }, + { + rule: 'R44 swipe-runtime-cutover', + command: 'swipe', + subject: 'coordinate swipe', + tier: 'request-scoped', + execution: 'device-runtime', + legacyRetirement: { + // `swipe` owned no adapter of its own: it normalized to a coordinate fling and executed + // through the same plan dispatcher `gesture` did (retired by R42). Its whole retirement is + // admission data — the capability bucket the row's automatic columns reject, plus the + // overlay membership below. + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS'], + }, + runtimeTypeNames: ['GestureRuntimeOperations'], + operations: { names: ['performGesturePlan', 'gestureViewport'] }, + singularExecution: { + routes: ['handleInteractionCommands'], + operations: ['performGesturePlan', 'gestureViewport'], + // Shared with `gesture` the way `find` shares the selector family's owners: a swipe series + // binds once and re-enters the same bound executor per repetition. + operationOwners: { + performGesturePlan: ['bindGestureTier'], + gestureViewport: ['selectGestureViewport'], + }, + }, + }, { rule: 'R34 viewport-runtime-cutover', command: 'viewport', diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index a0eb209a3..a6f2d5147 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -1,6 +1,8 @@ import { applicationLifecycleOperationFacts, + gestureRuntimeOperationFacts, screenshotRuntimeOperationFacts, + scrollRuntimeOperationFacts, elementTextRuntimeOperationFacts, snapshotRuntimeOperationFacts, type RuntimeOperationFact, @@ -38,6 +40,14 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre setViewport: unavailable, focusPoint: unavailable, typeText: unavailable, + ...gestureRuntimeOperationFacts({ + plan: unavailable, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: unavailable, + viewport: unavailable, + }), + ...scrollRuntimeOperationFacts({ scroll: unavailable }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: unavailable }), }); diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index 0b4eb5f5c..c9e3fde8e 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -330,6 +330,12 @@ test('web supports only the initial browser interaction slice', () => { 'fill', 'focus', 'find', + // `gesture` and `swipe` (R42/R44) join the migrated commands here for the same reason + // `focus`, `find`, `screenshot`, `scroll`, `snapshot`, `type` and `wait` already do: a + // command whose admission comes from exact owner facts carries no capability-matrix row, + // and a command with no row is not decided by this matrix at all. The web owner refuses + // every gesture tier — `platform-web/src/runtime.test.ts` is where that cell is pinned. + 'gesture', 'get', 'hover', 'press', @@ -337,6 +343,7 @@ test('web supports only the initial browser interaction slice', () => { 'screenshot', 'scroll', 'snapshot', + 'swipe', 'type', 'wait', ], @@ -348,14 +355,12 @@ test('web supports only the initial browser interaction slice', () => { 'app-switcher', 'back', 'clipboard', - 'gesture', 'home', 'keyboard', 'longpress', 'perf', 'orientation', 'settings', - 'swipe', 'trigger-app-event', ], [{ device: webDevice, expected: false, label: 'on web' }], diff --git a/src/core/__tests__/capability-plugin-routing-parity.test.ts b/src/core/__tests__/capability-plugin-routing-parity.test.ts index 34a8e574d..be8151b40 100644 --- a/src/core/__tests__/capability-plugin-routing-parity.test.ts +++ b/src/core/__tests__/capability-plugin-routing-parity.test.ts @@ -265,15 +265,12 @@ test('HarmonyOS static capabilities omit runtime-backed command admissions', () 'back', 'click', 'fill', - 'gesture', 'home', 'keyboard', 'longpress', 'perf', 'press', - 'scroll', 'settings', - 'swipe', ]); }); diff --git a/src/core/__tests__/dispatch-interactions.test.ts b/src/core/__tests__/dispatch-interactions.test.ts index 82fb14444..fa82330c1 100644 --- a/src/core/__tests__/dispatch-interactions.test.ts +++ b/src/core/__tests__/dispatch-interactions.test.ts @@ -11,7 +11,11 @@ vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOrigi return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; }); -import { handleFillCommand, handlePressCommand } from '../dispatch-interactions.ts'; +import { + handleFillCommand, + handleLongPressCommand, + handlePressCommand, +} from '../dispatch-interactions.ts'; import type { Interactor } from '@agent-device/contracts/interaction'; import type { RunnerCommand } from '../../platforms/apple/core/runner/runner-contract.ts'; import { AppError } from '@agent-device/kernel/errors'; @@ -399,3 +403,17 @@ test('handlePressCommand on Android keeps the direct path even with hold', async assert.equal(longPresses.length, 3); assert.equal(result.pressed, true); }); + +test('dispatch longpress explains direct platform coordinate requirement', async () => { + const interactor = {} as unknown as Interactor; + + await assert.rejects( + () => handleLongPressCommand(interactor, ['@e40', '900']), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /longpress requires x y/i.test(error.message) && + /open daemon session/i.test(String(error.details?.hint)) && + /snapshot -i/i.test(String(error.details?.hint)), + ); +}); diff --git a/src/core/__tests__/dispatch-scroll.test.ts b/src/core/__tests__/dispatch-scroll.test.ts deleted file mode 100644 index 8d78cdcb4..000000000 --- a/src/core/__tests__/dispatch-scroll.test.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { test } from 'vitest'; -import assert from 'node:assert/strict'; -import { dispatchCommand } from '../dispatch.ts'; -import { handleLongPressCommand, handleScrollCommand } from '../dispatch-interactions.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import type { Interactor } from '@agent-device/contracts/interaction'; -import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; - -test('dispatch scroll rejects mixing amount and --pixels', async () => { - await assert.rejects( - () => dispatchCommand(IOS_SIMULATOR, 'scroll', ['down', '0.4'], undefined, { pixels: 240 }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /either a relative amount or --pixels/i.test(error.message), - ); -}); - -test('dispatch scroll forwards pixels and duration without reporting ignored duration', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { ok: true }; - }, - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['down'], { - pixels: 200, - durationMs: 50, - }); - - assert.deepEqual(calls, [ - { - direction: 'down', - options: { - amount: undefined, - pixels: 200, - durationMs: 50, - releaseBehavior: 'controlled', - }, - }, - ]); - assert.equal(result.pixels, 200); - assert.equal(result.durationMs, undefined); -}); - -test('dispatch scroll reports duration when the interactor honored it', async () => { - const interactor = { - scroll: async () => ({ pixels: 200, durationMs: 50 }), - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['down'], { - pixels: 200, - durationMs: 50, - }); - - assert.equal(result.pixels, 200); - assert.equal(result.durationMs, 50); -}); - -test('dispatch scroll rejects duration above the shared cap', async () => { - const interactor = { - scroll: async () => { - throw new Error('scroll should be rejected before backend call'); - }, - } as unknown as Interactor; - - await assert.rejects( - () => handleScrollCommand(interactor, ['down'], { pixels: 200, durationMs: 10_001 }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /durationMs.*at most 10000/i.test(error.message), - ); -}); - -test('dispatch scroll bottom rejects blind scrolling without snapshot support', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - } as unknown as Interactor; - - await assert.rejects( - () => handleScrollCommand(interactor, ['bottom'], undefined), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - /requires snapshot support/i.test(error.message), - ); - - assert.equal(calls.length, 0); -}); - -test('dispatch longpress explains direct platform coordinate requirement', async () => { - const interactor = {} as unknown as Interactor; - - await assert.rejects( - () => handleLongPressCommand(interactor, ['@e40', '900']), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /longpress requires x y/i.test(error.message) && - /open daemon session/i.test(String(error.details?.hint)) && - /snapshot -i/i.test(String(error.details?.hint)), - ); -}); - -test('dispatch scroll bottom does not scroll when no hidden content is below', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - snapshot: async () => makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['bottom'], undefined); - - assert.equal(calls.length, 0); - assert.equal(result.direction, 'down'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 0); - assert.match(String(result.message), /Already at bottom/); -}); - -test('dispatch scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const snapshotScopes: unknown[] = []; - const snapshots = [ - makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - ]; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - snapshot: async (options: any) => { - snapshotScopes.push(options.scope); - return snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)]; - }, - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['bottom'], undefined); - - assert.equal(calls.length, 1); - assert.deepEqual(calls[0], { - direction: 'down', - options: { - amount: undefined, - pixels: undefined, - durationMs: undefined, - releaseBehavior: 'inertial', - }, - }); - assert.equal(result.passes, 1); - assert.equal(result.lastPass, 1); - assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); -}); - -test('dispatch scroll bottom tolerates unchanged signatures while hidden content advances', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const snapshots = [ - makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - makeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), - ]; - let snapshotIndex = 0; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - snapshot: async () => snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['bottom'], undefined); - - assert.equal(calls.length, 2); - assert.equal(result.passes, 2); -}); - -test('dispatch scroll bottom keeps scoped snapshot failures scoped', async () => { - let snapshotCount = 0; - const interactor = { - scroll: async () => ({}), - snapshot: async (options: any) => { - snapshotCount += 1; - if (options.scope) throw new Error('scoped snapshot failed'); - return makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }); - }, - } as unknown as Interactor; - - await assert.rejects( - () => handleScrollCommand(interactor, ['bottom'], undefined), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /scoped container/i.test(error.message) && - error.details?.scope === 'Messages', - ); - assert.equal(snapshotCount, 2); -}); - -function makeScrollSnapshot(options: { hiddenBelow: boolean; message: string }) { - return { - backend: 'xctest' as const, - nodes: [ - { - index: 1, - type: 'ScrollView', - label: 'Messages', - hiddenContentBelow: options.hiddenBelow ? true : undefined, - rect: { x: 0, y: 100, width: 400, height: 600 }, - }, - { - index: 2, - parentIndex: 1, - type: 'Button', - label: options.message, - rect: { x: 0, y: 640, width: 400, height: 56 }, - }, - ], - truncated: false, - }; -} diff --git a/src/core/__tests__/gesture-capabilities.test.ts b/src/core/__tests__/gesture-capabilities.test.ts deleted file mode 100644 index 28e37b767..000000000 --- a/src/core/__tests__/gesture-capabilities.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import type { - GestureCommandInput, - GestureSemanticInput, -} from '@agent-device/contracts/interaction'; -import { - normalizePublicGesture, - normalizePublicSwipeMotion, -} from '@agent-device/contracts/interaction'; -import { requireGestureSupported } from '../capabilities.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import type { DeviceInfo } from '@agent-device/kernel/device'; - -const oneFingerPan: GestureSemanticInput = { - intent: 'pan', - origin: { x: 100, y: 200 }, - delta: { x: 40, y: -20 }, -}; -const twoFingerPan: GestureSemanticInput = { ...oneFingerPan, pointerCount: 2 }; -const pinch: GestureSemanticInput = { intent: 'pinch', scale: 1.2 }; -const fling: GestureSemanticInput = { - intent: 'fling', - direction: 'left', - origin: { x: 100, y: 200 }, -}; -const drag: GestureCommandInput = { - intent: 'drag', - source: 'id="source"', - destination: 'id="destination"', -}; - -const device = (fields: Partial): DeviceInfo => ({ - platform: 'android', - id: 'test-device', - name: 'Test device', - kind: 'emulator', - ...fields, -}); - -function assertSupported(input: GestureCommandInput, target: DeviceInfo): void { - assert.doesNotThrow(() => requireGestureSupported(input, target)); -} - -function assertUnsupported(input: GestureCommandInput, target: DeviceInfo, expected: RegExp): void { - assert.throws( - () => requireGestureSupported(input, target), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - expected.test(error.message), - ); -} - -test('Android phones and emulators support single- and multi-touch gesture plans', () => { - for (const kind of ['device', 'emulator'] as const) { - const target = device({ kind }); - assertSupported(oneFingerPan, target); - assertSupported(twoFingerPan, target); - assertSupported(pinch, target); - } -}); - -test('target-authored drag is admitted only where adapters preserve every authored phase', () => { - for (const kind of ['device', 'emulator'] as const) { - assertSupported(drag, device({ kind, target: 'mobile' })); - } - for (const appleOs of ['ios', 'ipados'] as const) { - for (const kind of ['device', 'simulator'] as const) { - assertSupported(drag, device({ platform: 'apple', appleOs, kind, target: 'mobile' })); - } - } - assertSupported(drag, device({ platform: 'apple', kind: 'simulator', target: 'mobile' })); - - const inexactBackends = [ - device({ target: 'tv' }), - device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }), - device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }), - device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }), - device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }), - device({ platform: 'linux', kind: 'device', target: 'desktop' }), - device({ platform: 'vega', kind: 'device', target: 'tv' }), - device({ platform: 'web', kind: 'device', target: 'desktop' }), - ]; - for (const target of inexactBackends) { - assert.throws( - () => requireGestureSupported(drag, target), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - error.details?.gesture === 'drag' && - /source hold, timed movement, and destination hold/.test(String(error.details?.hint)), - ); - } -}); - -test('iOS and iPadOS simulators support multi-touch while physical devices do not', () => { - for (const appleOs of ['ios', 'ipados'] as const) { - const simulator = device({ platform: 'apple', appleOs, kind: 'simulator' }); - const physical = device({ platform: 'apple', appleOs, kind: 'device' }); - assertSupported(oneFingerPan, simulator); - assertSupported(twoFingerPan, simulator); - assertSupported(pinch, simulator); - assertSupported(oneFingerPan, physical); - assertUnsupported(twoFingerPan, physical, /physical iOS devices/); - assert.throws( - () => requireGestureSupported(pinch, physical), - (error: unknown) => - error instanceof AppError && /iOS-simulator only/.test(String(error.details?.hint)), - ); - } -}); - -test('TV, spatial, watch, desktop, Linux, and web gesture policy stays explicit', () => { - const androidTv = device({ target: 'tv' }); - const tvOs = device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }); - const visionOs = device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }); - const watchOs = device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }); - const macOs = device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }); - const linux = device({ platform: 'linux', kind: 'device', target: 'desktop' }); - const web = device({ platform: 'web', kind: 'device', target: 'desktop' }); - - assertSupported(oneFingerPan, androidTv); - assertUnsupported(twoFingerPan, androidTv, /Android TV/); - assert.throws( - () => requireGestureSupported(twoFingerPan, androidTv), - (error: unknown) => - error instanceof AppError && - /Android TV has no touch input/.test(String(error.details?.hint)), - ); - assertUnsupported(twoFingerPan, tvOs, /tvOS/); - assertUnsupported(twoFingerPan, visionOs, /visionOS/i); - assertUnsupported(oneFingerPan, watchOs, /watchos/); - assertSupported(oneFingerPan, macOs); - assertUnsupported(twoFingerPan, macOs, /macOS/); - assertSupported(oneFingerPan, linux); - assertSupported( - normalizePublicSwipeMotion({ from: { x: 10, y: 20 }, to: { x: 110, y: 20 } }).gesture, - linux, - ); - assertSupported(normalizePublicGesture({ kind: 'swipe', preset: 'left' }).gesture, linux); - assertUnsupported(fling, linux, /Linux/); - assertUnsupported(twoFingerPan, linux, /linux/i); - assertUnsupported(oneFingerPan, web, /web/); -}); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 9b801bd0c..f6f3f3332 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -3,12 +3,6 @@ import { commandDescriptors } from './command-descriptor/registry.ts'; import { tryGetPlugin } from './platform-plugin-registry.ts'; import { registerBuiltinPlatformPlugins } from './interactors/register-builtins.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { AppError } from '@agent-device/kernel/errors'; -import type { - GestureCommandInput, - GestureSemanticInput, -} from '@agent-device/contracts/interaction'; -import { assertAppleMultiTouchSupported } from '@agent-device/contracts/platform'; // Populate the PlatformPlugin registry once at module load (idempotent; registers // only lazy closures, so no leaf code is imported and CLI cold-start is unaffected @@ -43,16 +37,13 @@ const HARMONYOS_SUPPORTED_COMMANDS = new Set([ 'click', 'fill', 'home', - 'gesture', 'keyboard', 'longpress', 'press', - 'scroll', 'settings', - 'swipe', ]); const WEB_QUERY_COMMANDS = ['audio'] as const; -const WEB_INTERACTION_COMMANDS = ['click', 'fill', 'hover', 'press', 'scroll'] as const; +const WEB_INTERACTION_COMMANDS = ['click', 'fill', 'hover', 'press'] as const; const WEB_SUPPORTED_COMMANDS = new Set([ ...WEB_QUERY_COMMANDS, ...WEB_INTERACTION_COMMANDS, @@ -171,77 +162,3 @@ export function supportedPlatformsForCommand(command: string): string[] { } return supported; } - -export function requireGestureSupported(input: GestureCommandInput, device: DeviceInfo): void { - if (input.intent === 'drag') { - requireTargetAuthoredDragSupported(input, device); - return; - } - if (device.platform === 'web' || device.appleOs === 'watchos') { - throw unsupportedGesture(input, gesturePlatformMessage(input, device)); - } - if (isMultiTouchGesture(input)) { - requireMultiTouchGestureSupported(input, device); - return; - } - if (device.appleOs === 'visionos') { - throw unsupportedGesture(input, gesturePlatformMessage(input, device)); - } - // Linux can preserve public coordinate/preset swipe through its drag primitive, but cannot - // honor the speed semantics authored by `gesture fling`. - if (input.intent === 'fling' && 'direction' in input && device.platform === 'linux') { - throw unsupportedGesture(input, 'gesture fling is not supported on Linux'); - } -} - -function requireTargetAuthoredDragSupported( - input: Extract, - device: DeviceInfo, -): void { - if (supportsTargetAuthoredDrag(device)) return; - throw unsupportedGesture( - input, - gesturePlatformMessage(input, device), - 'Target-authored drag requires an adapter that preserves source hold, timed movement, and destination hold; it is supported on Android touch devices and iOS/iPadOS.', - ); -} - -function supportsTargetAuthoredDrag(device: DeviceInfo): boolean { - if (device.platform === 'android') { - return device.target !== 'tv'; - } - if (device.platform !== 'apple') return false; - if (device.appleOs === undefined) return device.target !== 'desktop' && device.target !== 'tv'; - return device.appleOs === 'ios' || device.appleOs === 'ipados'; -} - -function isMultiTouchGesture(input: GestureSemanticInput): boolean { - if (input.intent === 'pan') return ('pointerCount' in input ? input.pointerCount : 1) === 2; - return input.intent === 'pinch' || input.intent === 'rotate' || input.intent === 'transform'; -} - -function requireMultiTouchGestureSupported(input: GestureSemanticInput, device: DeviceInfo): void { - if (device.platform === 'android') { - if (device.target !== 'tv') return; - throw unsupportedGesture( - input, - `gesture ${input.intent} is not supported on Android TV`, - 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.', - ); - } - if (device.platform !== 'apple') { - throw unsupportedGesture(input, gesturePlatformMessage(input, device)); - } - assertAppleMultiTouchSupported(device, input.intent); -} - -function gesturePlatformMessage(input: GestureCommandInput, device: DeviceInfo): string { - return `gesture ${input.intent} is not supported on ${device.appleOs ?? device.platform}`; -} - -function unsupportedGesture(input: GestureCommandInput, message: string, hint?: string): AppError { - return new AppError('UNSUPPORTED_OPERATION', message, { - gesture: input.intent, - ...(hint ? { hint } : {}), - }); -} diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index 2245f9a1a..a5d0d0b39 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -59,6 +59,7 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.events, PUBLIC_COMMANDS.find, PUBLIC_COMMANDS.focus, + PUBLIC_COMMANDS.gesture, PUBLIC_COMMANDS.get, PUBLIC_COMMANDS.install, PUBLIC_COMMANDS.installFromSource, @@ -71,9 +72,11 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.record, PUBLIC_COMMANDS.reinstall, PUBLIC_COMMANDS.replay, + PUBLIC_COMMANDS.scroll, PUBLIC_COMMANDS.shutdown, PUBLIC_COMMANDS.screenshot, PUBLIC_COMMANDS.snapshot, + PUBLIC_COMMANDS.swipe, PUBLIC_COMMANDS.test, PUBLIC_COMMANDS.trace, PUBLIC_COMMANDS.type, @@ -215,6 +218,9 @@ test('generic route commands that reach platform dispatch declare the dispatch f PUBLIC_COMMANDS.gesture, PUBLIC_COMMANDS.focus, PUBLIC_COMMANDS.screenshot, + // R43 retired scroll's dispatch leaf with its capability bucket: the bound + // `scrollDirection` operation is its only execution. + PUBLIC_COMMANDS.scroll, PUBLIC_COMMANDS.viewport, ]); @@ -331,7 +337,12 @@ test('capability-checked command list is built from descriptor capabilities', () false, 'snapshot admission comes from exact device-runtime facts', ); - assert.ok(expectedNames.has(PUBLIC_COMMANDS.gesture), 'gesture remains capability-checked'); + assert.equal( + expectedNames.has(PUBLIC_COMMANDS.gesture), + false, + 'gesture admission comes from exact device-runtime facts', + ); + assert.ok(expectedNames.has(PUBLIC_COMMANDS.click), 'click remains capability-checked'); assert.equal( expectedNames.has(PUBLIC_COMMANDS.capabilities), false, diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 091a4183e..338cb666e 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -1,4 +1,3 @@ -import type { CommandCapability } from '../capabilities.ts'; // The typed-flags request from contracts/, not the daemon's server-side refinement: these // descriptors read `command`, `positionals` and `flags` and never touch `internal`. import type { DispatchedCommand } from '@agent-device/contracts/command'; @@ -37,6 +36,8 @@ import { shutdownTargetUse, findRuntimePlanUses, focusRuntimeUse, + gestureRuntimePlanUses, + scrollRuntimePlanUses, typeTextRuntimeUse, viewportRuntimeUse, } from '@agent-device/contracts/platform'; @@ -228,11 +229,6 @@ const VEGA_VVD = { emulator: true }; const LINUX_DEVICE = { device: true }; const LINUX_NONE = {}; -const ALL_DEVICE_COMMAND_CAPABILITY = { - apple: APPLE_SIM_AND_DEVICE, - android: ANDROID_ALL, - linux: LINUX_DEVICE, -} satisfies CommandCapability; // --------------------------------------------------------------------------- // ADR 0019 §6 platform-execution modes. Every descriptor declares one; there is // no registry-entry default (see `readDeclaredPlatformExecution`). @@ -1251,10 +1247,12 @@ export const RAW_COMMAND_DESCRIPTORS = [ refFrameEffect: 'may-invalidate', androidBlockingDialogGuard: true, }, - capability: ALL_DEVICE_COMMAND_CAPABILITY, + // R42 retires this command's capability bucket: admission is the owner's gesture-tier facts, + // which the retired `requireGestureSupported` used to decide inside the daemon. The declared + // uses are the four tiers one gesture input can select between (ADR 0019 §9). timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: gestureRuntimePlanUses }, }, { name: 'home', @@ -1317,13 +1315,24 @@ export const RAW_COMMAND_DESCRIPTORS = [ }, { name: 'scroll', - ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), + ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/scroll-runtime.ts'] as const } : {}), catalog: { group: 'public' }, frameworkTier: 'core', - ...GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS, + // R43 retires this command's capability bucket and its `dispatch` leaf together: admission is + // the owner's `scrollDirection` fact, and the only execution is the bound operation. The + // remaining traits are `GENERIC_MUTATING_LINUX_DEVICE_COMMAND_TRAITS` minus those two. + recordsSessionAction: true, + recordingEffect: 'mutates-app', + deviceClaimPolicy: 'require-owner', + daemon: { + route: 'generic', + refFrameEffect: 'may-invalidate', + androidBlockingDialogGuard: true, + }, + batchable: true, timeoutPolicy: postActionObservationTimeoutPolicy('scroll', DEFAULT_TIMEOUT_POLICY), postActionObservation: postActionObservation('scroll'), - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: scrollRuntimePlanUses }, }, { name: 'swipe', @@ -1338,10 +1347,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ refFrameEffect: 'may-invalidate', androidBlockingDialogGuard: true, }, - capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, + // R44 retires this command's capability bucket. A swipe always normalizes to a coordinate + // fling, so it selects the same tier set `gesture` does and declares the same uses. timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: gestureRuntimePlanUses }, }, { name: 'focus', diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 6d790e87f..ea6933256 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -1,18 +1,10 @@ import { - assertExclusiveScrollDistanceInputs, getClickButtonValidationError, - honoredScrollDurationMs, MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE, - normalizeScrollDurationMs, - parseScrollDirection, - resolveScrollExecutionOptions, resolveClickButton, type ClickButton, type Interactor, type RunnerCallOptions, - type ScrollCommandOptions, - type ScrollDirection, - type ResolvedScrollExecutionOptions, } from '@agent-device/contracts/interaction'; import { readFillBackendResult } from './fill-backend-result.ts'; import { @@ -24,13 +16,6 @@ import { } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { RunnerSequenceStep } from '../platforms/apple/core/runner/runner-contract.ts'; -import { - captureScrollEdgeState, - formatScrollEdgeMessage, - runScrollEdgePasses, - type ScrollEdge, - type ScrollEdgeState, -} from '../utils/scroll-edge-state.ts'; import { successText, withSuccessText } from '../utils/success-text.ts'; import { readPointPositionals } from '../utils/validation.ts'; import type { DispatchContext } from './dispatch-context.ts'; @@ -42,11 +27,6 @@ import { shouldUseIosPressSequence, } from './dispatch-series.ts'; -type ScrollTarget = { - direction: ScrollDirection; - edge?: ScrollEdge; -}; - export async function handleLongPressCommand( interactor: Interactor, positionals: string[], @@ -494,128 +474,6 @@ function runnerOptionsFromContext(context: DispatchContext | undefined): RunnerC }; } -export async function handleScrollCommand( - interactor: Interactor, - positionals: string[], - context: DispatchContext | undefined, -): Promise> { - const directionInput = positionals[0]; - const amount = positionals[1] ? Number(positionals[1]) : undefined; - const pixels = context?.pixels; - const durationMs = context?.durationMs; - if (!directionInput) throw new AppError('INVALID_ARGS', 'scroll requires direction'); - assertScrollCommandInputs(amount, pixels, durationMs); - - const target = parseScrollTarget(directionInput); - const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); - const { interactionResult, completedPasses } = await runDispatchedScroll( - interactor, - context, - target, - options, - ); - - const result = buildDispatchedScrollResult(target, options, completedPasses, interactionResult); - return withSuccessText( - result, - formatScrollEdgeMessage(target.direction, target.edge, completedPasses, amount, pixels), - ); -} - -function assertScrollCommandInputs( - amount: number | undefined, - pixels: number | undefined, - durationMs: number | undefined, -): void { - assertScrollAmountInput(amount); - normalizeScrollDurationMs(durationMs); - assertExclusiveScrollDistanceInputs({ amount, pixels }); -} - -function assertScrollAmountInput(amount: number | undefined): void { - if (amount !== undefined && !Number.isFinite(amount)) { - throw new AppError('INVALID_ARGS', 'scroll amount must be a number'); - } -} - -async function runDispatchedScroll( - interactor: Interactor, - context: DispatchContext | undefined, - target: ScrollTarget, - options: ResolvedScrollExecutionOptions, -): Promise<{ interactionResult: Record; completedPasses: number }> { - if (target.edge) { - const edge = target.edge; - const edgeResult = await runScrollEdgePasses({ - edge, - captureState: async (scope) => - await captureVerifiedScrollEdgeState(interactor, context, edge, scope), - scroll: async () => await interactor.scroll(target.direction, options), - }); - return { - interactionResult: edgeResult.result ?? {}, - completedPasses: edgeResult.passes, - }; - } - - return { - interactionResult: (await interactor.scroll(target.direction, options)) ?? {}, - completedPasses: 1, - }; -} - -function buildDispatchedScrollResult( - target: ScrollTarget, - options: ScrollCommandOptions, - completedPasses: number, - interactionResult: Record, -): Record { - const durationMs = honoredScrollDurationMs(interactionResult); - return { - direction: target.direction, - ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), - ...(options.amount !== undefined ? { amount: options.amount } : {}), - ...(options.pixels !== undefined ? { pixels: options.pixels } : {}), - ...(durationMs !== undefined ? { durationMs } : {}), - ...interactionResult, - }; -} - -async function captureVerifiedScrollEdgeState( - interactor: Interactor, - context: DispatchContext | undefined, - edge: ScrollEdge, - scope?: string, -): Promise { - if (typeof interactor.snapshot !== 'function') { - throw new AppError( - 'UNSUPPORTED_OPERATION', - `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, - ); - } - const snapshot = interactor.snapshot; - return await captureScrollEdgeState({ - edge, - scope, - captureNodes: async (snapshotScope) => - ( - await snapshot({ - appBundleId: context?.appBundleId, - scope: snapshotScope, - }) - ).nodes ?? [], - }); -} - -function parseScrollTarget(input: string): { - direction: ReturnType; - edge?: 'top' | 'bottom'; -} { - if (input === 'bottom') return { direction: 'down', edge: 'bottom' }; - if (input === 'top') return { direction: 'up', edge: 'top' }; - return { direction: parseScrollDirection(input) }; -} - function formatPressMessage(params: { x: number; y: number; button?: ClickButton }): string { if (params.button && params.button !== 'primary') { return `Clicked ${params.button} (${params.x}, ${params.y})`; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index b71fbb31a..02589f702 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -1,5 +1,5 @@ import { parseDeviceRotation } from '@agent-device/contracts/device'; -import type { GesturePlan, Interactor, RunnerContext } from '@agent-device/contracts/interaction'; +import type { Interactor, RunnerContext } from '@agent-device/contracts/interaction'; import { parseTvRemoteButton } from '@agent-device/contracts/interaction'; import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -17,7 +17,6 @@ import { handleHoverCommand, handleLongPressCommand, handlePressCommand, - handleScrollCommand, } from './dispatch-interactions.ts'; import { getInteractor } from './interactors.ts'; @@ -82,18 +81,6 @@ async function dispatchWithInteractor( ); } -export async function dispatchGesturePlan( - device: DeviceInfo, - plan: GesturePlan, - context?: DispatchContext, -): Promise | void> { - const interactor = await getInteractor(device, runnerContextFromDispatchContext(context)); - if (!interactor.performGesture) { - throw new AppError('UNSUPPORTED_OPERATION', 'Gesture execution is unavailable'); - } - return await interactor.performGesture(plan); -} - export async function dispatchGestureViewport( device: DeviceInfo, context?: DispatchContext, @@ -145,8 +132,6 @@ const DISPATCH_HANDLERS: Record = { hover: ({ interactor, positionals }) => handleHoverCommand(interactor, positionals), fill: ({ interactor, positionals, context }) => handleFillCommand(interactor, positionals, context), - scroll: ({ interactor, positionals, context }) => - handleScrollCommand(interactor, positionals, context), 'trigger-app-event': ({ device, interactor, positionals, context }) => handleTriggerAppEventCommand(device, interactor, positionals, context), back: async ({ interactor, context }) => { diff --git a/src/daemon/__tests__/gesture-admission-parity.test.ts b/src/daemon/__tests__/gesture-admission-parity.test.ts new file mode 100644 index 000000000..e00ee2f11 --- /dev/null +++ b/src/daemon/__tests__/gesture-admission-parity.test.ts @@ -0,0 +1,178 @@ +import { expect, test } from 'vitest'; +import type { + GestureCommandInput, + GestureSemanticInput, +} from '@agent-device/contracts/interaction'; +import { + normalizePublicGesture, + normalizePublicSwipeMotion, +} from '@agent-device/contracts/interaction'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createPlatformRuntimeGateway } from '../../platform-runtime.ts'; +import { createRequestRuntimeBindings } from '../request-runtime-binding.ts'; +import { resolveBoundGestureRuntime } from '../gesture-runtime.ts'; + +/** + * The parity artifact for R42/R44 (ADR 0019 §6). + * + * This is the retired `requireGestureSupported` suite's device matrix, re-pointed at what + * replaced it: the REAL composed runtime gateway's owner facts, admitted through the real daemon + * gesture admission. Every assertion below — admitted or refused, message and hint — is the + * behavior `main` produced before the cutover, so a fact cell that drifts from the admission it + * restates fails here rather than on a device. + */ +const gateway = createPlatformRuntimeGateway({ + resolveSessionArtifacts: () => ({ + outputPath: '/sessions/parity/app.log', + pidPath: '/sessions/parity/app-log.pid', + }), + sessionsDir: '/sessions', +}); + +const oneFingerPan: GestureSemanticInput = { + intent: 'pan', + origin: { x: 100, y: 200 }, + delta: { x: 40, y: -20 }, +}; +const twoFingerPan: GestureSemanticInput = { ...oneFingerPan, pointerCount: 2 }; +const pinch: GestureSemanticInput = { intent: 'pinch', scale: 1.2 }; +const fling: GestureSemanticInput = { + intent: 'fling', + direction: 'left', + origin: { x: 100, y: 200 }, +}; +const drag: GestureCommandInput = { + intent: 'drag', + source: 'id="source"', + destination: 'id="destination"', +}; + +const device = (fields: Partial): DeviceInfo => ({ + platform: 'android', + id: 'test-device', + name: 'Test device', + kind: 'emulator', + ...fields, +}); + +/** The production binding seam, so admission runs against the real inspect-then-bind path. */ +async function admit(input: GestureCommandInput, target: DeviceInfo) { + const bindings = createRequestRuntimeBindings({ + gateway, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + admitDeviceClaim: async () => {}, + }); + try { + return await resolveBoundGestureRuntime({ + device: target, + input, + inspectFacts: bindings.inspectFacts, + bindDevice: bindings.bindDevice, + }); + } finally { + await bindings[Symbol.asyncDispose](); + } +} + +async function expectAdmitted(input: GestureCommandInput, target: DeviceInfo): Promise { + const resolved = await admit(input, target); + expect(resolved.ok, `${input.intent} should be admitted on ${target.platform}`).toBe(true); +} + +async function expectRefused( + input: GestureCommandInput, + target: DeviceInfo, + message: RegExp, + hint?: RegExp, +): Promise { + const resolved = await admit(input, target); + expect(resolved.ok).toBe(false); + if (resolved.ok) return; + expect(resolved.response.error.code).toBe('UNSUPPORTED_OPERATION'); + expect(resolved.response.error.message).toMatch(message); + if (hint) expect(String(resolved.response.error.hint)).toMatch(hint); +} + +test('Android phones and emulators admit single- and multi-touch gesture plans', async () => { + for (const kind of ['device', 'emulator'] as const) { + const target = device({ kind }); + await expectAdmitted(oneFingerPan, target); + await expectAdmitted(twoFingerPan, target); + await expectAdmitted(pinch, target); + } +}); + +test('target-authored drag is admitted only where adapters preserve every authored phase', async () => { + for (const kind of ['device', 'emulator'] as const) { + await expectAdmitted(drag, device({ kind, target: 'mobile' })); + } + for (const appleOs of ['ios', 'ipados'] as const) { + for (const kind of ['device', 'simulator'] as const) { + await expectAdmitted(drag, device({ platform: 'apple', appleOs, kind, target: 'mobile' })); + } + } + await expectAdmitted(drag, device({ platform: 'apple', kind: 'simulator', target: 'mobile' })); + + const inexactBackends = [ + device({ target: 'tv' }), + device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }), + device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }), + device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }), + device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }), + device({ platform: 'linux', kind: 'device', target: 'desktop' }), + device({ platform: 'vega', kind: 'device', target: 'tv' }), + device({ platform: 'web', kind: 'device', target: 'desktop' }), + ]; + for (const target of inexactBackends) { + await expectRefused( + drag, + target, + /^gesture drag is not supported on /, + /source hold, timed movement, and destination hold/, + ); + } +}); + +test('iOS and iPadOS simulators admit multi-touch while physical devices do not', async () => { + for (const appleOs of ['ios', 'ipados'] as const) { + const simulator = device({ platform: 'apple', appleOs, kind: 'simulator' }); + const physical = device({ platform: 'apple', appleOs, kind: 'device' }); + await expectAdmitted(oneFingerPan, simulator); + await expectAdmitted(twoFingerPan, simulator); + await expectAdmitted(pinch, simulator); + await expectAdmitted(oneFingerPan, physical); + await expectRefused(twoFingerPan, physical, /physical iOS devices/); + await expectRefused(pinch, physical, /physical iOS devices/, /iOS-simulator only/); + } +}); + +test('TV, spatial, watch, desktop, Linux, and web gesture policy stays explicit', async () => { + const androidTv = device({ target: 'tv' }); + const tvOs = device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }); + const visionOs = device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }); + const watchOs = device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }); + const macOs = device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }); + const linux = device({ platform: 'linux', kind: 'device', target: 'desktop' }); + const web = device({ platform: 'web', kind: 'device', target: 'desktop' }); + + await expectAdmitted(oneFingerPan, androidTv); + await expectRefused(twoFingerPan, androidTv, /Android TV/, /Android TV has no touch input/); + await expectRefused(twoFingerPan, tvOs, /tvOS/); + await expectRefused(twoFingerPan, visionOs, /visionOS/); + await expectRefused(oneFingerPan, watchOs, /watchos/); + await expectAdmitted(oneFingerPan, macOs); + await expectRefused(twoFingerPan, macOs, /macOS/); + await expectAdmitted(oneFingerPan, linux); + await expectAdmitted( + normalizePublicSwipeMotion({ from: { x: 10, y: 20 }, to: { x: 110, y: 20 } }).gesture, + linux, + ); + await expectAdmitted(normalizePublicGesture({ kind: 'swipe', preset: 'left' }).gesture, linux); + await expectRefused(fling, linux, /gesture fling is not supported on Linux/); + await expectRefused(twoFingerPan, linux, /linux/); + await expectRefused(oneFingerPan, web, /web/); +}); diff --git a/src/daemon/__tests__/request-handler-chain.test.ts b/src/daemon/__tests__/request-handler-chain.test.ts index 85c23a7e5..94775cd81 100644 --- a/src/daemon/__tests__/request-handler-chain.test.ts +++ b/src/daemon/__tests__/request-handler-chain.test.ts @@ -14,6 +14,8 @@ import { } from '../../__tests__/test-utils/index.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { dispatchSwipeViaRuntime } from '../handlers/interaction-gesture.ts'; +import { createPlatformRuntimeGateway } from '../../platform-runtime.ts'; +import { createRequestRuntimeBindings } from '../request-runtime-binding.ts'; import { createLocalLinuxToolProvider, withLinuxToolProvider, @@ -177,10 +179,31 @@ test('duration-less public coordinate swipe retains Linux drag behavior', async }, }); + // R44: swipe binds its gesture tier before executing, so this drives the REAL Linux owner + // through the composed gateway. Linux advertises no `gestureViewport`, so the coordinate frame + // still comes from the capture below — the preferred-operation fallback, unchanged. + const gateway = createPlatformRuntimeGateway({ + resolveSessionArtifacts: () => ({ + outputPath: '/sessions/linux-swipe/app.log', + pidPath: '/sessions/linux-swipe/app-log.pid', + }), + sessionsDir: '/sessions', + }); + const bindings = createRequestRuntimeBindings({ + gateway, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + admitDeviceClaim: async () => {}, + }); const response = await withLinuxToolProvider( provider, async () => await dispatchSwipeViaRuntime({ + inspectFacts: bindings.inspectFacts, + bindDevice: bindings.bindDevice, req: { ...makeRequest('swipe'), session: 'linux-swipe', diff --git a/src/daemon/__tests__/request-router-android-modal.test.ts b/src/daemon/__tests__/request-router-android-modal.test.ts index dfdc59970..731a180e1 100644 --- a/src/daemon/__tests__/request-router-android-modal.test.ts +++ b/src/daemon/__tests__/request-router-android-modal.test.ts @@ -20,7 +20,11 @@ vi.mock('../../core/dispatch.ts', async (importOriginal) => { }; }); -import { createRequestHandler } from './test-device-runtime-gateway.ts'; +import { + createRequestHandler, + gestureDeviceRuntimeGateway, + gestureRuntimeSpies, +} from './test-device-runtime-gateway.ts'; import type { SessionState } from '../types.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; @@ -117,6 +121,7 @@ test('generic Android gesture commands dismiss blocking system dialogs during re dispatchResult = {}; execCalls.length = 0; dispatchCalls.length = 0; + gestureRuntimeSpies.scrollDirection.mockClear(); const sessionStore = makeSessionStore('agent-device-router-android-modal-'); sessionStore.set('default', makeAndroidSession('default')); @@ -130,6 +135,7 @@ test('generic Android gesture commands dismiss blocking system dialogs during re leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -141,7 +147,13 @@ test('generic Android gesture commands dismiss blocking system dialogs during re }); expect(response.ok).toBe(true); - expect(dispatchCalls).toEqual([['scroll', 'down', '0.55']]); + // R43: `scroll` reaches the device through its bound operation, so the dispatcher sees nothing. + expect(dispatchCalls).toEqual([]); + expect(gestureRuntimeSpies.scrollDirection).toHaveBeenCalledTimes(1); + expect(gestureRuntimeSpies.scrollDirection.mock.calls[0]?.[0]).toMatchObject({ + direction: 'down', + options: { amount: 0.55 }, + }); expect(execCalls).toEqual([['-s', 'emulator-5554', 'shell', 'input', 'tap', '210', '640']]); expect(openAndroidApp).toHaveBeenCalledWith( expect.objectContaining({ id: 'emulator-5554' }), @@ -153,9 +165,14 @@ test('generic Android gesture commands dismiss blocking system dialogs during re test('generic Android gesture commands continue when recording dialog inspection fails', async () => { snapshotCalls = 0; snapshotMode = 'throws'; - dispatchResult = { warning: 'The platform response already carried a warning.' }; execCalls.length = 0; dispatchCalls.length = 0; + gestureRuntimeSpies.scrollDirection.mockClear(); + // The owner's own result is what the readiness warning has to merge with, so the bound + // operation carries it now that the dispatcher no longer executes this command. + gestureRuntimeSpies.scrollDirection.mockResolvedValueOnce({ + warning: 'The platform response already carried a warning.', + }); const sessionStore = makeSessionStore('agent-device-router-android-modal-'); sessionStore.set('default', makeAndroidSession('default')); @@ -170,6 +187,7 @@ test('generic Android gesture commands continue when recording dialog inspection leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -181,7 +199,13 @@ test('generic Android gesture commands continue when recording dialog inspection }); expect(response.ok).toBe(true); - expect(dispatchCalls).toEqual([['scroll', 'down', '0.55']]); + // R43: `scroll` reaches the device through its bound operation, so the dispatcher sees nothing. + expect(dispatchCalls).toEqual([]); + expect(gestureRuntimeSpies.scrollDirection).toHaveBeenCalledTimes(1); + expect(gestureRuntimeSpies.scrollDirection.mock.calls[0]?.[0]).toMatchObject({ + direction: 'down', + options: { amount: 0.55 }, + }); expect(execCalls).toEqual([]); expect(openAndroidApp).not.toHaveBeenCalled(); expect(snapshotCalls).toBe(1); @@ -200,6 +224,7 @@ test('generic Android gesture commands skip local dialog recovery for provider d dispatchResult = {}; execCalls.length = 0; dispatchCalls.length = 0; + gestureRuntimeSpies.scrollDirection.mockClear(); const sessionStore = makeSessionStore('agent-device-router-android-modal-provider-'); const session = makeAndroidSession('default'); @@ -223,6 +248,7 @@ test('generic Android gesture commands skip local dialog recovery for provider d deviceInventoryGateways: createTestDeviceInventoryGateways(), providerDeviceRuntimeScope: providers.providerDeviceRuntimeScope, trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -234,7 +260,13 @@ test('generic Android gesture commands skip local dialog recovery for provider d }); expect(response.ok).toBe(true); - expect(dispatchCalls).toEqual([['scroll', 'down', '0.55']]); + // R43: `scroll` reaches the device through its bound operation, so the dispatcher sees nothing. + expect(dispatchCalls).toEqual([]); + expect(gestureRuntimeSpies.scrollDirection).toHaveBeenCalledTimes(1); + expect(gestureRuntimeSpies.scrollDirection.mock.calls[0]?.[0]).toMatchObject({ + direction: 'down', + options: { amount: 0.55 }, + }); expect(execCalls).toEqual([]); expect(snapshotCalls).toBe(0); }); diff --git a/src/daemon/__tests__/request-router-recording-health.test.ts b/src/daemon/__tests__/request-router-recording-health.test.ts index 1cc3fd186..5ed3421d3 100644 --- a/src/daemon/__tests__/request-router-recording-health.test.ts +++ b/src/daemon/__tests__/request-router-recording-health.test.ts @@ -5,42 +5,32 @@ import path from 'node:path'; vi.mock('../../core/dispatch.ts', async (importOriginal) => { const actual = await importOriginal(); - return { - ...actual, - dispatchCommand: vi.fn(async () => ({})), - dispatchGesturePlan: vi.fn(async () => ({})), - dispatchGestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 390, height: 844 })), - }; + return { ...actual, dispatchCommand: vi.fn(async () => ({})) }; }); vi.mock('../../platforms/apple/core/runner/runner-client.ts', () => ({ getRunnerSessionSnapshot: vi.fn(), })); -import { - dispatchCommand, - dispatchGesturePlan, - dispatchGestureViewport, -} from '../../core/dispatch.ts'; +import { dispatchCommand } from '../../core/dispatch.ts'; import { getRunnerSessionSnapshot } from '../../platforms/apple/core/runner/runner-client.ts'; -import { createRequestHandler } from './test-device-runtime-gateway.ts'; +import { + createRequestHandler, + gestureDeviceRuntimeGateway, + gestureRuntimeSpies, +} from './test-device-runtime-gateway.ts'; import type { SessionState } from '../types.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { makeTestScreenRecordingResource } from '../../__tests__/test-utils/screen-recording-live-handle.ts'; const mockDispatch = vi.mocked(dispatchCommand); -const mockDispatchGesturePlan = vi.mocked(dispatchGesturePlan); -const mockDispatchGestureViewport = vi.mocked(dispatchGestureViewport); const mockGetRunnerSessionSnapshot = vi.mocked(getRunnerSessionSnapshot); beforeEach(() => { mockDispatch.mockReset(); mockDispatch.mockResolvedValue({}); - mockDispatchGesturePlan.mockReset(); - mockDispatchGesturePlan.mockResolvedValue({}); - mockDispatchGestureViewport.mockReset(); - mockDispatchGestureViewport.mockResolvedValue({ x: 0, y: 0, width: 390, height: 844 }); + for (const spy of Object.values(gestureRuntimeSpies)) spy.mockClear(); mockGetRunnerSessionSnapshot.mockReset(); }); @@ -130,6 +120,7 @@ test('router allows canonical iOS simulator gestures during overlay recording af leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -143,8 +134,8 @@ test('router allows canonical iOS simulator gestures during overlay recording af expect(response.ok).toBe(true); expect(mockGetRunnerSessionSnapshot).not.toHaveBeenCalled(); - expect(mockDispatchGestureViewport).toHaveBeenCalledOnce(); - expect(mockDispatchGesturePlan).toHaveBeenCalledOnce(); + expect(gestureRuntimeSpies.gestureViewport).toHaveBeenCalledOnce(); + expect(gestureRuntimeSpies.performMultiTouchGesturePlan).toHaveBeenCalledOnce(); const recording = sessionStore.get('default')?.screenRecording?.handle.inspect(); expect(recording?.invalidatedReason).toBeUndefined(); expect(recording?.gestureEvents).toHaveLength(1); diff --git a/src/daemon/__tests__/request-router-screenshot.test.ts b/src/daemon/__tests__/request-router-screenshot.test.ts index 26699bde4..a148b955a 100644 --- a/src/daemon/__tests__/request-router-screenshot.test.ts +++ b/src/daemon/__tests__/request-router-screenshot.test.ts @@ -234,10 +234,9 @@ test('router serializes concurrent commands for the same device across sessions' writeSolidPng(input.outPath); await gate('screenshot'); }, - }); - mockDispatch.mockImplementation(async (_device, command) => { - await gate(command); - return {}; + onScroll: async () => { + await gate('scroll'); + }, }); const handler = createRequestHandler({ diff --git a/src/daemon/__tests__/screenshot-runtime-fixture.ts b/src/daemon/__tests__/screenshot-runtime-fixture.ts index d41e0eee7..4fff31bb3 100644 --- a/src/daemon/__tests__/screenshot-runtime-fixture.ts +++ b/src/daemon/__tests__/screenshot-runtime-fixture.ts @@ -28,6 +28,8 @@ export type ScreenshotRuntimeFixtureOptions = Readonly<{ /** Replaces the default "write a solid PNG at the requested path" capture behavior. */ onCapture?: (input: CaptureScreenshotInput) => Promise | void; snapshotResult?: (input: CaptureSnapshotInput) => SnapshotResult; + /** Gate for the neighbouring bound `scroll`, used by the device-lock serialization tests. */ + onScroll?: () => Promise | void; }>; export type ScreenshotRuntimeFixture = Readonly<{ @@ -60,6 +62,12 @@ export function screenshotRuntimeFixture( async (input: CaptureSnapshotInput): Promise => options.snapshotResult?.(input) ?? { nodes: [], backend: 'android' }, ); + // R43: `scroll` is the neighbouring command the device-lock tests use to prove serialization, + // and it now reaches the platform through a bound operation like the screenshot beside it. + const scrollDirection = vi.fn(async () => { + await options.onScroll?.(); + return {}; + }); // The unavailable gateway is the exhaustive fact catalog; only the capture cells are overridden. const facts = async (device: DeviceInfo): Promise> => { @@ -69,6 +77,7 @@ export function screenshotRuntimeFixture( operations: { ...base.operations, ...screenshotRuntimeOperationFacts({ capture: options.capture ?? available }), + scrollDirection: available, ...snapshotRuntimeOperationFacts({ capture: options.snapshot ?? available, customActions: options.snapshot ?? available, @@ -89,6 +98,7 @@ export function screenshotRuntimeFixture( captureSnapshot, captureSnapshotWithCustomActions: captureSnapshot, captureSnapshotWithoutActiveApp: captureSnapshot, + scrollDirection, }, [Symbol.asyncDispose]: async () => {}, }; diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts new file mode 100644 index 000000000..548da97a9 --- /dev/null +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -0,0 +1,301 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import type { + BoundDeviceRuntime, + PlatformRuntimeOperations, + RuntimeFacts, +} from '@agent-device/contracts/platform'; +import type { DaemonCommandContext } from '../context.ts'; +import { resolveBoundScrollRuntime } from '../scroll-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import { unavailableDeploymentSnapshotAndShutdownOperationFacts } from '../../__tests__/test-utils/runtime-operation-facts.ts'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; + +/** + * The retired `handleScrollCommand` suite, re-pointed at the bound runtime (R43). Every + * assertion is the behavior `main` produced: the same parse rejections, the same execution + * options handed to the owner, the same edge-pass loop, and the same scoped-capture failure. + * + * What moved is WHERE the edge refusal happens — the retired leaf discovered a missing snapshot + * mid-command, and admission now proves the capture before any pass runs (ADR 0019 §6). + */ +type ScrollCall = { direction: string; options: unknown }; + +function bindings(options: { + scroll: (direction: string, scrollOptions: unknown) => Promise | void>; + captureSnapshot?: (input: { options?: { scope?: string } }) => Promise; +}): { inspectFacts: InspectDeviceRuntimeFacts; bindDevice: BindDeviceRuntime } { + const available = { available: true } as const; + const facts = { + device: { family: 'apple', kind: 'simulator', providerMode: 'local' }, + operations: { + ...unavailableDeploymentSnapshotAndShutdownOperationFacts, + scrollDirection: available, + ...(options.captureSnapshot ? { captureSnapshot: available } : {}), + }, + } as unknown as RuntimeFacts; + return { + inspectFacts: async () => facts, + bindDevice: (async () => + ({ + facts, + operations: { + scrollDirection: async (input: { direction: string; options: unknown }) => + await options.scroll(input.direction, input.options), + ...(options.captureSnapshot ? { captureSnapshot: options.captureSnapshot } : {}), + }, + }) as unknown as BoundDeviceRuntime) as unknown as BindDeviceRuntime, + }; +} + +async function runScroll( + positionals: string[], + context: Partial, + options: Parameters[0], +): Promise> { + const resolved = await resolveBoundScrollRuntime({ + device: IOS_SIMULATOR, + positionals, + context: context as DaemonCommandContext, + ...bindings(options), + }); + if (!resolved.ok) throw new AppError('UNSUPPORTED_OPERATION', 'admission refused the scroll'); + const data = await resolved.execute({ + dispatchContext: context as DaemonCommandContext, + } as Parameters[0]); + return (data ?? {}) as Record; +} + +test('bound scroll rejects mixing amount and --pixels', async () => { + await assert.rejects( + () => + runScroll( + ['down', '0.4'], + { pixels: 240 }, + { + scroll: async () => { + throw new Error('scroll should be rejected before the owner is reached'); + }, + }, + ), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /either a relative amount or --pixels/i.test(error.message), + ); +}); + +test('bound scroll forwards pixels and duration without reporting ignored duration', async () => { + const calls: ScrollCall[] = []; + const result = await runScroll( + ['down'], + { pixels: 200, durationMs: 50 }, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { ok: true }; + }, + }, + ); + + assert.deepEqual(calls, [ + { + direction: 'down', + options: { + amount: undefined, + pixels: 200, + durationMs: 50, + releaseBehavior: 'controlled', + }, + }, + ]); + assert.equal(result.pixels, 200); + assert.equal(result.durationMs, undefined); +}); + +test('bound scroll reports duration when the owner honored it', async () => { + const result = await runScroll( + ['down'], + { pixels: 200, durationMs: 50 }, + { + scroll: async () => ({ pixels: 200, durationMs: 50 }), + }, + ); + assert.equal(result.pixels, 200); + assert.equal(result.durationMs, 50); +}); + +test('bound scroll rejects duration above the shared cap', async () => { + await assert.rejects( + () => + runScroll( + ['down'], + { pixels: 200, durationMs: 10_001 }, + { + scroll: async () => { + throw new Error('scroll should be rejected before the owner is reached'); + }, + }, + ), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /durationMs.*at most 10000/i.test(error.message), + ); +}); + +test('bound scroll bottom refuses at admission when the owner declares no capture', async () => { + const calls: ScrollCall[] = []; + const resolved = await resolveBoundScrollRuntime({ + device: IOS_SIMULATOR, + positionals: ['bottom'], + context: {} as DaemonCommandContext, + ...bindings({ + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + }), + }); + + assert.equal(resolved.ok, false); + if (resolved.ok || resolved.response.ok) return; + assert.equal(resolved.response.error.code, 'UNSUPPORTED_OPERATION'); + assert.match(String(resolved.response.error.message), /requires snapshot support/i); + // The refusal is now proof-before-execution: no pass ran, and none could have. + assert.equal(calls.length, 0); +}); + +test('bound scroll bottom does not scroll when no hidden content is below', async () => { + const calls: ScrollCall[] = []; + const result = await runScroll( + ['bottom'], + {}, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + captureSnapshot: async () => + makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), + }, + ); + + assert.equal(calls.length, 0); + assert.equal(result.direction, 'down'); + assert.equal(result.edge, 'bottom'); + assert.equal(result.passes, 0); + assert.match(String(result.message), /Already at bottom/); +}); + +test('bound scroll bottom scrolls only while a scoped capture confirms hidden content', async () => { + const calls: ScrollCall[] = []; + const snapshotScopes: unknown[] = []; + const snapshots = [ + makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), + ]; + const result = await runScroll( + ['bottom'], + {}, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + captureSnapshot: async (input) => { + snapshotScopes.push(input.options?.scope); + return snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)]; + }, + }, + ); + + assert.equal(calls.length, 1); + assert.deepEqual(calls[0], { + direction: 'down', + options: { + amount: undefined, + pixels: undefined, + durationMs: undefined, + releaseBehavior: 'inertial', + }, + }); + assert.equal(result.passes, 1); + assert.equal(result.lastPass, 1); + assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); +}); + +test('bound scroll bottom tolerates unchanged signatures while hidden content advances', async () => { + const calls: ScrollCall[] = []; + const snapshots = [ + makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + makeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), + ]; + let snapshotIndex = 0; + const result = await runScroll( + ['bottom'], + {}, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + captureSnapshot: async () => snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], + }, + ); + + assert.equal(calls.length, 2); + assert.equal(result.passes, 2); +}); + +test('bound scroll bottom keeps scoped capture failures scoped', async () => { + let snapshotCount = 0; + await assert.rejects( + () => + runScroll( + ['bottom'], + {}, + { + scroll: async () => ({}), + captureSnapshot: async (input) => { + snapshotCount += 1; + if (input.options?.scope) throw new Error('scoped snapshot failed'); + return makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }); + }, + }, + ), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /scoped container/i.test(error.message) && + error.details?.scope === 'Messages', + ); + assert.equal(snapshotCount, 2); +}); + +function makeScrollSnapshot(options: { hiddenBelow: boolean; message: string }) { + return { + backend: 'xctest' as const, + nodes: [ + { + index: 1, + type: 'ScrollView', + label: 'Messages', + hiddenContentBelow: options.hiddenBelow ? true : undefined, + rect: { x: 0, y: 100, width: 400, height: 600 }, + }, + { + index: 2, + parentIndex: 1, + type: 'Button', + label: options.message, + rect: { x: 0, y: 640, width: 400, height: 56 }, + }, + ], + truncated: false, + }; +} diff --git a/src/daemon/__tests__/test-device-runtime-gateway.ts b/src/daemon/__tests__/test-device-runtime-gateway.ts index a81599525..23809909c 100644 --- a/src/daemon/__tests__/test-device-runtime-gateway.ts +++ b/src/daemon/__tests__/test-device-runtime-gateway.ts @@ -1,3 +1,4 @@ +import { vi } from 'vitest'; import { applicationLifecycleOperationFacts, availableApplicationLifecycleOperations, @@ -5,7 +6,9 @@ import { narrowDeviceBinding, type ApplicationLifecycleOperationFacts, type DeviceRuntimeGateway, + type GesturePlanInput, type PlatformRuntimeOperations, + type ScrollDirectionInput, } from '@agent-device/contracts/platform'; import { createRequestHandler as createProductionRequestHandler, @@ -141,6 +144,48 @@ export const unavailableDeviceRuntimeGateway: DeviceRuntimeGateway {}, }); +/** + * Spies for the gesture surface a router-level test drives. They replace the retired + * `dispatchGesturePlan` / `dispatchGestureViewport` module mocks: gestures now reach the platform + * through a bound operation, so the observation point is the operation, not the dispatcher. + */ +export const gestureRuntimeSpies = { + performGesturePlan: vi.fn(async (_input: GesturePlanInput) => ({})), + performDirectionalFlingPlan: vi.fn(async (_input: GesturePlanInput) => ({})), + performMultiTouchGesturePlan: vi.fn(async (_input: GesturePlanInput) => ({})), + performTargetAuthoredDrag: vi.fn(async (_input: GesturePlanInput) => ({})), + gestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 390, height: 844 })), + scrollDirection: vi.fn(async (_input: ScrollDirectionInput) => ({})), +}; + +/** The unavailable gateway plus an admitted gesture/scroll surface. */ +export const gestureDeviceRuntimeGateway: DeviceRuntimeGateway = + Object.freeze({ + inspectFacts: async (device) => (await gestureBinding(device)).facts, + bind: async (request) => await gestureBinding(request.device), + shutdown: async () => {}, + }); + +async function gestureBinding(device: DeviceInfo) { + const base = await unavailableBinding(device); + return { + ...base, + facts: { + ...base.facts, + operations: { + ...base.facts.operations, + performGesturePlan: available, + performDirectionalFlingPlan: available, + performMultiTouchGesturePlan: available, + performTargetAuthoredDrag: available, + gestureViewport: available, + scrollDirection: available, + }, + }, + operations: { ...base.operations, ...gestureRuntimeSpies }, + } as unknown as Awaited['bind']>>; +} + async function unavailableBinding(device: DeviceInfo) { return await unavailableDeviceRuntimeGateway.bind({ device, diff --git a/src/daemon/generic-runtime-execution.ts b/src/daemon/generic-runtime-execution.ts index 9c207a7c9..c2e6c8f6a 100644 --- a/src/daemon/generic-runtime-execution.ts +++ b/src/daemon/generic-runtime-execution.ts @@ -1,7 +1,9 @@ import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; import { resolveBoundFocusRuntime } from './focus-runtime.ts'; import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts'; +import { resolveBoundScrollRuntime } from './scroll-runtime.ts'; import type { ScreenshotRuntimeBindings } from './screenshot-runtime-binding.ts'; +import type { DaemonCommandContext } from './context.ts'; import type { DaemonRequest, SessionState } from './types.ts'; import { resolveBoundViewportRuntime } from './viewport-runtime.ts'; @@ -11,7 +13,12 @@ import { resolveBoundViewportRuntime } from './viewport-runtime.ts'; * name. `undefined` means the leaf still executes through legacy platform dispatch. */ export async function resolveGenericRuntimeExecution( - params: Readonly<{ req: DaemonRequest; session: SessionState }> & ScreenshotRuntimeBindings, + params: Readonly<{ + req: DaemonRequest; + session: SessionState; + context: DaemonCommandContext; + }> & + ScreenshotRuntimeBindings, ): Promise { switch (params.req.command) { case 'screenshot': @@ -23,6 +30,14 @@ export async function resolveGenericRuntimeExecution( inspectFacts: params.inspectFacts, bindDevice: params.bindDevice, }); + case 'scroll': + return await resolveBoundScrollRuntime({ + device: params.session.device, + positionals: params.req.positionals ?? [], + context: params.context, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); case 'viewport': return await resolveBoundViewportRuntime({ device: params.session.device, diff --git a/src/daemon/gesture-runtime.ts b/src/daemon/gesture-runtime.ts new file mode 100644 index 000000000..f27e2724f --- /dev/null +++ b/src/daemon/gesture-runtime.ts @@ -0,0 +1,176 @@ +import type { GestureCommandInput, GesturePlan } from '@agent-device/contracts/interaction'; +import { + gestureRefusalMessage, + resolveGestureRuntimePlan, + type GesturePlanInput, + type GestureRuntimeOperations, + type GestureRuntimePlan, +} from '@agent-device/contracts/platform'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import type { DaemonCommandContext } from './context.ts'; +import type { DaemonFailureResponse } from './handlers/response.ts'; +import { admitRuntimeOperations, type RuntimeAdmissionBindings } from './runtime-admission.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; + +/** + * One request's bound gesture authority. `gesture` and `swipe` both build their plans inside the + * shared client-side orchestration, which can execute several plans per request (`swipe --count`, + * a drag's resolved endpoints), so the executor is a closure over the single binding rather than + * a value frozen at bind time. + */ +export type BoundGestureExecutor = Readonly<{ + performPlan: ( + plan: GesturePlan, + context: DaemonCommandContext, + ) => Promise | void>; + /** + * Present only when the admitted owner advertised its own frame read. Absence is not a failure: + * the caller derives the frame from a capture instead, exactly as it does today for an owner + * without one (Linux). + */ + gestureViewport?: (context: DaemonCommandContext) => Promise; +}>; + +export type ResolvedGestureRuntime = + | Readonly<{ ok: false; response: DaemonFailureResponse }> + | Readonly<{ ok: true; gestures: BoundGestureExecutor }>; + +/** + * The one place `gesture` and `swipe` reach a device (ADR 0019). The gesture input selects ONE + * execution tier, admission inspects that tier's fact on the exact owner, and the handler binds + * once — before any plan is built, so a device that cannot synthesize this gesture is refused + * where the retired `requireGestureSupported` refused it rather than mid-series. + * + * The refusal message is composed from the tier and the device so every string the retired + * admission produced survives verbatim; the hint comes from the owner's own fact. + */ +export async function resolveBoundGestureRuntime( + params: { + device: DeviceInfo; + /** The normalized gesture — for `swipe`, the fling its motion normalizes to. */ + input: GestureCommandInput; + } & RuntimeAdmissionBindings, +): Promise { + const plan = resolveGestureRuntimePlan(params.input); + const admitted = await admitRuntimeOperations({ + command: 'gesture', + device: params.device, + required: plan.use.required, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + // The retired admission THREW an `AppError`, which the gesture handler's catch normalized — + // so the refusal is built the same way here. Going through `errorResponse` instead would + // drop the code's default hint and its `retriable` classification from the wire shape. + unavailableResponse: (unavailable) => ({ + ok: false, + error: normalizeError( + new AppError( + 'UNSUPPORTED_OPERATION', + gestureRefusalMessage(params.device, plan.tier, params.input.intent), + { + gesture: params.input.intent, + ...(unavailable.hint === undefined ? {} : { hint: unavailable.hint }), + }, + ), + ), + }), + }); + if (admitted.type === 'response') return { ok: false, response: admitted.response }; + // One bind, with the exactly-typed use the tier selected (ADR 0019 §9). + return { ok: true, gestures: await bindGestureTier(admitted.bind, params.device, plan) }; +} + +/** + * The ONE place each bound gesture tier executes (R42/R44, shared by `gesture` and `swipe`). + * + * Each branch binds its own exactly-typed use, so the operation it calls is non-optional by + * construction — no cast, no non-null repair, and exactly one lexical owner per tier, which is + * what lets the cutover gate prove no parallel route exists. The branches read alike because + * the tiers share their mechanics; what differs is the cell each one proves. + */ +async function bindGestureTier( + bind: Extract>, { type: 'admitted' }>['bind'], + device: DeviceInfo, + plan: GestureRuntimePlan, +): Promise { + switch (plan.tier) { + case 'plan': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performGesturePlan(gesturePlanInput(gesturePlan, context)), + ...selectGestureViewport(runtime), + }; + } + case 'directional-fling': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performDirectionalFlingPlan( + gesturePlanInput(gesturePlan, context), + ), + ...selectGestureViewport(runtime), + }; + } + case 'multi-touch': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performMultiTouchGesturePlan( + gesturePlanInput(gesturePlan, context), + ), + ...selectGestureViewport(runtime), + }; + } + case 'target-authored-drag': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performTargetAuthoredDrag( + gesturePlanInput(gesturePlan, context), + ), + ...selectGestureViewport(runtime), + }; + } + } +} + +/** + * The owner's own frame read, present only when its facts advertised it. Absence is not a + * failure and not a fallback: the caller derives the frame from a capture instead, which is how + * a Linux gesture resolves its viewport today. + */ +function selectGestureViewport( + runtime: Readonly<{ + operations: Readonly<{ gestureViewport?: GestureRuntimeOperations['gestureViewport'] }>; + }>, +): Pick { + const { gestureViewport } = runtime.operations; + const selected = gestureViewport ? { operations: { gestureViewport } } : undefined; + return Object.freeze({ + ...(selected + ? { + gestureViewport: async (context: DaemonCommandContext) => + await selected.operations.gestureViewport(gestureViewportInput(context)), + } + : {}), + }); +} + +/** The neutral intent one gesture carries, projected from a resolved command context. */ +function gesturePlanInput(plan: GesturePlan, context: DaemonCommandContext): GesturePlanInput { + return { + plan, + ...(context.appBundleId === undefined ? {} : { options: { appBundleId: context.appBundleId } }), + execution: runtimeExecutionFromContext(context), + }; +} + +function gestureViewportInput(context: DaemonCommandContext) { + return { + ...(context.appBundleId === undefined ? {} : { options: { appBundleId: context.appBundleId } }), + execution: runtimeExecutionFromContext(context), + }; +} diff --git a/src/daemon/handlers/__tests__/gesture-runtime-bindings.fixtures.ts b/src/daemon/handlers/__tests__/gesture-runtime-bindings.fixtures.ts new file mode 100644 index 000000000..943efd95e --- /dev/null +++ b/src/daemon/handlers/__tests__/gesture-runtime-bindings.fixtures.ts @@ -0,0 +1,63 @@ +import { vi } from 'vitest'; +import type { + BoundDeviceRuntime, + PlatformRuntimeOperations, + RuntimeFacts, +} from '@agent-device/contracts/platform'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import type { + BindDeviceRuntime, + InspectDeviceRuntimeFacts, +} from '../../request-runtime-binding.ts'; +import { unavailableDeploymentSnapshotAndShutdownOperationFacts } from '../../../__tests__/test-utils/runtime-operation-facts.ts'; + +const available = Object.freeze({ available: true } as const); + +/** + * A runtime that admits every gesture tier, with one spy per operation. + * + * The counters on `inspectFacts` / `bindDevice` are the ADR 0019 §9 regression surface: a handler + * that binds per repetition, or re-inspects after resolving a target, shows up here as a count + * greater than one (the defect #1944's P1 fixed). + */ +export function gestureRuntimeBindingsFixture( + options: Readonly<{ viewport?: Rect; unavailable?: readonly string[] }> = {}, +) { + const unavailable = new Set(options.unavailable ?? []); + const cell = (operation: string) => + unavailable.has(operation) + ? ({ available: false, reason: 'unsupported-platform-leaf' } as const) + : available; + const operationSpies = { + performGesturePlan: vi.fn(async () => ({})), + performDirectionalFlingPlan: vi.fn(async () => ({})), + performMultiTouchGesturePlan: vi.fn(async () => ({})), + performTargetAuthoredDrag: vi.fn(async () => ({})), + gestureViewport: vi.fn( + async () => options.viewport ?? ({ x: 0, y: 0, width: 400, height: 800 } as Rect), + ), + }; + const facts = { + device: { family: 'apple', kind: 'simulator', providerMode: 'local' }, + operations: { + ...unavailableDeploymentSnapshotAndShutdownOperationFacts, + performGesturePlan: cell('performGesturePlan'), + performDirectionalFlingPlan: cell('performDirectionalFlingPlan'), + performMultiTouchGesturePlan: cell('performMultiTouchGesturePlan'), + performTargetAuthoredDrag: cell('performTargetAuthoredDrag'), + gestureViewport: cell('gestureViewport'), + }, + } as unknown as RuntimeFacts; + const inspectFacts = vi.fn(async () => facts) as unknown as InspectDeviceRuntimeFacts & + ReturnType; + const bindDevice = vi.fn( + async () => + ({ + facts, + operations: Object.fromEntries( + Object.entries(operationSpies).filter(([name]) => !unavailable.has(name)), + ), + }) as unknown as BoundDeviceRuntime, + ) as unknown as BindDeviceRuntime & ReturnType; + return { ...operationSpies, facts, inspectFacts, bindDevice }; +} diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index 910f8f4e2..6b33617fc 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -12,6 +12,8 @@ import { type MaterializedAppSource, type PlatformRuntimeOperations, type RuntimeFacts, + gestureRuntimeOperationFacts, + scrollRuntimeOperationFacts, } from '@agent-device/contracts/platform'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -355,6 +357,14 @@ function sourceRuntimeFacts( setViewport: unavailable, focusPoint: unavailable, typeText: unavailable, + ...gestureRuntimeOperationFacts({ + plan: unavailable, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: unavailable, + viewport: unavailable, + }), + ...scrollRuntimeOperationFacts({ scroll: unavailable }), readTextAtPoint: unavailable, deployApp: unavailable, materializeAppSource: materializationAvailable ? { available: true } : unavailable, diff --git a/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts b/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts index 7de89e204..26602f1dd 100644 --- a/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts +++ b/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts @@ -7,15 +7,6 @@ import { import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; import { activateCompleteRefFrame, refFrameState } from '../../ref-frame.ts'; -vi.mock('../../../core/dispatch.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - dispatchGestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 400, height: 800 })), - dispatchGesturePlan: vi.fn(async () => ({})), - }; -}); - vi.mock('../interaction-snapshot.ts', async (importOriginal) => { const actual = await importOriginal(); return { @@ -26,15 +17,14 @@ vi.mock('../interaction-snapshot.ts', async (importOriginal) => { }; }); -import { dispatchGesturePlan } from '../../../core/dispatch.ts'; import { handleInteractionCommands } from '../interaction.ts'; +import { gestureRuntimeBindingsFixture } from './gesture-runtime-bindings.fixtures.ts'; -const mockDispatchGesturePlan = vi.mocked(dispatchGesturePlan); const contextFromFlags = () => ({}); +let gestures = gestureRuntimeBindingsFixture(); beforeEach(() => { - mockDispatchGesturePlan.mockClear(); - mockDispatchGesturePlan.mockResolvedValue({}); + gestures = gestureRuntimeBindingsFixture(); }); function makeDragSession(sessionName: string) { @@ -96,6 +86,8 @@ async function runDrag(sessionStore: ReturnType, sessio sessionName, sessionStore, contextFromFlags, + inspectFacts: gestures.inspectFacts, + bindDevice: gestures.bindDevice, }); } @@ -129,7 +121,11 @@ test('recorded ref drag dispatches once and stores portable selectors with both expect(response.data).not.toHaveProperty('selectorChain'); expect(response.data).not.toHaveProperty('targetEvidence'); } - expect(mockDispatchGesturePlan).toHaveBeenCalledTimes(1); + // ADR 0019 §9 regression guard: a drag resolves two targets and still takes exactly one + // inspection and one bind — the #1944 P1 shape. + expect(gestures.performTargetAuthoredDrag).toHaveBeenCalledTimes(1); + expect(gestures.inspectFacts).toHaveBeenCalledTimes(1); + expect(gestures.bindDevice).toHaveBeenCalledTimes(1); expect(refFrameState(session)).toBe('expired'); const recorded = session.actions[0]; @@ -163,5 +159,5 @@ test('a second ref drag is rejected before dispatch after the first drag expires currentGeneration: 42, }); } - expect(mockDispatchGesturePlan).toHaveBeenCalledTimes(1); + expect(gestures.performTargetAuthoredDrag).toHaveBeenCalledTimes(1); }); diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts index 3140a1e66..a9fb1ab9e 100644 --- a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -86,6 +86,8 @@ function elementReadFacts(device: DeviceInfo): RuntimeFacts { screenshot: { available: false, reason: 'owner-capability-missing' }, viewport: { available: false, reason: 'owner-capability-missing' }, focus: { available: false, reason: 'owner-capability-missing' }, + gesture: { available: false, reason: 'owner-capability-missing' }, + scroll: { available: false, reason: 'owner-capability-missing' }, typeText: { available: false, reason: 'owner-capability-missing' }, elementText: { available: false, reason: 'owner-capability-missing' }, readiness: { available: false, reason: 'unsupported-device-kind' }, @@ -135,6 +137,8 @@ test('appstate rejects web before Android app-state backend dispatch', async () screenshot: { available: false, reason: 'unsupported-platform-leaf' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, focus: { available: false, reason: 'unsupported-platform-leaf' }, + gesture: { available: false, reason: 'unsupported-platform-leaf' }, + scroll: { available: false, reason: 'unsupported-platform-leaf' }, typeText: { available: false, reason: 'unsupported-platform-leaf' }, elementText: { available: false, reason: 'unsupported-platform-leaf' }, readiness: { available: false, reason: 'unsupported-platform-leaf' }, diff --git a/src/daemon/handlers/interaction-gesture.ts b/src/daemon/handlers/interaction-gesture.ts index 88a82c3a6..3d1bf1527 100644 --- a/src/daemon/handlers/interaction-gesture.ts +++ b/src/daemon/handlers/interaction-gesture.ts @@ -20,7 +20,7 @@ import { splitRefGenerationSuffix, type Point, } from '@agent-device/kernel/snapshot'; -import { requireGestureSupported } from '../../core/capabilities.ts'; +import { resolveBoundGestureRuntime, type BoundGestureExecutor } from '../gesture-runtime.ts'; import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; import { sleep } from '../../utils/timeouts.ts'; import { ensureAndroidBlockingSystemDialogReady } from '../android-system-dialog.ts'; @@ -49,6 +49,19 @@ type GestureInteractionOutcome = { recordedTargets?: { source: RecordedTargetCapture; destination: RecordedTargetCapture }; }; +/** + * A refused gesture short-circuits with the admission's own response. The retired + * `requireGestureSupported` threw, so the refusal skipped the after-command dialog check and + * returned the normalized error; returning it here preserves that control flow exactly. + */ +type GestureInteractionResult = GestureInteractionOutcome | Readonly<{ refused: DaemonResponse }>; + +function isRefusal( + result: GestureInteractionResult, +): result is Readonly<{ refused: DaemonResponse }> { + return 'refused' in result; +} + export async function dispatchGestureViaRuntime( params: GestureHandlerParams, ): Promise { @@ -60,14 +73,22 @@ export async function dispatchGestureViaRuntime( async function runGestureInteraction( params: GestureHandlerParams, session: SessionState, -): Promise { +): Promise { const input = readGesturePayload(params.req.input); const gesture = prepareGestureCommandInput(input, session); if (gesture.intent === 'pan' && params.req.internal?.gestureExecutionProfile) { gesture.executionProfile = params.req.internal.gestureExecutionProfile; } - requireGestureSupported(gesture, session.device); - const runtime = createGestureRuntime(params); + // ADR 0019 §9: the gesture input selects one execution tier and this is the request's ONE bind, + // taken before any plan is built — a drag binds here, not after its targets resolve. + const bound = await resolveBoundGestureRuntime({ + device: session.device, + input: gesture, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!bound.ok) return { refused: bound.response }; + const runtime = createGestureRuntime(params, bound.gestures); const context = { session: params.sessionName, requestId: params.req.meta?.requestId }; const result = await runPreparedGesture(runtime, context, gesture, params.req.internal); return buildGestureOutcome(input, gesture, result, params.req.flags); @@ -131,11 +152,19 @@ export async function dispatchSwipeViaRuntime( ): Promise { return await dispatchGestureInteraction(params, 'swipe', async (session) => { const input = readSwipeInput(params.req.input); - requireGestureSupported(normalizePublicSwipeMotion(input).gesture, session.device); + // One bind for the whole series: `--count N` executes the bound operation N times under a + // single binding, never one bind per repetition (ADR 0019 §9). + const bound = await resolveBoundGestureRuntime({ + device: session.device, + input: normalizePublicSwipeMotion(input).gesture, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!bound.ok) return { refused: bound.response }; const count = input.count ?? 1; const pauseMs = input.pauseMs ?? 0; const pattern = input.pattern ?? 'one-way'; - const runtime = createGestureRuntime(params); + const runtime = createGestureRuntime(params, bound.gestures); const result = await runSwipeRepetitions(runtime, params, input, count, pauseMs, pattern); return { positionals: swipeReplayPositionals(input), @@ -158,9 +187,10 @@ export async function dispatchSwipeViaRuntime( }); } -function createGestureRuntime(params: GestureHandlerParams) { +function createGestureRuntime(params: GestureHandlerParams, gestures: BoundGestureExecutor) { return createInteractionRuntime({ ...params, + gestures, pairedGestureViewport: params.req.internal?.gestureViewport, }); } @@ -168,7 +198,7 @@ function createGestureRuntime(params: GestureHandlerParams) { async function dispatchGestureInteraction( params: GestureHandlerParams, command: 'gesture' | 'swipe', - run: (session: SessionState) => Promise, + run: (session: SessionState) => Promise, ): Promise { const session = params.sessionStore.get(params.sessionName); if (!session) return noActiveSessionError(); @@ -183,6 +213,7 @@ async function dispatchGestureInteraction( phase: 'before-command', }); const outcome = await run(session); + if (isRefusal(outcome)) return outcome.refused; if (!providerDevice) { await ensureAndroidBlockingSystemDialogReady({ session, diff --git a/src/daemon/handlers/interaction-runtime.ts b/src/daemon/handlers/interaction-runtime.ts index d7641abad..cf2cef7ac 100644 --- a/src/daemon/handlers/interaction-runtime.ts +++ b/src/daemon/handlers/interaction-runtime.ts @@ -1,8 +1,4 @@ -import { - dispatchCommand, - dispatchGesturePlan, - dispatchGestureViewport, -} from '../../core/dispatch.ts'; +import { dispatchCommand } from '../../core/dispatch.ts'; import { publicPlatformString } from '@agent-device/kernel/device'; import type { AgentDeviceBackend, @@ -26,10 +22,18 @@ import { getRequestSignal } from '../../request/cancel.ts'; import { buildAppleRunnerRequestOptions } from '../apple-runner-options.ts'; import { isLocalIosRunnerSession } from '../direct-ios-selector.ts'; import { confirmIosOffscreenTargetVisible } from '../offscreen-target-probe.ts'; +import type { BoundGestureExecutor } from '../gesture-runtime.ts'; +import type { DaemonCommandContext } from '../context.ts'; type InteractionRuntimeParams = InteractionHandlerParams & { captureSnapshotForSession: CaptureSnapshotForSession; pairedGestureViewport?: Rect; + /** + * The request's single gesture binding (ADR 0019), supplied only by the `gesture`/`swipe` + * handler. Every other interaction command leaves it out, and the backend then exposes no + * gesture members at all — the touch leaves that share this backend execute no gestures. + */ + gestures?: BoundGestureExecutor; }; export function createInteractionRuntime(params: InteractionRuntimeParams) { @@ -63,6 +67,8 @@ function createInteractionBackend( ): AgentDeviceBackend { const { req, session } = params; const webProvider = resolveNativeWebInteractionProvider(session); + const gestureContext = () => + params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath); return { platform: publicPlatformString(session.device), captureSnapshot: async (context, options): Promise => ({ @@ -79,12 +85,7 @@ function createInteractionBackend( }, ), }), - resolveGestureViewport: async () => - params.pairedGestureViewport ?? - (await dispatchGestureViewport( - session.device, - params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - )), + ...gestureBackendMembers(params, session, gestureContext), // #1542: iOS-only escape hatch for the off-screen refusal double-check. // Local (non-provider) iOS sessions get a direct, AX-tree-independent // probe (deliberately NOT skipped while postGestureStabilization is @@ -188,15 +189,38 @@ function createInteractionBackend( ), ); }, + }; +} + +/** + * The gesture members, present only for the `gesture`/`swipe` handler that bound them (R42/R44). + * Every other interaction command shares this backend and executes no gestures, so it gets + * neither member — the backend holds no gesture reach it cannot prove. + * + * A replay-supplied viewport still wins over the owner's own read, exactly as before; an owner + * with no frame read answers `undefined` and the caller derives the frame from a capture, which + * is how a Linux gesture resolves its coordinate frame today. + */ +function gestureBackendMembers( + params: InteractionRuntimeParams, + session: SessionState, + gestureContext: () => DaemonCommandContext, +): Pick { + const gestures = params.gestures; + const pairedGestureViewport = params.pairedGestureViewport; + if (!gestures) { + return pairedGestureViewport + ? { resolveGestureViewport: async (): Promise => pairedGestureViewport } + : {}; + } + return { + resolveGestureViewport: async (): Promise => + pairedGestureViewport ?? (await gestures.gestureViewport?.(gestureContext())), performGesture: async (_context, plan): Promise => { + // ADR 0014 side-effect seam: the plan is built; expire the ref frame synchronously before + // executing so a later step cannot reuse it. expireRefFrame(session); - return toBackendActionResult( - await dispatchGesturePlan( - session.device, - plan, - params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - ), - ); + return toBackendActionResult(await gestures.performPlan(plan, gestureContext())); }, }; } diff --git a/src/daemon/request-router.ts b/src/daemon/request-router.ts index 96a281875..042211831 100644 --- a/src/daemon/request-router.ts +++ b/src/daemon/request-router.ts @@ -429,6 +429,13 @@ async function dispatchGenericForLockedScope(params: { const runtimeExecution = await resolveGenericRuntimeExecution({ req: lockedScope.req, session, + // `scroll` parses its distance/timing flags during admission, so the resolved context is + // needed before the dispatcher builds its own. + context: lockedScope.contextFromFlags( + lockedScope.req.flags, + session.appBundleId, + session.trace?.outPath, + ), inspectFacts: lockedScope.inspectFacts, bindDevice: lockedScope.bindDevice, }); diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts new file mode 100644 index 000000000..64aa03982 --- /dev/null +++ b/src/daemon/scroll-runtime.ts @@ -0,0 +1,238 @@ +import { + assertExclusiveScrollDistanceInputs, + honoredScrollDurationMs, + normalizeScrollDurationMs, + parseScrollDirection, + resolveScrollExecutionOptions, + type ResolvedScrollExecutionOptions, + type ScrollCommandOptions, + type ScrollDirection, +} from '@agent-device/contracts/interaction'; +import { + resolveScrollRuntimePlan, + type CaptureSnapshotInput, + type ScrollDirectionInput, + type ScrollRuntimeOperations, + type SnapshotResult, +} from '@agent-device/contracts/platform'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { + captureScrollEdgeState, + formatScrollEdgeMessage, + runScrollEdgePasses, + type ScrollEdge, + type ScrollEdgeState, +} from '../utils/scroll-edge-state.ts'; +import { withSuccessText } from '../utils/success-text.ts'; +import type { DaemonCommandContext } from './context.ts'; +import { errorResponse } from './handlers/response.ts'; +import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; +import { admitRuntimeOperations, type RuntimeAdmissionBindings } from './runtime-admission.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; + +type ScrollTarget = Readonly<{ + direction: ScrollDirection; + edge?: ScrollEdge; +}>; + +/** + * What the executor needs. `captureSnapshot` is present only for an edge scroll, whose use + * required it — an ordinary scroll binds without a capture and cannot reach one. + */ +type BoundScrollOperations = Readonly<{ + operations: Readonly<{ + scrollDirection: ScrollRuntimeOperations['scrollDirection']; + captureSnapshot?: (input: CaptureSnapshotInput) => Promise; + }>; +}>; + +/** `scroll bottom` scrolls down to the edge; `scroll top` scrolls up to it. */ +function parseScrollTarget(input: string): ScrollTarget { + if (input === 'bottom') return { direction: 'down', edge: 'bottom' }; + if (input === 'top') return { direction: 'up', edge: 'top' }; + return { direction: parseScrollDirection(input) }; +} + +function assertScrollCommandInputs( + amount: number | undefined, + pixels: number | undefined, + durationMs: number | undefined, +): void { + if (amount !== undefined && !Number.isFinite(amount)) { + throw new AppError('INVALID_ARGS', 'scroll amount must be a number'); + } + normalizeScrollDurationMs(durationMs); + assertExclusiveScrollDistanceInputs({ amount, pixels }); +} + +/** + * The one place `scroll` reaches a device (ADR 0019). Admission inspects the exact owner's + * `scrollDirection` fact — plus `captureSnapshot` for an edge scroll, which cannot verify hidden + * content without one — and binds once, before the dispatcher runs. + * + * The whole positional/flag parse happens here rather than inside the executor so an invalid + * `scroll` is rejected exactly where the retired leaf rejected it: before any device work. + */ +export async function resolveBoundScrollRuntime( + params: { + device: DeviceInfo; + positionals: readonly string[]; + context: DaemonCommandContext; + } & RuntimeAdmissionBindings, +): Promise { + const directionInput = params.positionals[0]; + const amount = params.positionals[1] ? Number(params.positionals[1]) : undefined; + const pixels = params.context.pixels; + const durationMs = params.context.durationMs; + if (!directionInput) throw new AppError('INVALID_ARGS', 'scroll requires direction'); + assertScrollCommandInputs(amount, pixels, durationMs); + + const target = parseScrollTarget(directionInput); + const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); + const plan = resolveScrollRuntimePlan({ edge: target.edge !== undefined }); + const edge = target.edge; + const admitted = await admitRuntimeOperations({ + command: 'scroll', + device: params.device, + required: plan.use.required, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + // The retired leaf refused an unsupported edge scroll by naming what the edge needs, so the + // capture requirement keeps saying so rather than collapsing into "scroll is not supported". + ...(edge === undefined + ? {} + : { unavailableResponse: (unavailable) => scrollEdgeUnsupported(edge, unavailable.hint) }), + }); + if (admitted.type === 'response') return { ok: false, response: admitted.response }; + // One bind, with the exactly-typed use the parsed target selected (ADR 0019 §9). The branches + // exist so each `use` keeps its literal type; only one of them ever runs. + const runtime: BoundScrollOperations = + plan.kind === 'edge' + ? await admitted.bind(params.device, plan.use) + : await admitted.bind(params.device, plan.use); + return { + ok: true, + execute: async ({ dispatchContext }) => + await executeBoundScroll(runtime, target, options, { amount, pixels }, dispatchContext), + }; +} + +function scrollEdgeUnsupported(edge: ScrollEdge, hint: string | undefined) { + return errorResponse( + 'UNSUPPORTED_OPERATION', + `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, + undefined, + hint === undefined ? undefined : { hint }, + ); +} + +/** + * The ONE place a bound `scrollDirection` executes (R43). An edge scroll repeats the pass while + * the verified state still changes; every other scroll is a single pass. + */ +async function executeBoundScroll( + runtime: BoundScrollOperations, + target: ScrollTarget, + options: ResolvedScrollExecutionOptions, + distance: Readonly<{ amount?: number; pixels?: number }>, + context: DaemonCommandContext, +): Promise> { + const scroll = async () => + await runtime.operations.scrollDirection(scrollInput(target.direction, options, context)); + const { interactionResult, completedPasses } = target.edge + ? await runEdgeScroll(runtime, target.edge, scroll, context) + : { interactionResult: (await scroll()) ?? {}, completedPasses: 1 }; + return withSuccessText( + buildScrollResult(target, options, completedPasses, interactionResult), + formatScrollEdgeMessage( + target.direction, + target.edge, + completedPasses, + distance.amount, + distance.pixels, + ), + ); +} + +async function runEdgeScroll( + runtime: BoundScrollOperations, + edge: ScrollEdge, + scroll: () => Promise | void>, + context: DaemonCommandContext, +): Promise<{ interactionResult: Record; completedPasses: number }> { + const edgeResult = await runScrollEdgePasses({ + edge, + captureState: async (scope) => await captureEdgeState(runtime, edge, scope, context), + scroll, + }); + return { + interactionResult: edgeResult.result ?? {}, + completedPasses: edgeResult.passes, + }; +} + +/** + * The edge verification read. Admission proved the capture, so a missing operation here is a + * contract bug rather than a refusal to degrade around (ADR 0019 §2). + */ +async function captureEdgeState( + runtime: BoundScrollOperations, + edge: ScrollEdge, + scope: string | undefined, + context: DaemonCommandContext, +): Promise { + const captureSnapshot = runtime.operations.captureSnapshot; + if (!captureSnapshot) { + throw new AppError( + 'COMMAND_FAILED', + `scroll ${edge} admitted a snapshot capture the bound runtime did not provide`, + { reason: 'runtime-operation-missing' }, + ); + } + return await captureScrollEdgeState({ + edge, + scope, + captureNodes: async (snapshotScope) => + ( + await captureSnapshot({ + options: { + ...(context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }), + scope: snapshotScope, + }, + execution: runtimeExecutionFromContext(context), + }) + ).nodes ?? [], + }); +} + +function buildScrollResult( + target: ScrollTarget, + options: ScrollCommandOptions, + completedPasses: number, + interactionResult: Record, +): Record { + const durationMs = honoredScrollDurationMs(interactionResult); + return { + direction: target.direction, + ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), + ...(options.amount !== undefined ? { amount: options.amount } : {}), + ...(options.pixels !== undefined ? { pixels: options.pixels } : {}), + ...(durationMs !== undefined ? { durationMs } : {}), + ...interactionResult, + }; +} + +/** The neutral intent one scroll carries, projected from a resolved command context. */ +function scrollInput( + direction: ScrollDirection, + options: ResolvedScrollExecutionOptions, + context: DaemonCommandContext, +): ScrollDirectionInput { + return { + direction, + options, + ...(context.appBundleId === undefined ? {} : { target: { appBundleId: context.appBundleId } }), + execution: runtimeExecutionFromContext(context), + }; +} diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index 5661eea4c..d85d376ee 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -51,6 +51,8 @@ describe('composed platform runtime gateway', () => { elementText: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, lifecycle: applicationLifecycleOperationFacts({ resolveOpenTarget: unavailable, @@ -129,6 +131,8 @@ describe('composed platform runtime gateway', () => { screenshot: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, elementText: unavailable, lifecycle: applicationLifecycleOperationFacts({ diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index 76eeaedcf..3269a3462 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -306,6 +306,8 @@ function unavailableProviderBinding( screenshot: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, elementText: unavailable, lifecycle: unavailableProviderLifecycleFacts(unavailable), @@ -327,6 +329,8 @@ function unavailableProviderFacts(runtime: ProviderDeviceRuntime, device: Device screenshot: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, elementText: unavailable, readiness: unavailable, diff --git a/src/platforms/apple/core/__tests__/interactions.test.ts b/src/platforms/apple/core/__tests__/interactions.test.ts index 9a05e78d0..a20344af0 100644 --- a/src/platforms/apple/core/__tests__/interactions.test.ts +++ b/src/platforms/apple/core/__tests__/interactions.test.ts @@ -6,7 +6,10 @@ import { iosRunnerOverrides, performGestureApple } from '../../interactions.ts'; import { runAppleRunnerCommand } from '../runner/runner-client.ts'; import { AppError } from '@agent-device/kernel/errors'; import { TEXT_ENTRY_ROUTES, type GesturePlan } from '@agent-device/contracts/interaction'; -import { requireGestureSupported } from '../../../../core/capabilities.ts'; +import { + gestureRefusalMessage, + PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, +} from '@agent-device/contracts/platform'; import { IOS_TEST_DEVICE, IOS_TEST_SIMULATOR, @@ -184,29 +187,18 @@ test('performGestureApple sends exact two-pointer pan samples through gesture', }); test('Apple admission and execution share the same multi-touch refusal', async () => { - let admissionError: AppError | undefined; - try { - requireGestureSupported( - { - intent: 'pan', - origin: { x: 100, y: 200 }, - delta: { x: 80, y: -40 }, - pointerCount: 2, - }, - IOS_TEST_DEVICE, - ); - } catch (error) { - if (error instanceof AppError) admissionError = error; - } - assert.ok(admissionError); + // Admission is now the Apple owner's `performMultiTouchGesturePlan` fact composed with the + // shared refusal wording (R42); execution keeps its defensive adapter check. The two must + // still say the same thing, which is what this pins. + const admissionMessage = gestureRefusalMessage(IOS_TEST_DEVICE, 'multi-touch', 'pan'); await assert.rejects( () => performGestureApple(IOS_TEST_DEVICE, {}, {}, twoFingerPanPlan()), (error: unknown) => error instanceof AppError && error.code === 'UNSUPPORTED_OPERATION' && - error.message === admissionError.message && - error.details?.hint === admissionError.details?.hint, + error.message === admissionMessage && + error.details?.hint === PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, ); assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); }); diff --git a/test/integration/smoke-tvos-platform-coverage.test.ts b/test/integration/smoke-tvos-platform-coverage.test.ts index af5066732..2fa74d593 100644 --- a/test/integration/smoke-tvos-platform-coverage.test.ts +++ b/test/integration/smoke-tvos-platform-coverage.test.ts @@ -5,11 +5,9 @@ import test from 'node:test'; import { TVOS_SIMULATOR } from '../../src/__tests__/test-utils/device-fixtures.ts'; import { PUBLIC_COMMANDS } from '../../src/command-catalog.ts'; -import { - isCommandSupportedOnDevice, - requireGestureSupported, -} from '../../src/core/capabilities.ts'; -import { AppError } from '@agent-device/kernel/errors'; +import { isCommandSupportedOnDevice } from '../../src/core/capabilities.ts'; +import { createPlatformRuntimeGateway } from '../../src/platform-runtime.ts'; +import { gestureRefusalMessage } from '@agent-device/contracts/platform'; import { TVOS_COVERAGE_GAP_ISSUE, TVOS_PLATFORM_COVERAGE, @@ -103,23 +101,26 @@ test('tvOS capability denials match the mechanical capability matrix', () => { assert.equal('admission' in audio ? audio.admission : undefined, 'host-dependent'); }); -test('tvOS gesture contract preserves the typed multi-touch denial', () => { +test('tvOS gesture contract preserves the typed multi-touch denial', async () => { assert.equal(TVOS_PLATFORM_COVERAGE[PUBLIC_COMMANDS.gesture].level, 'command-contract'); - assert.throws( - () => - requireGestureSupported( - { - intent: 'pan', - origin: { x: 100, y: 200 }, - delta: { x: 40, y: -20 }, - pointerCount: 2, - }, - TVOS_SIMULATOR, - ), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - /tvOS has no touch input/.test(String(error.details?.hint)), + // R42: the denial is now the Apple owner's own fact for the tvOS leaf, and the refusal a + // gesture reports is that fact's hint under the shared wording. + const facts = await createPlatformRuntimeGateway({ + resolveSessionArtifacts: () => ({ + outputPath: '/sessions/tvos/app.log', + pidPath: '/sessions/tvos/app-log.pid', + }), + sessionsDir: '/sessions', + }).inspectFacts(TVOS_SIMULATOR); + const multiTouch = facts.operations.performMultiTouchGesturePlan; + assert.equal(multiTouch.available, false); + assert.match( + String(multiTouch.available === false ? multiTouch.hint : ''), + /tvOS has no touch input/, + ); + assert.equal( + gestureRefusalMessage(TVOS_SIMULATOR, 'multi-touch', 'pan'), + 'gesture pan is not supported on tvOS', ); }); diff --git a/test/integration/smoke-web-platform-coverage.test.ts b/test/integration/smoke-web-platform-coverage.test.ts index 06f3078d4..721f4e544 100644 --- a/test/integration/smoke-web-platform-coverage.test.ts +++ b/test/integration/smoke-web-platform-coverage.test.ts @@ -40,8 +40,10 @@ test('web coverage exhaustively classifies the public catalog', () => { test('web coverage report has the expected classification counts', () => { assert.deepEqual(WEB_PLATFORM_COVERAGE_CLASSIFICATION_SUMMARY, { - capabilityDenial: 15, - contract: 12, + // R42/R44 moved `gesture` and `swipe` off the capability matrix onto the web owner's own + // facts, so their evidence is a contract claim rather than a mechanical denial. + capabilityDenial: 13, + contract: 14, gap: 15, live: 12, total: 54, diff --git a/test/integration/tvos-e2e/coverage-manifest.ts b/test/integration/tvos-e2e/coverage-manifest.ts index 0b2d6cb46..b5caae6da 100644 --- a/test/integration/tvos-e2e/coverage-manifest.ts +++ b/test/integration/tvos-e2e/coverage-manifest.ts @@ -176,7 +176,7 @@ export const TVOS_PLATFORM_COVERAGE = { 'the existing provider scenario maps Back to the tvOS Menu remote press', ), [C.gesture]: contract( - 'src/core/__tests__/gesture-capabilities.test.ts', + 'src/daemon/__tests__/gesture-admission-parity.test.ts', 'TV, spatial, watch, desktop, Linux, and web gesture policy stays explicit', 'the typed Apple gesture policy refuses tvOS multi-touch while preserving the narrower gesture contract', ), diff --git a/test/integration/web-e2e/coverage-manifest.ts b/test/integration/web-e2e/coverage-manifest.ts index 30e21e40a..92a012fa8 100644 --- a/test/integration/web-e2e/coverage-manifest.ts +++ b/test/integration/web-e2e/coverage-manifest.ts @@ -145,7 +145,13 @@ export const WEB_PLATFORM_COVERAGE = { [C.get]: live('get reads the ready marker text from the fixture'), [C.is]: live('is visible passes for the Submit order control'), [C.back]: denial('Web capability model rejects native back navigation'), - [C.gesture]: denial('Web capability model rejects touch gesture input'), + // R42/R44: the web refusal moved from the capability matrix to the web owner's own gesture + // facts, so the evidence is the cell test rather than a mechanical matrix denial. + [C.gesture]: contract( + 'packages/platform-web/src/runtime.test.ts', + 'admits web scrolling and refuses every gesture tier', + 'the web runtime owner declares every gesture tier unavailable', + ), [C.home]: denial('Web capability model rejects native Home navigation'), [C.tvRemote]: denial('Web capability model rejects TV remote input'), [C.orientation]: denial('Web capability model rejects native orientation changes'), @@ -154,7 +160,11 @@ export const WEB_PLATFORM_COVERAGE = { 'scroll by pixels', 'web scroll moves the provider-backed page by the requested pixels', ), - [C.swipe]: denial('Web capability model rejects touch swipe input'), + [C.swipe]: contract( + 'packages/platform-web/src/runtime.test.ts', + 'admits web scrolling and refuses every gesture tier', + 'swipe shares the gesture tiers the web runtime owner declares unavailable', + ), [C.focus]: contract( 'src/core/__tests__/web-interactor.test.ts', 'web interactor delegates first-slice operations to the scoped provider',