From f65590afdd6c04e1a5209c0dcc98c58fbf613c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 19:32:25 +0200 Subject: [PATCH 1/3] refactor(android): raw is the acquired tree; one presentation for dialog recovery; residues declared (#1832 C3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - C3: the three regular-projection pruners (invisible subtrees, stale application windows, covered same-window surfaces) move out of parseUiHierarchyTree into the projection as a non-mutating classification (collectAndroidHiddenNodes in ui-hierarchy-visibility.ts). --raw presents the acquired tree; interactive ⊆ regular ⊆ raw by construction. Hidden-content hints and the scope root are derived per projection over retained children, which is what the mutating pruners implied. Property-checked identical to main for regular/-i/depth/scope over 12,000 random tree × projection pairs; raw grew on 2,514/3,000 and never shrank. - Android blocking-dialog recovery routes through buildSnapshotState (the one presentation), which moves to src/daemon/snapshot-state.ts below the daemon-server type cycle; importers repointed, its tests mirror the module. - Freshness route signature drops role/selected (Android never carries them). - Residues declared at their sites and in CONTEXT.md; docs + CHANGELOG. - ui-hierarchy.ts split by question: node predicates (ui-hierarchy-node.ts), regular-projection visibility (ui-hierarchy-visibility.ts), scope (ui-hierarchy-scope.ts); 974 → 634 LOC. --- CHANGELOG.md | 1 + CONTEXT.md | 15 + .../AccessibilityTreeCapture.java | 4 + .../snapshothelper/AccessibilityTreeXml.java | 3 + src/daemon/__tests__/generic-settle.test.ts | 2 +- src/daemon/__tests__/snapshot-state.test.ts | 409 ++++++++++++++++++ src/daemon/android-snapshot-freshness.ts | 8 +- src/daemon/android-system-dialog.ts | 11 +- .../interaction-ios-tap-outcome-fixtures.ts | 2 +- .../__tests__/interaction-settle.test.ts | 2 +- .../__tests__/interaction-touch-fixtures.ts | 2 +- .../handlers/__tests__/interaction.test.ts | 2 +- .../__tests__/snapshot-capture.test.ts | 409 +----------------- src/daemon/handlers/snapshot-capture.ts | 91 +--- .../ios/transitions.test.ts | 2 +- src/daemon/snapshot-state.ts | 106 +++++ .../android/__tests__/snapshot.test.ts | 6 +- .../__tests__/ui-hierarchy-scope.test.ts | 2 +- .../android/__tests__/ui-hierarchy.test.ts | 88 ++-- .../android/snapshot-helper-capture.ts | 3 + src/platforms/android/ui-hierarchy-node.ts | 74 ++++ .../android/ui-hierarchy-visibility.ts | 308 +++++++++++++ src/platforms/android/ui-hierarchy.ts | 408 ++--------------- website/docs/docs/commands.md | 6 +- 24 files changed, 1052 insertions(+), 912 deletions(-) create mode 100644 src/daemon/__tests__/snapshot-state.test.ts create mode 100644 src/daemon/snapshot-state.ts create mode 100644 src/platforms/android/ui-hierarchy-node.ts create mode 100644 src/platforms/android/ui-hierarchy-visibility.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 133d81915e..60940f51ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - `--udid` with `--platform android` (and `--serial` with an Apple platform) now fails as the flag mistake it is — `INVALID_ARGS` naming the right flag — instead of reaching device resolution and answering `No Apple device with UDID emulator-5580` for an explicitly Android request. `--udid` addresses Apple devices, `--serial` addresses Android and HarmonyOS; matching pairs and requests that name no platform are unchanged. - Breaking (`--session-lock strip`): a device selector that names a different device than the bound session is no longer silently discarded. `strip` exists to drop redundant platform/scope selectors; when it also dropped `--udid`/`--serial`/`--device`, the command kept running against the *bound* device instead of the one the caller named — a wrong-device action that looks like a success. Such a request now fails with `INVALID_ARGS` under both `reject` and `strip`, and the error carries the two identities structurally (`requestedDevice`, `boundDevice`) plus a hint offering the two real recoveries: close the bound session if the requested device is intended, or remove the selector if the bound device is. The hint no longer suggests `--session-lock strip` for an identity conflict, since following that advice is what produced the wrong-device run. Scope-only stripping (`--platform`, `--target`, `--ios-simulator-device-set`, `--android-device-allowlist`) is unchanged. - iOS regular snapshots now apply one backend-neutral eligibility rule after every capture backend: a node survives when its accessibility type is interactive or it carries a non-empty label, identifier, or value. This removes the tree backend's extra "hittable non-Other" membership path and drops unlabeled decorative nodes consistently; labeled images, identifier-only nodes, and value-only nodes still survive. Raw snapshot membership is unchanged. +- Android `snapshot --raw` is now the acquired accessibility tree (#1832 C3): the three regular-projection pruners — nodes Android marks invisible, stale application windows, and covered same-window surfaces — no longer run at parse time, so `--raw` keeps everything the helper serialized (normalization only) and pruned content is recoverable for diagnosis. Regular and `-i` output is unchanged (property-checked identical to the previous implementation over 12,000 random tree × projection pairs, hidden-content hints included). Also: Android blocking-dialog recovery now reads the same daemon presentation an agent's `snapshot` sees instead of a hand-rolled subset; the Android freshness route signature no longer keys on `role`/`selected`, fields the Android backend never carries; and the Android helper's declared fidelity residues (no `checked`/`checkable`/`selected`/`long-clickable`, 5000-node cap before scoping, API-level cache-reset divergence) are recorded in `CONTEXT.md`. - `agent-device mcp` now carries its own usage guidance, so MCP-only clients (Codex CLI, Cursor, custom agents) no longer depend on a separately installed skill (#1833). The handshake `instructions` — returned by both `server/discover` and, newly, the legacy `initialize` — is a compact (< 2 KB, the Claude Code truncation limit) workflow card: start with `open {app, foreground: true}` instead of probing, act with `settle: true` and continue from the diff, verify with `wait`/`is`/`get`/`find`, copy `@refs` byte-for-byte, recover from sparse/AX-unavailable, follow error hints, `close`. A new MCP-only `help` tool serves the full guides on demand: no `topic` returns the CLI's decision card; `topic` returns `agent-device help ` verbatim (workflow, gestures, scripting, tv, macos, web, remote, debugging, …, or any tool name for its complete flag reference), prefixed with the one-line CLI→tool-property mapping. `help` is router-owned rather than a command descriptor, so it appears in `tools/list` only — not in the CLI, Node client, or `batch` — and its description tells the model it is not a startup step. Legacy `initialize` gains the optional `instructions` field; no other legacy field changes. - Android `snapshot --scope` (and every selector command's `--scope`, e.g. `press "Save" --scope Panel`) now resolves scope exactly once, inside the Android projection, under the shared scope specification: the scope root is the first node **in document order** whose label, value, or identifier contains the scope text (case-insensitive) **and whose subtree still has content in the projection you asked for**, the result is that subtree re-rooted at depth 0, and no match returns an empty snapshot (#1832). That second clause is what makes `snapshot -i --scope panel` return the button inside a structural container `-i` drops, and stops a decorative heading that happens to match from emptying the snapshot. Before, Android ran two passes with contradictory rules — a breadth-first platform match that fell back to the full tree on a miss, then the daemon's document-order pass — so a shallower later container could win over an earlier match, and an interaction capture whose scope reached only the daemon layer was silently unscoped. `--depth` under `--scope` counts from the scope root, filtering the depths the response prints (a node shown at depth 0 is never hidden by `--depth 0`), and ancestor context above the scope root (a clickable row, a list) still shapes `-i` membership inside it. The rule is pinned by `contracts/fixtures/snapshot-scope-policy.json`, the same golden table the iOS runner will consume (#1797). Also new: `androidSnapshot.occlusionScanUnavailable: true` discloses an API 23 capture, where the helper cannot report `drawing-order` and covered same-window surfaces are therefore not pruned. - New `hover ` command for `--platform web` (#1783). It moves the pointer over the target without pressing, so hover-gated UI — a message row's `...` toolbar, a menu that opens on pointer enter — becomes reachable through agent-device the way it already was through the underlying `agent-browser` backend (`mouse move`). It is a member of the targeted-touch family: same `@ref`/selector/coordinate targeting, occlusion and off-screen guards, and `--settle` (the settled diff carries the revealed controls with fresh refs, e.g. `+ @e4 [button] "Delete"`), but no `--verify`, since hover reveals rather than activates. `hover @ref` publishes as a portable selector line in recorded scripts, and the Node client exposes `interactions.hover`. Hover is a pointer state that touch platforms do not have, so `capabilities` advertises it on web only and iOS/Android/Linux reject it during admission with `UNSUPPORTED_OPERATION` and a hint naming `--platform web`; `longpress` remains the mobile hold-gesture verb. diff --git a/CONTEXT.md b/CONTEXT.md index 256807fa0d..1ef5d8c979 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -241,6 +241,21 @@ task touches: typed `WebView` root and WebKit's `Other -> StaticText` wrapper pairs. It keeps raw diagnostics unchanged, presents ordinary wrapper text as `StaticText`, and presents wrappers carrying WebKit's numeric HTML heading level as `Heading`. +- Android snapshot projections (#1832 C3): the parsed helper tree is the acquired tree and is never + mutated. `--raw` presents it whole (normalization only). The regular projection additionally hides + what `collectAndroidHiddenNodes` classifies — nodes Android marks invisible to users, stale + application windows, and same-window surfaces a higher drawing-order sibling covers — and the + interactive projection applies membership on top. `interactive ⊆ regular ⊆ raw` holds by + construction; hidden-content hints and the scope root are derived per projection. +- Android declared residues (#1832): fidelity limits of the helper acquisition that no presentation + can repair, disclosed rather than papered over. (1) API 23 helper trees carry no `drawing-order`, + so the covered-surface rule cannot run — `androidSnapshot.occlusionScanUnavailable`. (2) The + helper serializes no `checked`/`checkable`/`selected`/`long-clickable`, so toggle and selection + state is invisible and nothing on Android may key on those fields (the freshness route signature + does not). (3) The helper caps at 5000 nodes before any projection, so a scoped target past the + cap vanishes with only the generic `truncated`. (4) Accessibility-cache reset differs by API level + (`clearCache()` on 34+, `setServiceInfo` re-apply below), an undisclosed acquisition-freshness + difference. - AX-unavailable target invalidation: iOS/macOS runner behavior where a root accessibility snapshot failure such as `kAXErrorIllegalArgument` marks the cached `XCUIApplication` target handle suspect. The runner fails closed for degraded interactive snapshots, clears the cached target, and lets the diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java index 9f8898b38b..2904fd6732 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java @@ -69,6 +69,10 @@ private static void clearAccessibilityCache(UiAutomation automation) { // destination inside one AndroidComposeView. A persistent helper session reuses a single // UiAutomation connection across captures, and its per-connection accessibility node cache is // not always invalidated by such a swap. Drop the cache before each traversal. + // + // Declared residue (agent-device #1832): the reset mechanism differs by API level — the public + // clearCache() on 34+, a setServiceInfo() re-apply below — and the two are not known to be + // equivalent. This is an acquisition-freshness difference the payload does not disclose. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { try { automation.clearCache(); diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 23cd09ddf6..abfe311c0e 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -121,6 +121,9 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean } } + // Declared residue (agent-device #1832): checked / checkable / selected / long-clickable are not + // serialized, so toggle and selection state is invisible to agents. Adding them is a helper + // protocol change (new attributes + host parser + fields on the wire node), tracked there. private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder())); diff --git a/src/daemon/__tests__/generic-settle.test.ts b/src/daemon/__tests__/generic-settle.test.ts index 1e689f3b99..2512d6d27b 100644 --- a/src/daemon/__tests__/generic-settle.test.ts +++ b/src/daemon/__tests__/generic-settle.test.ts @@ -7,7 +7,7 @@ import { activateCompleteRefFrame } from '../ref-frame.ts'; import { setSessionSnapshot } from '../session-snapshot.ts'; import type { SessionStore } from '../session-store.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; -import { buildSnapshotState } from '../handlers/snapshot-capture.ts'; +import { buildSnapshotState } from '../snapshot-state.ts'; // #1638 `--settle` on the GENERIC daemon route (scroll/back): the settled diff, // its refs, and the ref-frame/generation dance are the same contract the touch diff --git a/src/daemon/__tests__/snapshot-state.test.ts b/src/daemon/__tests__/snapshot-state.test.ts new file mode 100644 index 0000000000..ee62cdf64d --- /dev/null +++ b/src/daemon/__tests__/snapshot-state.test.ts @@ -0,0 +1,409 @@ +import { expect, test } from 'vitest'; +import { buildSnapshotState } from '../snapshot-state.ts'; + +test('buildSnapshotState handles undefined nodes gracefully', () => { + const state = buildSnapshotState({ nodes: undefined, truncated: undefined }, undefined); + expect(state.nodes).toEqual([]); + expect(state.truncated).toBeUndefined(); + expect(state.createdAt).toBeGreaterThan(0); +}); + +test('buildSnapshotState handles completely empty data object', () => { + const state = buildSnapshotState({}, undefined); + expect(state.nodes).toEqual([]); + expect(state.truncated).toBeUndefined(); +}); + +test('buildSnapshotState carries structured snapshot quality verdicts', () => { + const state = buildSnapshotState( + { + nodes: [{ index: 0, type: 'Application' }], + backend: 'xctest', + quality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }, + { snapshotInteractiveOnly: true }, + ); + + expect(state.snapshotQuality).toMatchObject({ + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }); +}); + +test('buildSnapshotState handles nodes with missing fields', () => { + const state = buildSnapshotState( + { + nodes: [ + { index: 0 } as any, + { index: 1, depth: undefined, type: undefined, label: undefined } as any, + ], + truncated: false, + backend: 'android', + }, + undefined, + ); + expect(state.nodes).toHaveLength(2); + expect(state.nodes[0]?.ref).toBeTruthy(); + expect(state.nodes[1]?.ref).toBeTruthy(); +}); + +test('buildSnapshotState marks comparisonSafe false for filtered Android snapshots', () => { + const nodes = [{ index: 0, depth: 0, type: 'android.widget.TextView', label: 'A' }]; + + const interactiveOnly = buildSnapshotState( + { nodes, backend: 'android' }, + { snapshotInteractiveOnly: true }, + ); + expect(interactiveOnly.comparisonSafe).toBe(false); + + const withDepth = buildSnapshotState({ nodes, backend: 'android' }, { snapshotDepth: 2 }); + expect(withDepth.comparisonSafe).toBe(false); + + const withScope = buildSnapshotState({ nodes, backend: 'android' }, { snapshotScope: 'Header' }); + expect(withScope.comparisonSafe).toBe(false); + + const unfiltered = buildSnapshotState({ nodes, backend: 'android' }, {}); + expect(unfiltered.comparisonSafe).toBe(true); +}); + +test('buildSnapshotState applies iOS interactive presentation for xctest snapshots', () => { + const rowRect = { x: 16, y: 293, width: 370, height: 52 }; + const state = buildSnapshotState( + { + nodes: [ + { index: 0, depth: 0, type: 'Application', label: 'Settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, + { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, + { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, + ], + backend: 'xctest', + }, + { snapshotInteractiveOnly: true }, + ); + + expect(state.nodes.map((node) => [node.type, node.label, node.parentIndex])).toEqual([ + ['Application', 'Settings', undefined], + ['CollectionView', undefined, 0], + ['Cell', 'General', 1], + ]); +}); + +test('buildSnapshotState marks content covered by floating overlays as visible but blocked', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'TabBar', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ], + backend: 'xctest', + }, + undefined, + ); + + const covered = state.nodes.find((node) => node.label === 'Save draft'); + expect(covered).toMatchObject({ + label: 'Save draft', + hittable: false, + interactionBlocked: 'covered', + presentationHints: ['covered'], + }); + expect(state.nodes.some((node) => node.type === 'TabBar')).toBe(true); +}); + +test('buildSnapshotState marks Android app content covered by IME overlays as blocked', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + bundleId: 'org.example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Push Article', + bundleId: 'org.example', + rect: { x: 40, y: 600, width: 180, height: 56 }, + hittable: true, + }, + { + index: 2, + depth: 1, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 0, y: 400, width: 390, height: 444 }, + }, + ], + backend: 'android', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Push Article')).toMatchObject({ + hittable: false, + interactionBlocked: 'covered', + presentationHints: ['covered'], + }); +}); + +test('buildSnapshotState treats large Android IME subtrees as one overlay root', () => { + const imeChildren = Array.from({ length: 2000 }, (_, offset) => ({ + index: offset + 3, + depth: 2, + parentIndex: 2, + type: 'android.widget.TextView', + label: `Keyboard suggestion ${offset}`, + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: offset % 300, y: 500 + (offset % 200), width: 80, height: 32 }, + })); + + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + bundleId: 'org.example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Covered action', + bundleId: 'org.example', + rect: { x: 40, y: 620, width: 180, height: 56 }, + hittable: true, + }, + { + index: 2, + depth: 1, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 0, y: 400, width: 390, height: 444 }, + }, + ...imeChildren, + ], + backend: 'android', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Covered action')).toMatchObject({ + hittable: false, + interactionBlocked: 'covered', + }); +}); + +test('buildSnapshotState does not treat later generic hittable containers as covers', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Visible action', + rect: { x: 40, y: 100, width: 160, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'CollectionView', + label: 'Content list', + rect: { x: 0, y: 80, width: 390, height: 600 }, + hittable: true, + }, + ], + backend: 'xctest', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Visible action')).toMatchObject({ + hittable: true, + }); + expect( + state.nodes.find((node) => node.label === 'Visible action')?.interactionBlocked, + ).toBeUndefined(); +}); + +test('buildSnapshotState does not let covered overlays cover earlier targets', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Top action', + rect: { x: 20, y: 30, width: 120, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Middle action', + rect: { x: 20, y: 170, width: 120, height: 44 }, + hittable: true, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'ToolBar', + rect: { x: 0, y: 0, width: 390, height: 300 }, + hittable: true, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'Sheet', + rect: { x: 0, y: 120, width: 390, height: 724 }, + hittable: true, + }, + ], + backend: 'xctest', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Middle action')).toMatchObject({ + interactionBlocked: 'covered', + }); + expect(state.nodes.find((node) => node.type === 'ToolBar')).toMatchObject({ + interactionBlocked: 'covered', + }); + expect(state.nodes.find((node) => node.label === 'Top action')).toMatchObject({ + hittable: true, + }); + expect( + state.nodes.find((node) => node.label === 'Top action')?.interactionBlocked, + ).toBeUndefined(); +}); + +test('buildSnapshotState leaves raw snapshot hittability untouched', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'TabBar', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ], + backend: 'xctest', + }, + { snapshotRaw: true }, + ); + + expect(state.nodes.find((node) => node.label === 'Save draft')).toMatchObject({ + hittable: true, + }); + expect( + state.nodes.find((node) => node.label === 'Save draft')?.interactionBlocked, + ).toBeUndefined(); +}); + +test('buildSnapshotState returns empty nodes when scoped snapshot has no label match', () => { + const nodes = [ + { index: 0, depth: 0, type: 'Window', label: 'Root' }, + { index: 1, depth: 1, type: 'Button', label: 'Search' }, + ]; + + const state = buildSnapshotState( + { nodes, backend: 'xctest' }, + { snapshotScope: 'zzzz-no-match-token' }, + ); + + expect(state.nodes).toEqual([]); +}); + +test('buildSnapshotState preserves macOS helper scope behavior', () => { + const state = buildSnapshotState( + { + nodes: [ + { index: 0, depth: 0, type: 'Window', label: 'Desktop surface' }, + { index: 1, depth: 1, parentIndex: 0, type: 'Button', label: 'Target' }, + ], + backend: 'macos-helper', + }, + { snapshotScope: 'missing scope' }, + ); + + expect(state.nodes.map((node) => node.label)).toEqual(['Desktop surface', 'Target']); + expect(state.nodes.every((node) => node.ref)).toBe(true); +}); diff --git a/src/daemon/android-snapshot-freshness.ts b/src/daemon/android-snapshot-freshness.ts index 9770f286ab..b961983a39 100644 --- a/src/daemon/android-snapshot-freshness.ts +++ b/src/daemon/android-snapshot-freshness.ts @@ -80,17 +80,21 @@ export function isNavigationSensitiveAction(command: string): boolean { return command === 'press' || command === 'click' || command === 'back' || command === 'open'; } +/** + * Route signature of an Android snapshot, from the fields the Android backend actually carries. + * The helper serializes no `role`, `selected`, `checked` or `long-clickable` (declared residue, + * #1832), so a signature keying on them would compare constants and claim discrimination it does + * not have. + */ export function buildSnapshotSignatures(nodes: SnapshotState['nodes']): string[] { return nodes.map((node) => [ node.depth ?? 0, node.type ?? '', - node.role ?? '', node.label ?? '', node.value ?? '', node.identifier ?? '', node.enabled === false ? 'disabled' : 'enabled', - node.selected === true ? 'selected' : 'unselected', node.hittable === true ? 'hittable' : 'not-hittable', ].join('|'), ); diff --git a/src/daemon/android-system-dialog.ts b/src/daemon/android-system-dialog.ts index 77ce1003b0..7685d75ffc 100644 --- a/src/daemon/android-system-dialog.ts +++ b/src/daemon/android-system-dialog.ts @@ -8,9 +8,9 @@ import { snapshotAndroid } from '../platforms/android/snapshot.ts'; import { runAndroidAdb } from '../platforms/android/adb.ts'; import { emitDiagnostic } from '../utils/diagnostics.ts'; import { AppError } from '@agent-device/kernel/errors'; -import { centerOfRect, attachRefs, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { centerOfRect, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { sleep } from '../utils/timeouts.ts'; -import { pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; +import { buildSnapshotState } from './snapshot-state.ts'; import { expireRefFrame } from './ref-frame.ts'; import type { SessionState } from './types.ts'; @@ -261,11 +261,16 @@ function formatAndroidBlockingDialogFocus(focus: AndroidBlockingDialogFocus): st return focus.package ? `${focus.focusedWindow} (package ${focus.package})` : focus.focusedWindow; } +/** + * Blocking-dialog detection reads the SAME presentation an agent's `snapshot` would see: one + * daemon presentation (normalize, group prune, occlusion annotation, refs) rather than a hand-rolled + * subset that could disagree with it about which button is on top (#1832, the #1784 pattern). + */ async function readAndroidSnapshotNodes(session: SessionState): Promise { const rawSnapshot = await snapshotAndroid(session.device, { interactiveOnly: false, }); - return attachRefs(pruneGroupNodes(rawSnapshot.nodes)); + return buildSnapshotState({ nodes: rawSnapshot.nodes, backend: 'android' }, undefined).nodes; } async function tapAndroidDialogButton( diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts index 2db0bcd4c6..3410bce824 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts +++ b/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts @@ -1,5 +1,5 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { buildSnapshotState } from '../snapshot-capture.ts'; +import { buildSnapshotState } from '../../snapshot-state.ts'; export const profileNodes: RawSnapshotNode[] = [ { diff --git a/src/daemon/handlers/__tests__/interaction-settle.test.ts b/src/daemon/handlers/__tests__/interaction-settle.test.ts index 895022c203..e0909df8a3 100644 --- a/src/daemon/handlers/__tests__/interaction-settle.test.ts +++ b/src/daemon/handlers/__tests__/interaction-settle.test.ts @@ -4,7 +4,7 @@ import { handleInteractionCommands } from '../interaction.ts'; import type { SessionStore } from '../../session-store.ts'; import type { SessionState } from '../../types.ts'; import type { SnapshotBackend } from '@agent-device/kernel/snapshot'; -import { buildSnapshotState } from '../snapshot-capture.ts'; +import { buildSnapshotState } from '../../snapshot-state.ts'; import { setSessionSnapshot } from '../../session-snapshot.ts'; import { activateCompleteRefFrame } from '../../ref-frame.ts'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; diff --git a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts b/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts index 8f29cd9f8d..a43d03118a 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts @@ -10,7 +10,7 @@ import { activateCompleteRefFrame } from '../../ref-frame.ts'; import type { SessionStore } from '../../session-store.ts'; import type { SessionState } from '../../types.ts'; import { handleInteractionCommands } from '../interaction.ts'; -import { buildSnapshotState } from '../snapshot-capture.ts'; +import { buildSnapshotState } from '../../snapshot-state.ts'; /** * Shared factories for the interaction touch handler tests. Named pure diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index c9e58b056b..cf7402d109 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -10,7 +10,7 @@ import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts import { expireRefFrame } from '../../ref-frame.ts'; import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; import { handleInteractionCommands } from '../interaction.ts'; -import { buildSnapshotState } from '../snapshot-capture.ts'; +import { buildSnapshotState } from '../../snapshot-state.ts'; import { contextFromFlags, makeSession, diff --git a/src/daemon/handlers/__tests__/snapshot-capture.test.ts b/src/daemon/handlers/__tests__/snapshot-capture.test.ts index a9c59fc22b..840239f59f 100644 --- a/src/daemon/handlers/__tests__/snapshot-capture.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-capture.test.ts @@ -1,5 +1,5 @@ import { expect, test, vi } from 'vitest'; -import { buildSnapshotState, captureSnapshotData } from '../snapshot-capture.ts'; +import { captureSnapshotData } from '../snapshot-capture.ts'; import { buildSnapshotVisibility } from '../../../snapshot/snapshot-visibility.ts'; import { ANDROID_EMULATOR, @@ -10,100 +10,6 @@ import { const captureSnapshotWithInteractor = vi.hoisted(() => vi.fn()); vi.mock('../snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor })); -test('buildSnapshotState handles undefined nodes gracefully', () => { - const state = buildSnapshotState({ nodes: undefined, truncated: undefined }, undefined); - expect(state.nodes).toEqual([]); - expect(state.truncated).toBeUndefined(); - expect(state.createdAt).toBeGreaterThan(0); -}); - -test('buildSnapshotState handles completely empty data object', () => { - const state = buildSnapshotState({}, undefined); - expect(state.nodes).toEqual([]); - expect(state.truncated).toBeUndefined(); -}); - -test('buildSnapshotState carries structured snapshot quality verdicts', () => { - const state = buildSnapshotState( - { - nodes: [{ index: 0, type: 'Application' }], - backend: 'xctest', - quality: { - state: 'sparse', - backend: 'private-ax', - reason: 'sparse tree', - reasonCode: 'sparse-tree', - }, - }, - { snapshotInteractiveOnly: true }, - ); - - expect(state.snapshotQuality).toMatchObject({ - state: 'sparse', - backend: 'private-ax', - reason: 'sparse tree', - reasonCode: 'sparse-tree', - }); -}); - -test('buildSnapshotState handles nodes with missing fields', () => { - const state = buildSnapshotState( - { - nodes: [ - { index: 0 } as any, - { index: 1, depth: undefined, type: undefined, label: undefined } as any, - ], - truncated: false, - backend: 'android', - }, - undefined, - ); - expect(state.nodes).toHaveLength(2); - expect(state.nodes[0]?.ref).toBeTruthy(); - expect(state.nodes[1]?.ref).toBeTruthy(); -}); - -test('buildSnapshotState marks comparisonSafe false for filtered Android snapshots', () => { - const nodes = [{ index: 0, depth: 0, type: 'android.widget.TextView', label: 'A' }]; - - const interactiveOnly = buildSnapshotState( - { nodes, backend: 'android' }, - { snapshotInteractiveOnly: true }, - ); - expect(interactiveOnly.comparisonSafe).toBe(false); - - const withDepth = buildSnapshotState({ nodes, backend: 'android' }, { snapshotDepth: 2 }); - expect(withDepth.comparisonSafe).toBe(false); - - const withScope = buildSnapshotState({ nodes, backend: 'android' }, { snapshotScope: 'Header' }); - expect(withScope.comparisonSafe).toBe(false); - - const unfiltered = buildSnapshotState({ nodes, backend: 'android' }, {}); - expect(unfiltered.comparisonSafe).toBe(true); -}); - -test('buildSnapshotState applies iOS interactive presentation for xctest snapshots', () => { - const rowRect = { x: 16, y: 293, width: 370, height: 52 }; - const state = buildSnapshotState( - { - nodes: [ - { index: 0, depth: 0, type: 'Application', label: 'Settings' }, - { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, - { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, - { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, - ], - backend: 'xctest', - }, - { snapshotInteractiveOnly: true }, - ); - - expect(state.nodes.map((node) => [node.type, node.label, node.parentIndex])).toEqual([ - ['Application', 'Settings', undefined], - ['CollectionView', undefined, 0], - ['Cell', 'General', 1], - ]); -}); - test('iOS interactive capture does not send local presentation scope to XCTest', async () => { captureSnapshotWithInteractor.mockClear(); captureSnapshotWithInteractor.mockResolvedValueOnce({ nodes: [], backend: 'xctest' }); @@ -151,319 +57,6 @@ test('snapshot capture preserves backend scope outside iOS interactive presentat ]); }); -test('buildSnapshotState marks content covered by floating overlays as visible but blocked', () => { - const state = buildSnapshotState( - { - nodes: [ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Save draft', - rect: { x: 16, y: 790, width: 140, height: 44 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'TabBar', - rect: { x: 0, y: 760, width: 390, height: 84 }, - hittable: true, - }, - ], - backend: 'xctest', - }, - undefined, - ); - - const covered = state.nodes.find((node) => node.label === 'Save draft'); - expect(covered).toMatchObject({ - label: 'Save draft', - hittable: false, - interactionBlocked: 'covered', - presentationHints: ['covered'], - }); - expect(state.nodes.some((node) => node.type === 'TabBar')).toBe(true); -}); - -test('buildSnapshotState marks Android app content covered by IME overlays as blocked', () => { - const state = buildSnapshotState( - { - nodes: [ - { - index: 0, - depth: 0, - type: 'android.widget.FrameLayout', - bundleId: 'org.example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'android.widget.Button', - label: 'Push Article', - bundleId: 'org.example', - rect: { x: 40, y: 600, width: 180, height: 56 }, - hittable: true, - }, - { - index: 2, - depth: 1, - type: 'android.widget.FrameLayout', - bundleId: 'com.google.android.inputmethod.latin', - rect: { x: 0, y: 400, width: 390, height: 444 }, - }, - ], - backend: 'android', - }, - undefined, - ); - - expect(state.nodes.find((node) => node.label === 'Push Article')).toMatchObject({ - hittable: false, - interactionBlocked: 'covered', - presentationHints: ['covered'], - }); -}); - -test('buildSnapshotState treats large Android IME subtrees as one overlay root', () => { - const imeChildren = Array.from({ length: 2000 }, (_, offset) => ({ - index: offset + 3, - depth: 2, - parentIndex: 2, - type: 'android.widget.TextView', - label: `Keyboard suggestion ${offset}`, - bundleId: 'com.google.android.inputmethod.latin', - rect: { x: offset % 300, y: 500 + (offset % 200), width: 80, height: 32 }, - })); - - const state = buildSnapshotState( - { - nodes: [ - { - index: 0, - depth: 0, - type: 'android.widget.FrameLayout', - bundleId: 'org.example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'android.widget.Button', - label: 'Covered action', - bundleId: 'org.example', - rect: { x: 40, y: 620, width: 180, height: 56 }, - hittable: true, - }, - { - index: 2, - depth: 1, - type: 'android.widget.FrameLayout', - bundleId: 'com.google.android.inputmethod.latin', - rect: { x: 0, y: 400, width: 390, height: 444 }, - }, - ...imeChildren, - ], - backend: 'android', - }, - undefined, - ); - - expect(state.nodes.find((node) => node.label === 'Covered action')).toMatchObject({ - hittable: false, - interactionBlocked: 'covered', - }); -}); - -test('buildSnapshotState does not treat later generic hittable containers as covers', () => { - const state = buildSnapshotState( - { - nodes: [ - { - index: 0, - depth: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Visible action', - rect: { x: 40, y: 100, width: 160, height: 44 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'CollectionView', - label: 'Content list', - rect: { x: 0, y: 80, width: 390, height: 600 }, - hittable: true, - }, - ], - backend: 'xctest', - }, - undefined, - ); - - expect(state.nodes.find((node) => node.label === 'Visible action')).toMatchObject({ - hittable: true, - }); - expect( - state.nodes.find((node) => node.label === 'Visible action')?.interactionBlocked, - ).toBeUndefined(); -}); - -test('buildSnapshotState does not let covered overlays cover earlier targets', () => { - const state = buildSnapshotState( - { - nodes: [ - { - index: 0, - depth: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Top action', - rect: { x: 20, y: 30, width: 120, height: 44 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Middle action', - rect: { x: 20, y: 170, width: 120, height: 44 }, - hittable: true, - }, - { - index: 3, - depth: 1, - parentIndex: 0, - type: 'ToolBar', - rect: { x: 0, y: 0, width: 390, height: 300 }, - hittable: true, - }, - { - index: 4, - depth: 1, - parentIndex: 0, - type: 'Sheet', - rect: { x: 0, y: 120, width: 390, height: 724 }, - hittable: true, - }, - ], - backend: 'xctest', - }, - undefined, - ); - - expect(state.nodes.find((node) => node.label === 'Middle action')).toMatchObject({ - interactionBlocked: 'covered', - }); - expect(state.nodes.find((node) => node.type === 'ToolBar')).toMatchObject({ - interactionBlocked: 'covered', - }); - expect(state.nodes.find((node) => node.label === 'Top action')).toMatchObject({ - hittable: true, - }); - expect( - state.nodes.find((node) => node.label === 'Top action')?.interactionBlocked, - ).toBeUndefined(); -}); - -test('buildSnapshotState leaves raw snapshot hittability untouched', () => { - const state = buildSnapshotState( - { - nodes: [ - { - index: 0, - depth: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Save draft', - rect: { x: 16, y: 790, width: 140, height: 44 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'TabBar', - rect: { x: 0, y: 760, width: 390, height: 84 }, - hittable: true, - }, - ], - backend: 'xctest', - }, - { snapshotRaw: true }, - ); - - expect(state.nodes.find((node) => node.label === 'Save draft')).toMatchObject({ - hittable: true, - }); - expect( - state.nodes.find((node) => node.label === 'Save draft')?.interactionBlocked, - ).toBeUndefined(); -}); - -test('buildSnapshotState returns empty nodes when scoped snapshot has no label match', () => { - const nodes = [ - { index: 0, depth: 0, type: 'Window', label: 'Root' }, - { index: 1, depth: 1, type: 'Button', label: 'Search' }, - ]; - - const state = buildSnapshotState( - { nodes, backend: 'xctest' }, - { snapshotScope: 'zzzz-no-match-token' }, - ); - - expect(state.nodes).toEqual([]); -}); - -test('buildSnapshotState preserves macOS helper scope behavior', () => { - const state = buildSnapshotState( - { - nodes: [ - { index: 0, depth: 0, type: 'Window', label: 'Desktop surface' }, - { index: 1, depth: 1, parentIndex: 0, type: 'Button', label: 'Target' }, - ], - backend: 'macos-helper', - }, - { snapshotScope: 'missing scope' }, - ); - - expect(state.nodes.map((node) => node.label)).toEqual(['Desktop surface', 'Target']); - expect(state.nodes.every((node) => node.ref)).toBe(true); -}); - test('buildSnapshotVisibility returns non-partial for empty node list', () => { const vis = buildSnapshotVisibility({ nodes: [], backend: 'android' }); expect(vis.partial).toBe(false); diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index feb00fd47b..21561d2a95 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -5,29 +5,22 @@ import { type SnapshotCaptureAnnotations, } from '@agent-device/contracts/capture'; import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; -import { isAndroidInputMethodNode } from '@agent-device/contracts/platform'; import { - attachRefs, - buildSnapshotPresentationKey, findNodeByRef, normalizeRef, - snapshotPresentationOptionsFromFlags, type RawSnapshotNode, type SnapshotBackend, type SnapshotState, } from '@agent-device/kernel/snapshot'; -import { annotateCoveredSnapshotNodes } from '../../snapshot/snapshot-occlusion.ts'; import { resolveRefLabel } from '../../core/snapshot-node-lookup.ts'; -import { scopeSnapshotNodes } from '../../snapshot/snapshot-desktop-surface.ts'; import { captureSnapshotWithInteractor } from './snapshot-interactor-capture.ts'; -import { normalizeSnapshotTree, pruneGroupNodes } from '../../core/snapshot-tree-ingestion.ts'; +import { buildSnapshotState } from '../snapshot-state.ts'; import { clearAndroidSnapshotFreshness, type AndroidFreshnessMode, } from '../android-snapshot-freshness.ts'; import { contextFromFlags } from '../context.ts'; import { resolveDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts'; -import { presentIosInteractiveSnapshot } from '../snapshot-presentation/ios/index.ts'; import type { SessionState } from '../types.ts'; import { errorResponse, type DaemonFailureResponse } from './response.ts'; @@ -164,51 +157,6 @@ function resolveSnapshotStateFlags( }; } -export function buildSnapshotState( - data: { - nodes?: RawSnapshotNode[]; - truncated?: boolean; - backend?: SnapshotBackend; - quality?: unknown; - }, - flags: - | (Pick & - Partial>) - | undefined, -): SnapshotState { - const rawNodes = data?.nodes ?? []; - const snapshotRaw = flags?.snapshotRaw; - const normalizedNodes = normalizeSnapshotTree(snapshotRaw ? rawNodes : pruneGroupNodes(rawNodes)); - const presentableNodes = shouldPresentIosInteractiveSnapshot(data?.backend, flags) - ? presentIosInteractiveSnapshot(normalizedNodes) - : normalizedNodes; - const scopedNodes = - flags?.snapshotScope && backendScopesAfterWire(data?.backend) - ? scopeSnapshotNodes(presentableNodes, flags.snapshotScope) - : presentableNodes; - const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality; - const nodes = attachRefs( - snapshotRaw - ? scopedNodes - : annotateCoveredSnapshotNodes(scopedNodes, { - isAdditionalOverlayNode: - data?.backend === 'android' ? isAndroidInputMethodNode : undefined, - }), - ); - return { - nodes, - truncated: data?.truncated, - createdAt: Date.now(), - backend: data?.backend, - ...(snapshotQuality ? { snapshotQuality } : {}), - presentationKey: buildSnapshotPresentationKey(snapshotPresentationOptionsFromFlags(flags)), - // Only broad Android snapshots become freshness baselines. If the user asked for a scoped - // or filtered view, preserve that output contract but avoid pretending it is safe for - // route-level comparisons on the next capture. - comparisonSafe: isAndroidComparisonSafeSnapshot(data?.backend, flags), - }; -} - function snapshotCaptureFlagsForBackend( device: SessionState['device'], flags: CommandFlags | undefined, @@ -224,43 +172,6 @@ function snapshotCaptureFlagsForBackend( return { ...flags, snapshotScope: undefined }; } -/** - * Scope resolves once per snapshot. Android resolves it inside its projection (the platform - * matcher implements the shared scope specification, `@agent-device/contracts/snapshot`), and the - * macOS helper scopes at capture; a second pass here would re-match inside an already-scoped tree - * and hand the two layers different no-match semantics (#1832 C2). - */ -function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean { - return backend !== 'macos-helper' && backend !== 'android'; -} - -function shouldPresentIosInteractiveSnapshot( - backend: SnapshotBackend | undefined, - flags: - | (Pick & - Partial>) - | undefined, -): boolean { - return ( - backend === 'xctest' && flags?.snapshotInteractiveOnly === true && flags.snapshotRaw !== true - ); -} - -function isAndroidComparisonSafeSnapshot( - backend: SnapshotBackend | undefined, - flags: - | (Pick & - Partial>) - | undefined, -): boolean { - return ( - backend === 'android' && - flags?.snapshotInteractiveOnly !== true && - typeof flags?.snapshotDepth !== 'number' && - !flags?.snapshotScope - ); -} - export function resolveSnapshotScope( snapshotScope: string | undefined, session: SessionState | undefined, diff --git a/src/daemon/snapshot-presentation/ios/transitions.test.ts b/src/daemon/snapshot-presentation/ios/transitions.test.ts index 8028d491be..6824ea402b 100644 --- a/src/daemon/snapshot-presentation/ios/transitions.test.ts +++ b/src/daemon/snapshot-presentation/ios/transitions.test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; import { createInteractionDevice } from '../../../commands/interaction/runtime/__tests__/test-utils/index.ts'; -import { buildSnapshotState } from '../../handlers/snapshot-capture.ts'; +import { buildSnapshotState } from '../../snapshot-state.ts'; import { presentIosInteractiveSnapshot } from './index.ts'; import { navigationTitleWithAppProvidedDetailsAffordanceNodes } from './transitions.fixtures.ts'; diff --git a/src/daemon/snapshot-state.ts b/src/daemon/snapshot-state.ts new file mode 100644 index 0000000000..d9526ca8bc --- /dev/null +++ b/src/daemon/snapshot-state.ts @@ -0,0 +1,106 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import { snapshotCaptureAnnotationsFrom } from '@agent-device/contracts/capture'; +import { isAndroidInputMethodNode } from '@agent-device/contracts/platform'; +import { + attachRefs, + buildSnapshotPresentationKey, + snapshotPresentationOptionsFromFlags, + type RawSnapshotNode, + type SnapshotBackend, + type SnapshotState, +} from '@agent-device/kernel/snapshot'; +import { annotateCoveredSnapshotNodes } from '../snapshot/snapshot-occlusion.ts'; +import { scopeSnapshotNodes } from '../snapshot/snapshot-desktop-surface.ts'; +import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; +import { presentIosInteractiveSnapshot } from './snapshot-presentation/ios/index.ts'; + +/** + * The ONE daemon presentation of a captured tree (ADR 0004 / #1797 "compaction layer"): normalize, + * group prune, iOS interactive presentation, post-wire scope for backends that do not scope in + * their own projection, occlusion annotation, refs. Every consumer of a captured tree — the + * snapshot command, selector captures, settle observation, Android blocking-dialog recovery — + * goes through here, so no two call sites can disagree about what a snapshot contains. + * + * Kept below the daemon-server type cycle on purpose: it needs no session state. + */ +export function buildSnapshotState( + data: { + nodes?: RawSnapshotNode[]; + truncated?: boolean; + backend?: SnapshotBackend; + quality?: unknown; + }, + flags: + | (Pick & + Partial>) + | undefined, +): SnapshotState { + const rawNodes = data?.nodes ?? []; + const snapshotRaw = flags?.snapshotRaw; + const normalizedNodes = normalizeSnapshotTree(snapshotRaw ? rawNodes : pruneGroupNodes(rawNodes)); + const presentableNodes = shouldPresentIosInteractiveSnapshot(data?.backend, flags) + ? presentIosInteractiveSnapshot(normalizedNodes) + : normalizedNodes; + const scopedNodes = + flags?.snapshotScope && backendScopesAfterWire(data?.backend) + ? scopeSnapshotNodes(presentableNodes, flags.snapshotScope) + : presentableNodes; + const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality; + const nodes = attachRefs( + snapshotRaw + ? scopedNodes + : annotateCoveredSnapshotNodes(scopedNodes, { + isAdditionalOverlayNode: + data?.backend === 'android' ? isAndroidInputMethodNode : undefined, + }), + ); + return { + nodes, + truncated: data?.truncated, + createdAt: Date.now(), + backend: data?.backend, + ...(snapshotQuality ? { snapshotQuality } : {}), + presentationKey: buildSnapshotPresentationKey(snapshotPresentationOptionsFromFlags(flags)), + // Only broad Android snapshots become freshness baselines. If the user asked for a scoped + // or filtered view, preserve that output contract but avoid pretending it is safe for + // route-level comparisons on the next capture. + comparisonSafe: isAndroidComparisonSafeSnapshot(data?.backend, flags), + }; +} + +/** + * Scope resolves once per snapshot. Android resolves it inside its projection (the platform + * matcher implements the shared scope specification, `@agent-device/contracts/snapshot`), and the + * macOS helper scopes at capture; a second pass here would re-match inside an already-scoped tree + * and hand the two layers different no-match semantics (#1832 C2). + */ +function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean { + return backend !== 'macos-helper' && backend !== 'android'; +} + +function shouldPresentIosInteractiveSnapshot( + backend: SnapshotBackend | undefined, + flags: + | (Pick & + Partial>) + | undefined, +): boolean { + return ( + backend === 'xctest' && flags?.snapshotInteractiveOnly === true && flags.snapshotRaw !== true + ); +} + +function isAndroidComparisonSafeSnapshot( + backend: SnapshotBackend | undefined, + flags: + | (Pick & + Partial>) + | undefined, +): boolean { + return ( + backend === 'android' && + flags?.snapshotInteractiveOnly !== true && + typeof flags?.snapshotDepth !== 'number' && + !flags?.snapshotScope + ); +} diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 19fb3e15e9..2bd25aff09 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -1632,11 +1632,11 @@ test('snapshotAndroid preserves bottomed-out hidden-above hints in interactive s assert.equal(scrollArea?.hiddenContentBelow, undefined); }); -test('buildUiHierarchySnapshot preserves hidden content hints from Android tree nodes', () => { +test('buildUiHierarchySnapshot derives hidden content hints from can-scroll-* on the presented node', () => { const xml = ` - + @@ -1647,8 +1647,6 @@ test('buildUiHierarchySnapshot preserves hidden content hints from Android tree const tree = parseUiHierarchyTree(xml); const scrollNode = tree.children[0]?.children[0]; assert.ok(scrollNode); - scrollNode.hiddenContentAbove = true; - scrollNode.hiddenContentBelow = true; const result = buildUiHierarchySnapshot(tree, 800, { interactiveOnly: true }); const scrollArea = result.nodes.find((node) => node.label === 'Messages'); diff --git a/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts b/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts index 274ba7b518..c3d90c92ac 100644 --- a/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy-scope.test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { buildSnapshotState } from '../../../daemon/handlers/snapshot-capture.ts'; +import { buildSnapshotState } from '../../../daemon/snapshot-state.ts'; import { parseUiHierarchy } from './ui-hierarchy-fixtures.ts'; // Android's scope leg of the golden table (#1832 C2). Android resolves `--scope` exactly once, diff --git a/src/platforms/android/__tests__/ui-hierarchy.test.ts b/src/platforms/android/__tests__/ui-hierarchy.test.ts index f514312c62..c0340f4b4c 100644 --- a/src/platforms/android/__tests__/ui-hierarchy.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy.test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { buildSnapshotState } from '../../../daemon/handlers/snapshot-capture.ts'; +import { buildSnapshotState } from '../../../daemon/snapshot-state.ts'; import { isNodeVisibleOnScreen } from '@agent-device/contracts/snapshot'; import { androidUiNodes } from '../ui-hierarchy.ts'; import { parseUiHierarchy } from './ui-hierarchy-fixtures.ts'; @@ -176,7 +176,7 @@ test('parseUiHierarchy discards stale inactive Android application windows', () `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Foreground article'), true, @@ -197,7 +197,7 @@ test('parseUiHierarchy keeps the active Android application overlay window', () `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Covered content'), false, @@ -218,7 +218,7 @@ test('parseUiHierarchy keeps only the top active Android application window', () `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Active stale content'), false, @@ -278,8 +278,8 @@ test('parseUiHierarchy reads an omitted clickable attribute the same as clickabl `; - const omitted = parseUiHierarchy(tree(''), 800, { raw: true }); - const explicitFalse = parseUiHierarchy(tree('clickable="false"'), 800, { raw: true }); + const omitted = parseUiHierarchy(tree(''), 800, {}); + const explicitFalse = parseUiHierarchy(tree('clickable="false"'), 800, {}); assert.deepEqual( explicitFalse.nodes.map((node) => [node.label, node.hittable]), @@ -298,21 +298,53 @@ test('parseUiHierarchy reads an omitted clickable attribute the same as clickabl } }); -test('parseUiHierarchy prunes Android nodes that are not visible to the user in raw snapshots', () => { +test('parseUiHierarchy hides Android nodes that are not visible to the user in regular snapshots', () => { const xml = ` `; - const result = parseUiHierarchy(xml, 800, { raw: true }); - assert.equal(result.nodes[0]!.visibleToUser, true); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Hidden drawer action'), false, ); }); +test('raw Android snapshots are the acquired tree: invisible, stale-window and covered subtrees stay (C3)', () => { + // The three regular-projection pruners are membership, not acquisition. `--raw` keeps every node + // the helper serialized (normalization only), so `interactive ⊆ regular ⊆ raw` holds by + // construction and pruned content is recoverable for diagnosis. + const xml = ` + + + + + + + + + + + + +`; + const labels = (options: Parameters[2]) => + parseUiHierarchy(xml, 800, options) + .nodes.map((node) => node.label) + .filter((label): label is string => Boolean(label)); + + assert.deepEqual(labels({ raw: true }), [ + 'Stale window action', + 'Foreground action', + 'Covered drawer action', + 'Invisible action', + ]); + assert.deepEqual(labels({}), ['Foreground action']); + assert.deepEqual(labels({ interactiveOnly: true }), ['Foreground action']); +}); + test('parseUiHierarchy prunes descendants of Android nodes that are not visible to the user', () => { const xml = ` @@ -322,7 +354,7 @@ test('parseUiHierarchy prunes descendants of Android nodes that are not visible `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Hidden drawer action'), false, @@ -350,7 +382,7 @@ test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foregrou `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Foreground action'), true, @@ -376,7 +408,7 @@ test('parseUiHierarchy keeps app content under a full-screen overlay holding one `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.deepEqual( result.nodes.filter((node) => node.label).map((node) => node.label), ['Editor', 'Save', 'dokit_contentview_id_DokitFrameLayout[1]', 'DoKit'], @@ -395,7 +427,7 @@ test('parseUiHierarchy keeps app content beside an empty labelled full-screen pl `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Toolbar action'), true, @@ -420,7 +452,7 @@ test('parseUiHierarchy keeps app content under an overlay whose only controls si `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.deepEqual( result.nodes.filter((node) => node.label).map((node) => node.label), ['Editor', 'Save', 'Debug menu', 'Frame stats'], @@ -443,7 +475,7 @@ test('parseUiHierarchy keeps app content under a focusable full-screen overlay h `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.deepEqual( result.nodes.filter((node) => node.label).map((node) => node.label), ['Your phone number', '208 379 7171', 'Attach'], @@ -466,7 +498,7 @@ test('parseUiHierarchy counts identifier-only markers toward what a covered sibl `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.identifier === 'home-body'), true, @@ -488,7 +520,7 @@ test('parseUiHierarchy compares presented footprints so a sparse overlay never c `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.deepEqual( result.nodes.filter((node) => node.label).map((node) => node.label), ['Top action', 'Bottom action', 'Badge'], @@ -505,7 +537,7 @@ test('parseUiHierarchy keeps visible identifier-only markers beside covering con `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.identifier === 'post-auth-screen'), true, @@ -528,7 +560,7 @@ test('parseUiHierarchy keeps visible side-by-side drawer and content subtrees', `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Visible drawer action'), true, @@ -551,7 +583,7 @@ test('parseUiHierarchy keeps lower siblings when drawing-order metadata is unava `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Foreground action'), true, @@ -574,7 +606,7 @@ test('parseUiHierarchy keeps overlapping siblings when drawing-order ties', () = `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'First tied action'), true, @@ -597,7 +629,7 @@ test('parseUiHierarchy keeps lower siblings below the covered-area threshold', ( `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Partial overlay action'), true, @@ -618,7 +650,7 @@ test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible ov `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Still visible action'), true, @@ -638,7 +670,7 @@ test('parseUiHierarchy keeps React Native content under a transparent Expo tools `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Agent Device Tester'), true, @@ -666,7 +698,7 @@ test('parseUiHierarchy keeps app content under a childless focusable full-screen `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Your phone number'), true, @@ -686,7 +718,7 @@ test('parseUiHierarchy keeps an overlapped text leaf drawn inside a composite wi `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === '+'), true, @@ -708,7 +740,7 @@ test('parseUiHierarchy still condemns a clickable leaf covered by a foreground s `; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal( result.nodes.some((node) => node.label === 'Modal action'), true, @@ -723,7 +755,7 @@ test('parseUiHierarchy ignores attribute-name prefix spoofing', () => { const xml = ""; - const result = parseUiHierarchy(xml, 800, { raw: true }); + const result = parseUiHierarchy(xml, 800, {}); assert.equal(result.nodes.length, 1); assert.equal(result.nodes[0]!.value, 'Actual'); }); diff --git a/src/platforms/android/snapshot-helper-capture.ts b/src/platforms/android/snapshot-helper-capture.ts index 33c7fdd9a7..c9f4036635 100644 --- a/src/platforms/android/snapshot-helper-capture.ts +++ b/src/platforms/android/snapshot-helper-capture.ts @@ -118,6 +118,9 @@ export function resolveAndroidSnapshotHelperCaptureOptions( timeoutMs + ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS, ), maxDepth: withDefault(options.maxDepth, 128), + // Declared residue (#1832): the helper caps the acquired tree device-side, BEFORE any projection + // or scope resolution runs, so a scoped target past the cap vanishes with only the generic + // `truncated` disclosure. Android acquisition is scope-blind (C2), so no hint can narrow it. maxNodes: withDefault(options.maxNodes, 5_000), packageName, runner: withDefault(options.instrumentationRunner, `${packageName}/.SnapshotInstrumentation`), diff --git a/src/platforms/android/ui-hierarchy-node.ts b/src/platforms/android/ui-hierarchy-node.ts new file mode 100644 index 0000000000..dd1a2c2022 --- /dev/null +++ b/src/platforms/android/ui-hierarchy-node.ts @@ -0,0 +1,74 @@ +import type { Rect } from '@agent-device/kernel/snapshot'; + +/** One parsed `` of an Android accessibility tree: reported facts only, plus its children. */ +export type AndroidUiHierarchy = { + type: string | null; + label: string | null; + value: string | null; + identifier: string | null; + packageName: string | null; + rect?: Rect; + enabled?: boolean; + visibleToUser?: boolean; + drawingOrder?: number; + focused?: boolean; + // Two independent facts, never collapsed, and never undefined: the helper omits false attributes + // while stock UiAutomator writes them out, so reading an absent attribute as a value gave two + // encodings of one control opposite answers. + clickable: boolean; + focusable: boolean; + depth: number; + parentIndex?: number; + scrollable?: boolean; + canScrollForward?: boolean; + canScrollBackward?: boolean; + windowIndex?: number; + windowType?: number; + windowLayer?: number; + windowActive?: boolean; + windowFocused?: boolean; + windowRect?: Rect; + children: AndroidUiHierarchy[]; +}; + +export type AndroidNode = AndroidUiHierarchy; + +/** A node a touch can act on. */ +export function isTouchTarget(node: AndroidNode): boolean { + return node.clickable; +} + +/** A node D-pad/keyboard traversal can land on. Normal for TV controls, which are rarely clickable. */ +function isFocusTarget(node: AndroidNode): boolean { + return node.focusable || node.focused === true; +} + +/** A node an agent can drive by either input model. This is what the public `hittable` projects. */ +export function isAgentTarget(node: AndroidNode): boolean { + return isTouchTarget(node) || isFocusTarget(node); +} + +/** Text or an address an agent can read or select by. */ +export function hasSemanticContent(node: AndroidNode): boolean { + return hasMeaningfulLabel(node) || hasMeaningfulIdentifier(node); +} + +export function hasMeaningfulLabel(node: AndroidNode): boolean { + const label = node.label?.trim() ?? ''; + return Boolean(label && !isGenericAndroidId(label)); +} + +function hasMeaningfulIdentifier(node: AndroidNode): boolean { + const identifier = node.identifier?.trim() ?? ''; + return Boolean(identifier && !isGenericAndroidId(identifier)); +} + +export function isGenericAndroidId(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed) return false; + return /^[\w.]+:id\/[\w.-]+$/i.test(trimmed); +} + +export function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect } { + return Boolean(node.rect && node.rect.width > 0 && node.rect.height > 0); +} diff --git a/src/platforms/android/ui-hierarchy-visibility.ts b/src/platforms/android/ui-hierarchy-visibility.ts new file mode 100644 index 0000000000..112bd5a3c2 --- /dev/null +++ b/src/platforms/android/ui-hierarchy-visibility.ts @@ -0,0 +1,308 @@ +import type { Rect } from '@agent-device/kernel/snapshot'; +import { + hasMeaningfulLabel, + hasPositiveRect, + hasSemanticContent, + isAgentTarget, + isTouchTarget, + type AndroidNode, + type AndroidUiHierarchy, +} from './ui-hierarchy-node.ts'; + +/** + * What the REGULAR projection hides: nodes Android marks invisible to users, stale application + * windows, and same-window surfaces a higher drawing-order sibling covers (#1733/#1806). + * + * This is a classification, never a mutation. The parsed tree is shared by `--raw` (which must see + * the acquired tree — C3), by the regular/interactive projections, and by hidden-content hint + * derivation, which builds a second projection from the same tree; a pruner that edited + * `node.children` in place made the tree depend on which projection ran first. + */ +export function collectAndroidHiddenNodes(root: AndroidUiHierarchy): ReadonlySet { + const hidden = new Set(); + collectInvisibleSubtrees(root, hidden); + collectInactiveApplicationWindows(root, hidden); + collectCoveredSubtrees(root, { footprintMemo: new WeakMap(), hidden }); + return hidden; +} + +function collectInvisibleSubtrees(node: AndroidNode, hidden: Set): void { + for (const child of node.children) { + if (child.visibleToUser === false) { + hidden.add(child); + continue; + } + collectInvisibleSubtrees(child, hidden); + } +} + +/** The children of `node` this projection still shows, in document order. */ +function retainedChildren(node: AndroidNode, hidden: ReadonlySet): AndroidNode[] { + return node.children.filter((child) => !hidden.has(child) && child.visibleToUser !== false); +} + +type AndroidFootprint = { + /** Boxes of what the subtree paints: touch targets, scrollables and labelled leaves. */ + paints: Rect[]; + /** Boxes of what an agent would see of the subtree: `paints` plus labelled/identified nodes. */ + shows: Rect[]; + hasAgentTarget: boolean; +}; + +type AndroidTreePruneState = { + footprintMemo: WeakMap; + hidden: Set; +}; + +type AndroidCoveringCandidate = { + node: AndroidNode; + drawingOrder: number; + footprint: Rect[]; +}; + +const ANDROID_WINDOW_TYPE_APPLICATION = 1; + +/** + * Focusability is traversal, not paint (#1733), and a label is an announcement, not paint (#1806): + * a container's content-desc describes its children and an empty labelled View draws nothing. Only + * a touch target is direct evidence that a node hides what lies under its box. + */ +function hasDirectOcclusionEvidence(node: AndroidNode): boolean { + return node.visibleToUser !== false && isTouchTarget(node); +} + +/** Evidence the node is a real surface because it contains something an agent could drive. */ +function hasDescendantOcclusionEvidence(node: AndroidNode, state: AndroidTreePruneState): boolean { + return retainedChildren(node, state.hidden).some( + (child) => subtreeFootprint(child, state).hasAgentTarget, + ); +} + +/** + * What a subtree paints and what it shows. Paint is the boxes of its touch-consuming surfaces + * (touch targets, scrollables) and labelled leaves: a full-screen debug overlay + * holding one floating icon paints only that icon, so it can only hide what sits under the icon, + * never the whole app behind it (#1806). Rects are kept apart rather than merged into one bounding + * box: two controls in opposite corners paint two corners, not the screen between them. + * + * Shows adds every labelled or identified node — a testID marker or a described container paints + * nothing, so it never helps a candidate cover, but an agent would still lose it, so it always + * counts toward what a covered sibling has. + */ +function subtreeFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { + const cached = state.footprintMemo.get(node); + if (cached !== undefined) return cached; + const footprint = hasPositiveRect(node) + ? footprintWithinBox(node, node.rect, state) + : childrenFootprint(node, state); + state.footprintMemo.set(node, footprint); + return footprint; +} + +function footprintWithinBox( + node: AndroidNode, + ownBox: Rect, + state: AndroidTreePruneState, +): AndroidFootprint { + if (paintsOwnBox(node, state.hidden)) { + // The whole box is painted; whatever it contains lies inside that box. + return { paints: [ownBox], shows: [ownBox], hasAgentTarget: isAgentTarget(node) }; + } + const footprint = childrenFootprint(node, state); + if (hasSemanticContent(node)) footprint.shows.push(ownBox); + return footprint; +} + +function childrenFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { + const footprint: AndroidFootprint = { + paints: [], + shows: [], + hasAgentTarget: isAgentTarget(node), + }; + for (const child of retainedChildren(node, state.hidden)) { + const childFootprint = subtreeFootprint(child, state); + footprint.hasAgentTarget ||= childFootprint.hasAgentTarget; + footprint.paints.push(...childFootprint.paints); + footprint.shows.push(...childFootprint.shows); + } + return footprint; +} + +/** Focusability is traversal, not paint (#1733); a container's label describes its children. */ +function paintsOwnBox(node: AndroidNode, hidden: ReadonlySet): boolean { + return ( + isTouchTarget(node) || + node.scrollable === true || + (retainedChildren(node, hidden).length === 0 && hasMeaningfulLabel(node)) + ); +} + +/** Fraction of the covered rects' union that lies under the covering rects' union. */ +function unionCoverage(coveringRects: Rect[], coveredRects: Rect[]): number { + const xs = compressedEdges([...coveringRects, ...coveredRects], (rect) => [ + rect.x, + rect.x + rect.width, + ]); + const ys = compressedEdges([...coveringRects, ...coveredRects], (rect) => [ + rect.y, + rect.y + rect.height, + ]); + const covering = markCells(coveringRects, xs, ys); + const covered = markCells(coveredRects, xs, ys); + let coveredArea = 0; + let overlapArea = 0; + for (let column = 0; column < xs.length - 1; column += 1) { + const width = xs[column + 1]! - xs[column]!; + for (let row = 0; row < ys.length - 1; row += 1) { + const cell = column * (ys.length - 1) + row; + if (!covered[cell]) continue; + const area = width * (ys[row + 1]! - ys[row]!); + coveredArea += area; + if (covering[cell]) overlapArea += area; + } + } + return coveredArea <= 0 ? 0 : overlapArea / coveredArea; +} + +function compressedEdges(rects: Rect[], edgesOf: (rect: Rect) => [number, number]): number[] { + return [...new Set(rects.flatMap(edgesOf))].sort((left, right) => left - right); +} + +function markCells(rects: Rect[], xs: number[], ys: number[]): Uint8Array { + const rows = ys.length - 1; + const cells = new Uint8Array((xs.length - 1) * rows); + for (const rect of rects) { + const firstColumn = xs.indexOf(rect.x); + const lastColumn = xs.indexOf(rect.x + rect.width); + const firstRow = ys.indexOf(rect.y); + const lastRow = ys.indexOf(rect.y + rect.height); + for (let column = firstColumn; column < lastColumn; column += 1) { + cells.fill(1, column * rows + firstRow, column * rows + lastRow); + } + } + return cells; +} + +/** + * A childless sibling that only presents: an RN screen-level testID, or a label drawn inside a + * higher sibling's box (Telegram's `+` over the country-code EditText). Geometry cannot tell a + * transparent overlay from an opaque one, and exempting a leaf cannot resurrect a covered surface. + */ +function isPresentationLeaf(node: AndroidNode, hidden: ReadonlySet): boolean { + return ( + retainedChildren(node, hidden).length === 0 && !isAgentTarget(node) && hasSemanticContent(node) + ); +} + +function collectCoveredSubtrees(node: AndroidNode, state: AndroidTreePruneState): void { + for (const child of retainedChildren(node, state.hidden)) { + collectCoveredSubtrees(child, state); + } + const siblings = retainedChildren(node, state.hidden); + if (siblings.length < 2) return; + const coveringCandidates = siblings + .map((sibling) => coveringCandidateOf(sibling, state)) + .filter((candidate) => candidate !== null); + if (coveringCandidates.length === 0) return; + for (const child of siblings) { + if (!shouldKeepAndroidSibling(child, coveringCandidates, state)) state.hidden.add(child); + } +} + +function shouldKeepAndroidSibling( + node: AndroidNode, + coveringCandidates: AndroidCoveringCandidate[], + state: AndroidTreePruneState, +): boolean { + return ( + isPresentationLeaf(node, state.hidden) || + !isCoveredByHigherDrawingOrderSibling(node, coveringCandidates, state) + ); +} + +/** + * Covered means everything an agent would see of the sibling lies under what the candidate paints, + * by actual overlapped area. Comparing footprints rather than boxes lets two stacked screens with the + * same layout margins still register as covered, while a sparse overlay never condemns a rich + * screen however far apart its controls sit. + */ +function isCoveredByHigherDrawingOrderSibling( + node: AndroidNode, + coveringCandidates: AndroidCoveringCandidate[], + state: AndroidTreePruneState, +): boolean { + if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) { + return false; + } + const shows = subtreeFootprint(node, state).shows; + const coveredRects = shows.length > 0 ? shows : [node.rect]; + for (const candidate of coveringCandidates) { + if (candidate.node === node || candidate.drawingOrder <= node.drawingOrder) { + continue; + } + if (unionCoverage(candidate.footprint, coveredRects) >= 0.9) { + return true; + } + } + return false; +} + +/** The single occlusion classification. Covering is never re-derived from a raw attribute. */ +function coveringCandidateOf( + node: AndroidNode, + state: AndroidTreePruneState, +): AndroidCoveringCandidate | null { + const { drawingOrder } = node; + if (node.visibleToUser === false || drawingOrder === undefined || !hasPositiveRect(node)) { + return null; + } + if (!hasDirectOcclusionEvidence(node) && !hasDescendantOcclusionEvidence(node, state)) { + return null; + } + const footprint = subtreeFootprint(node, state).paints; + return footprint.length > 0 ? { node, drawingOrder, footprint } : null; +} + +function collectInactiveApplicationWindows( + root: AndroidUiHierarchy, + hidden: Set, +): void { + const windows = retainedChildren(root, hidden).filter(isAndroidWindowRoot); + if (windows.length < 2) return; + + // Android can keep stale application windows in the accessibility tree after drawer and + // navigation transitions. Keep dialogs/system windows, but expose only the foreground + // application layer so agents do not act on content that is hidden from users. + const foregroundApplicationWindows = windows.filter( + (window) => isAndroidApplicationWindow(window) && isAndroidForegroundWindow(window), + ); + if (foregroundApplicationWindows.length === 0) return; + const foregroundLayer = highestAndroidWindowLayer(foregroundApplicationWindows); + + for (const window of retainedChildren(root, hidden)) { + if (!isAndroidApplicationWindow(window)) continue; + const keep = + isAndroidForegroundWindow(window) && + (foregroundLayer === undefined || window.windowLayer === foregroundLayer); + if (!keep) hidden.add(window); + } +} + +function highestAndroidWindowLayer(windows: AndroidNode[]): number | undefined { + const layers = windows + .map((window) => window.windowLayer) + .filter((layer): layer is number => layer !== undefined); + return layers.length > 0 ? Math.max(...layers) : undefined; +} + +function isAndroidWindowRoot(node: AndroidNode): boolean { + return node.windowIndex !== undefined || node.windowType !== undefined; +} + +function isAndroidApplicationWindow(node: AndroidNode): boolean { + return node.windowType === ANDROID_WINDOW_TYPE_APPLICATION; +} + +function isAndroidForegroundWindow(node: AndroidNode): boolean { + return node.windowActive === true || node.windowFocused === true; +} diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index 4fb4fa3e99..d300a49809 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -2,7 +2,16 @@ import type { RawSnapshotNode, Rect, SnapshotOptions } from '@agent-device/kerne import { parseBounds } from '@agent-device/kernel/bounds'; import { decodeXmlCharacterReferences } from '@agent-device/xml'; import { isScrollableType } from '@agent-device/contracts/snapshot'; +import { + isAgentTarget, + isGenericAndroidId, + type AndroidNode, + type AndroidUiHierarchy, +} from './ui-hierarchy-node.ts'; +import { collectAndroidHiddenNodes } from './ui-hierarchy-visibility.ts'; import { scopePresentedAndroidSnapshot } from './ui-hierarchy-scope.ts'; + +export type { AndroidUiHierarchy } from './ui-hierarchy-node.ts'; import { type AndroidSystemChromeProvenance, isAndroidSystemChromeWindowResourceId, @@ -106,6 +115,8 @@ type AndroidSnapshotBuildState = { options: SnapshotOptions; analysis: AndroidSnapshotAnalysis; interactiveDescendantMemo: Map; + /** Subtrees the regular projection hides (invisible / stale window / covered). Empty for raw. */ + hidden: ReadonlySet; truncated: boolean; }; @@ -125,6 +136,9 @@ export function buildUiHierarchySnapshot( options, analysis: analyzeAndroidTree(tree), interactiveDescendantMemo: new Map(), + // C3: raw is the acquired tree (normalization only); regular additionally hides what Android + // marks invisible, stale application windows, and covered same-window surfaces. + hidden: options.raw ? new Set() : collectAndroidHiddenNodes(tree), truncated: false, }; @@ -163,7 +177,7 @@ function walkUiHierarchyNode( state.truncated = true; return; } - if (depth > state.maxDepth) return; + if (depth > state.maxDepth || state.hidden.has(node)) return; const include = state.options.raw ? true @@ -221,8 +235,7 @@ function appendAndroidSnapshotNode( hittable: isAgentTarget(node) || undefined, depth: compactedAndroidNodeDepth(state.nodes, parentIndex), parentIndex, - ...(node.hiddenContentAbove ? { hiddenContentAbove: true } : {}), - ...(node.hiddenContentBelow ? { hiddenContentBelow: true } : {}), + ...androidScrollActionHints(node, state.hidden), ...(systemChrome ? { systemChrome: true } : {}), }); return currentIndex; @@ -240,6 +253,7 @@ function hasInteractiveDescendant(state: AndroidSnapshotBuildState, node: Androi if (cached !== undefined) return cached; for (const child of node.children) { if ( + !state.hidden.has(child) && child.visibleToUser !== false && (isAgentTarget(child) || hasInteractiveDescendant(state, child)) ) { @@ -396,40 +410,6 @@ function readXmlAttr(attrs: Map, name: string): string | null { return attrs.get(name) ?? null; } -export type AndroidUiHierarchy = { - type: string | null; - label: string | null; - value: string | null; - identifier: string | null; - packageName: string | null; - rect?: Rect; - enabled?: boolean; - visibleToUser?: boolean; - drawingOrder?: number; - focused?: boolean; - // Two independent facts, never collapsed, and never undefined: the helper omits false attributes - // while stock UiAutomator writes them out, so reading an absent attribute as a value gave two - // encodings of one control opposite answers. - clickable: boolean; - focusable: boolean; - depth: number; - parentIndex?: number; - hiddenContentAbove?: boolean; - hiddenContentBelow?: boolean; - scrollable?: boolean; - canScrollForward?: boolean; - canScrollBackward?: boolean; - windowIndex?: number; - windowType?: number; - windowLayer?: number; - windowActive?: boolean; - windowFocused?: boolean; - windowRect?: Rect; - children: AndroidNode[]; -}; - -type AndroidNode = AndroidUiHierarchy; - type AndroidNodeInclusionInfo = { type: string; hasMeaningfulText: boolean; @@ -438,26 +418,6 @@ type AndroidNodeInclusionInfo = { isVisual: boolean; }; -type AndroidFootprint = { - /** Boxes of what the subtree paints: touch targets, scrollables and labelled leaves. */ - paints: Rect[]; - /** Boxes of what an agent would see of the subtree: `paints` plus labelled/identified nodes. */ - shows: Rect[]; - hasAgentTarget: boolean; -}; - -type AndroidTreePruneState = { - footprintMemo: WeakMap; -}; - -type AndroidCoveringCandidate = { - node: AndroidNode; - drawingOrder: number; - footprint: Rect[]; -}; - -const ANDROID_WINDOW_TYPE_APPLICATION = 1; - export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { const root: AndroidUiHierarchy = { type: null, @@ -514,330 +474,46 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { } match = tokenRegex.exec(xml); } - // Raw Android snapshots are uncollapsed, but still agent-visible. The helper can expose - // aria-hidden/no-hide-descendants children, so prune nodes Android marks hidden to users. - pruneAndroidInvisibleSubtrees(root); - discardInactiveAndroidApplicationWindows(root); - // UiAutomation can expose covered React Native navigation surfaces in the same accessibility - // window. If a higher drawing-order sibling covers them, agents should see the foreground surface. - pruneAndroidCoveredSubtrees(root, { footprintMemo: new WeakMap() }); - applyAndroidScrollActionHints(root); return root; } -/** A node a touch can act on. */ -function isTouchTarget(node: AndroidNode): boolean { - return node.clickable; -} - -/** A node D-pad/keyboard traversal can land on. Normal for TV controls, which are rarely clickable. */ -function isFocusTarget(node: AndroidNode): boolean { - return node.focusable || node.focused === true; -} - -/** A node an agent can drive by either input model. This is what the public `hittable` projects. */ -function isAgentTarget(node: AndroidNode): boolean { - return isTouchTarget(node) || isFocusTarget(node); -} - -/** Text or an address an agent can read or select by. */ -function hasSemanticContent(node: AndroidNode): boolean { - return hasMeaningfulLabel(node) || hasMeaningfulIdentifier(node); -} - /** - * Focusability is traversal, not paint (#1733), and a label is an announcement, not paint (#1806): - * a container's content-desc describes its children and an empty labelled View draws nothing. Only - * a touch target is direct evidence that a node hides what lies under its box. + * Scroll-action hints (`hiddenContentAbove/Below`) for the presented node, from the helper's + * can-scroll-* attributes. Derived per projection over the children that projection shows: the + * overflow estimate that tells a horizontal list from a vertical one must not count children the + * regular projection hides, and raw must count them all. */ -function hasDirectOcclusionEvidence(node: AndroidNode): boolean { - return node.visibleToUser !== false && isTouchTarget(node); -} - -/** Evidence the node is a real surface because it contains something an agent could drive. */ -function hasDescendantOcclusionEvidence(node: AndroidNode, state: AndroidTreePruneState): boolean { - return node.children.some( - (child) => child.visibleToUser !== false && subtreeFootprint(child, state).hasAgentTarget, - ); -} - -/** - * What a subtree paints and what it shows. Paint is the boxes of its touch-consuming surfaces - * (touch targets, scrollables) and labelled leaves: a full-screen debug overlay - * holding one floating icon paints only that icon, so it can only hide what sits under the icon, - * never the whole app behind it (#1806). Rects are kept apart rather than merged into one bounding - * box: two controls in opposite corners paint two corners, not the screen between them. - * - * Shows adds every labelled or identified node — a testID marker or a described container paints - * nothing, so it never helps a candidate cover, but an agent would still lose it, so it always - * counts toward what a covered sibling has. - */ -function subtreeFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { - const cached = state.footprintMemo.get(node); - if (cached !== undefined) return cached; - const footprint = hasPositiveRect(node) - ? footprintWithinBox(node, node.rect, state) - : childrenFootprint(node, state); - state.footprintMemo.set(node, footprint); - return footprint; -} - -function footprintWithinBox( +function androidScrollActionHints( node: AndroidNode, - ownBox: Rect, - state: AndroidTreePruneState, -): AndroidFootprint { - if (paintsOwnBox(node)) { - // The whole box is painted; whatever it contains lies inside that box. - return { paints: [ownBox], shows: [ownBox], hasAgentTarget: isAgentTarget(node) }; - } - const footprint = childrenFootprint(node, state); - if (hasSemanticContent(node)) footprint.shows.push(ownBox); - return footprint; -} - -function childrenFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { - const footprint: AndroidFootprint = { - paints: [], - shows: [], - hasAgentTarget: isAgentTarget(node), + hidden: ReadonlySet, +): { hiddenContentAbove?: true; hiddenContentBelow?: true } { + if (!isVerticalScrollableNode(node, hidden)) return {}; + return { + ...(node.canScrollBackward ? { hiddenContentAbove: true as const } : {}), + ...(node.canScrollForward ? { hiddenContentBelow: true as const } : {}), }; - for (const child of node.children) { - if (child.visibleToUser === false) continue; - const childFootprint = subtreeFootprint(child, state); - footprint.hasAgentTarget ||= childFootprint.hasAgentTarget; - footprint.paints.push(...childFootprint.paints); - footprint.shows.push(...childFootprint.shows); - } - return footprint; -} - -/** Focusability is traversal, not paint (#1733); a container's label describes its children. */ -function paintsOwnBox(node: AndroidNode): boolean { - return ( - isTouchTarget(node) || - node.scrollable === true || - (node.children.length === 0 && hasMeaningfulLabel(node)) - ); -} - -/** Fraction of the covered rects' union that lies under the covering rects' union. */ -function unionCoverage(coveringRects: Rect[], coveredRects: Rect[]): number { - const xs = compressedEdges([...coveringRects, ...coveredRects], (rect) => [ - rect.x, - rect.x + rect.width, - ]); - const ys = compressedEdges([...coveringRects, ...coveredRects], (rect) => [ - rect.y, - rect.y + rect.height, - ]); - const covering = markCells(coveringRects, xs, ys); - const covered = markCells(coveredRects, xs, ys); - let coveredArea = 0; - let overlapArea = 0; - for (let column = 0; column < xs.length - 1; column += 1) { - const width = xs[column + 1]! - xs[column]!; - for (let row = 0; row < ys.length - 1; row += 1) { - const cell = column * (ys.length - 1) + row; - if (!covered[cell]) continue; - const area = width * (ys[row + 1]! - ys[row]!); - coveredArea += area; - if (covering[cell]) overlapArea += area; - } - } - return coveredArea <= 0 ? 0 : overlapArea / coveredArea; -} - -function compressedEdges(rects: Rect[], edgesOf: (rect: Rect) => [number, number]): number[] { - return [...new Set(rects.flatMap(edgesOf))].sort((left, right) => left - right); -} - -function markCells(rects: Rect[], xs: number[], ys: number[]): Uint8Array { - const rows = ys.length - 1; - const cells = new Uint8Array((xs.length - 1) * rows); - for (const rect of rects) { - const firstColumn = xs.indexOf(rect.x); - const lastColumn = xs.indexOf(rect.x + rect.width); - const firstRow = ys.indexOf(rect.y); - const lastRow = ys.indexOf(rect.y + rect.height); - for (let column = firstColumn; column < lastColumn; column += 1) { - cells.fill(1, column * rows + firstRow, column * rows + lastRow); - } - } - return cells; -} - -/** - * A childless sibling that only presents: an RN screen-level testID, or a label drawn inside a - * higher sibling's box (Telegram's `+` over the country-code EditText). Geometry cannot tell a - * transparent overlay from an opaque one, and exempting a leaf cannot resurrect a covered surface. - */ -function isPresentationLeaf(node: AndroidNode): boolean { - return node.children.length === 0 && !isAgentTarget(node) && hasSemanticContent(node); } -function pruneAndroidInvisibleSubtrees(node: AndroidNode): void { - let keptCount = 0; - for (const child of node.children) { - if (child.visibleToUser === false) continue; - pruneAndroidInvisibleSubtrees(child); - node.children[keptCount] = child; - keptCount += 1; - } - if (keptCount < node.children.length) { - node.children.length = keptCount; - } -} - -function pruneAndroidCoveredSubtrees(node: AndroidNode, state: AndroidTreePruneState): void { - for (const child of node.children) { - pruneAndroidCoveredSubtrees(child, state); - } - if (node.children.length < 2) { - return; - } - const siblings = node.children; - const coveringCandidates = siblings - .map((sibling) => coveringCandidateOf(sibling, state)) - .filter((candidate) => candidate !== null); - if (coveringCandidates.length === 0) return; - node.children = siblings.filter((child) => - shouldKeepAndroidSibling(child, coveringCandidates, state), - ); -} - -function shouldKeepAndroidSibling( - node: AndroidNode, - coveringCandidates: AndroidCoveringCandidate[], - state: AndroidTreePruneState, -): boolean { - return ( - isPresentationLeaf(node) || - !isCoveredByHigherDrawingOrderSibling(node, coveringCandidates, state) - ); -} - -/** - * Covered means everything an agent would see of the sibling lies under what the candidate paints, - * by actual overlapped area. Comparing footprints rather than boxes lets two stacked screens with the - * same layout margins still register as covered, while a sparse overlay never condemns a rich - * screen however far apart its controls sit. - */ -function isCoveredByHigherDrawingOrderSibling( - node: AndroidNode, - coveringCandidates: AndroidCoveringCandidate[], - state: AndroidTreePruneState, -): boolean { - if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) { - return false; - } - const shows = subtreeFootprint(node, state).shows; - const coveredRects = shows.length > 0 ? shows : [node.rect]; - for (const candidate of coveringCandidates) { - if (candidate.node === node || candidate.drawingOrder <= node.drawingOrder) { - continue; - } - if (unionCoverage(candidate.footprint, coveredRects) >= 0.9) { - return true; - } - } - return false; -} - -function hasMeaningfulIdentifier(node: AndroidNode): boolean { - const identifier = node.identifier?.trim() ?? ''; - return Boolean(identifier && !isGenericAndroidId(identifier)); -} - -/** The single occlusion classification. Covering is never re-derived from a raw attribute. */ -function coveringCandidateOf( - node: AndroidNode, - state: AndroidTreePruneState, -): AndroidCoveringCandidate | null { - const { drawingOrder } = node; - if (node.visibleToUser === false || drawingOrder === undefined || !hasPositiveRect(node)) { - return null; - } - if (!hasDirectOcclusionEvidence(node) && !hasDescendantOcclusionEvidence(node, state)) { - return null; - } - const footprint = subtreeFootprint(node, state).paints; - return footprint.length > 0 ? { node, drawingOrder, footprint } : null; -} - -function hasMeaningfulLabel(node: AndroidNode): boolean { - const label = node.label?.trim() ?? ''; - return Boolean(label && !isGenericAndroidId(label)); -} - -function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect } { - return Boolean(node.rect && node.rect.width > 0 && node.rect.height > 0); -} - -function applyAndroidScrollActionHints(root: AndroidUiHierarchy): void { - const stack = [...root.children]; - while (stack.length > 0) { - const node = stack.pop() as AndroidNode; - stack.push(...node.children); - if (!isVerticalScrollableNode(node)) continue; - if (node.canScrollBackward) node.hiddenContentAbove = true; - if (node.canScrollForward) node.hiddenContentBelow = true; - } -} - -function discardInactiveAndroidApplicationWindows(root: AndroidUiHierarchy): void { - const windows = root.children.filter(isAndroidWindowRoot); - if (windows.length < 2) return; - - // Android can keep stale application windows in the accessibility tree after drawer and - // navigation transitions. Keep dialogs/system windows, but expose only the foreground - // application layer so agents do not act on content that is hidden from users. - const foregroundApplicationWindows = windows.filter( - (window) => isAndroidApplicationWindow(window) && isAndroidForegroundWindow(window), - ); - if (foregroundApplicationWindows.length === 0) return; - const foregroundLayer = highestAndroidWindowLayer(foregroundApplicationWindows); - - root.children = root.children.filter((window) => { - if (!isAndroidApplicationWindow(window)) return true; - if (!isAndroidForegroundWindow(window)) return false; - return foregroundLayer === undefined || window.windowLayer === foregroundLayer; - }); -} - -function highestAndroidWindowLayer(windows: AndroidNode[]): number | undefined { - const layers = windows - .map((window) => window.windowLayer) - .filter((layer): layer is number => layer !== undefined); - return layers.length > 0 ? Math.max(...layers) : undefined; -} - -function isAndroidWindowRoot(node: AndroidNode): boolean { - return node.windowIndex !== undefined || node.windowType !== undefined; -} - -function isAndroidApplicationWindow(node: AndroidNode): boolean { - return node.windowType === ANDROID_WINDOW_TYPE_APPLICATION; -} - -function isAndroidForegroundWindow(node: AndroidNode): boolean { - return node.windowActive === true || node.windowFocused === true; -} - -function isVerticalScrollableNode(node: AndroidNode): boolean { +function isVerticalScrollableNode(node: AndroidNode, hidden: ReadonlySet): boolean { if (!node.scrollable || !isScrollableType(node.type)) return false; const type = `${node.type ?? ''}`.toLowerCase(); if (type.includes('horizontalscrollview')) return false; - const overflow = estimateChildOverflow(node); + const overflow = estimateChildOverflow(node, hidden); if (overflow && overflow.horizontal > overflow.vertical && overflow.horizontal > 16) { return false; } return true; } -function estimateChildOverflow(node: AndroidNode): { horizontal: number; vertical: number } | null { - if (!node.rect || node.children.length === 0) return null; - const childRects = node.children.map((child) => child.rect).filter((rect) => rect !== undefined); +function estimateChildOverflow( + node: AndroidNode, + hidden: ReadonlySet, +): { horizontal: number; vertical: number } | null { + const children = node.children.filter( + (child) => !hidden.has(child) && child.visibleToUser !== false, + ); + if (!node.rect || children.length === 0) return null; + const childRects = children.map((child) => child.rect).filter((rect) => rect !== undefined); if (childRects.length === 0) return null; const minX = Math.min(...childRects.map((rect) => rect.x)); const maxX = Math.max(...childRects.map((rect) => rect.x + rect.width)); @@ -954,12 +630,6 @@ function isStructuralAndroidType(type: string): boolean { return short.includes('layout') || short === 'viewgroup' || short === 'view'; } -function isGenericAndroidId(value: string): boolean { - const trimmed = value.trim(); - if (!trimmed) return false; - return /^[\w.]+:id\/[\w.-]+$/i.test(trimmed); -} - function analyzeAndroidTree(root: AndroidNode): AndroidSnapshotAnalysis { let rawNodeCount = 0; let maxDepth = 0; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index cca3d0322e..4a0cbe5a9f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -337,7 +337,11 @@ agent-device get attrs @e1 alongside the app root; `androidSnapshot.captureMode` and `androidSnapshot.windowCount` describe the capture. On API 23 the helper cannot report `drawing-order`, so covered same-window surfaces (for example a React Native screen left under the foreground one) are not pruned; - `androidSnapshot.occlusionScanUnavailable: true` discloses that capture shape. + `androidSnapshot.occlusionScanUnavailable: true` discloses that capture shape. Android + `--raw` is the acquired tree: it keeps nodes Android marks invisible, stale application windows, + and covered same-window surfaces that the default and `-i` views hide, so use it to see what a + pruned surface contained. The helper does not report `checked`/`selected` state, and it caps + captures at 5000 nodes before any `--scope` applies (`truncated: true`). - `--scope ` returns the subtree of the first node in document order whose label, value, or identifier contains the scope text (case-insensitive) and whose subtree still has content in the requested projection, re-rooted at depth 0; no match returns an empty snapshot rather than the From 775f2360b56558d89b231e172de6846f58f92500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 20:00:19 +0200 Subject: [PATCH 2/3] test: lower the snapshot.test.ts size pin to its new length --- src/__tests__/test-file-size-ratchet.test.ts | 2 +- .../snapshot-presentation/ios/publication-membership.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/__tests__/test-file-size-ratchet.test.ts b/src/__tests__/test-file-size-ratchet.test.ts index 6d77423363..a388cf638e 100644 --- a/src/__tests__/test-file-size-ratchet.test.ts +++ b/src/__tests__/test-file-size-ratchet.test.ts @@ -40,7 +40,7 @@ const PINNED_TEST_FILE_LINES: Readonly> = Object.freeze({ 'src/platforms/apple/core/__tests__/runner-session.test.ts': 2001, 'src/utils/__tests__/daemon-client.test.ts': 1910, 'src/utils/__tests__/output.test.ts': 1861, - 'src/platforms/android/__tests__/snapshot.test.ts': 1660, + 'src/platforms/android/__tests__/snapshot.test.ts': 1658, 'src/platforms/apple/core/__tests__/runner-client.test.ts': 1615, 'src/__tests__/client.test.ts': 1598, 'test/integration/provider-scenarios/android-lifecycle.test.ts': 1559, diff --git a/src/daemon/snapshot-presentation/ios/publication-membership.test.ts b/src/daemon/snapshot-presentation/ios/publication-membership.test.ts index f87d4d521c..a4531118eb 100644 --- a/src/daemon/snapshot-presentation/ios/publication-membership.test.ts +++ b/src/daemon/snapshot-presentation/ios/publication-membership.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { buildSnapshotState } from '../../handlers/snapshot-capture.ts'; +import { buildSnapshotState } from '../../snapshot-state.ts'; // End-to-end publication-membership contract for the acquire/present design (#1797, external // review pass 4 finding 1), exercised through the production interface. Runner presentation owns From f2d52b12dc531a824e5263c8b4922eb466bcef25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 19 Aug 2026 10:40:38 +0200 Subject: [PATCH 3/3] fix(android): dialog recovery acts on the presentation's occlusion result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review P1 on #1865: routing blocking-dialog recovery through buildSnapshotState made the occlusion result available but nothing consumed it. containsBlockingDialog scanned every node and findCloseAppButton returned the first text match with a rect, so a stale ANR surface left under the foreground one could still trigger recovery, and a covered "Close app" could be tapped ahead of the visible top button — the disagreement the routing was supposed to remove. Both decisions now filter through isSnapshotNodeInteractionBlocked, the shared predicate over the annotator's structured result. Two regressions cover it, both proven red against an unfiltered selection: a covered Close app preceding a visible one (asserts the visible center is tapped) and a fully covered dialog signal (asserts recovery does not trigger, no tap dispatched). Rebase reconciliation: screenshot-runtime.ts arrived on main (#1878) importing buildSnapshotState from its old home; repointed to src/daemon/snapshot-state.ts with the other importers. --- CHANGELOG.md | 2 +- .../android-system-dialog-occlusion.test.ts | 114 ++++++++++++++++++ src/daemon/android-system-dialog.ts | 16 ++- src/daemon/screenshot-runtime.ts | 3 +- 4 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 src/daemon/__tests__/android-system-dialog-occlusion.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 60940f51ab..3d258ffe2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - `--udid` with `--platform android` (and `--serial` with an Apple platform) now fails as the flag mistake it is — `INVALID_ARGS` naming the right flag — instead of reaching device resolution and answering `No Apple device with UDID emulator-5580` for an explicitly Android request. `--udid` addresses Apple devices, `--serial` addresses Android and HarmonyOS; matching pairs and requests that name no platform are unchanged. - Breaking (`--session-lock strip`): a device selector that names a different device than the bound session is no longer silently discarded. `strip` exists to drop redundant platform/scope selectors; when it also dropped `--udid`/`--serial`/`--device`, the command kept running against the *bound* device instead of the one the caller named — a wrong-device action that looks like a success. Such a request now fails with `INVALID_ARGS` under both `reject` and `strip`, and the error carries the two identities structurally (`requestedDevice`, `boundDevice`) plus a hint offering the two real recoveries: close the bound session if the requested device is intended, or remove the selector if the bound device is. The hint no longer suggests `--session-lock strip` for an identity conflict, since following that advice is what produced the wrong-device run. Scope-only stripping (`--platform`, `--target`, `--ios-simulator-device-set`, `--android-device-allowlist`) is unchanged. - iOS regular snapshots now apply one backend-neutral eligibility rule after every capture backend: a node survives when its accessibility type is interactive or it carries a non-empty label, identifier, or value. This removes the tree backend's extra "hittable non-Other" membership path and drops unlabeled decorative nodes consistently; labeled images, identifier-only nodes, and value-only nodes still survive. Raw snapshot membership is unchanged. -- Android `snapshot --raw` is now the acquired accessibility tree (#1832 C3): the three regular-projection pruners — nodes Android marks invisible, stale application windows, and covered same-window surfaces — no longer run at parse time, so `--raw` keeps everything the helper serialized (normalization only) and pruned content is recoverable for diagnosis. Regular and `-i` output is unchanged (property-checked identical to the previous implementation over 12,000 random tree × projection pairs, hidden-content hints included). Also: Android blocking-dialog recovery now reads the same daemon presentation an agent's `snapshot` sees instead of a hand-rolled subset; the Android freshness route signature no longer keys on `role`/`selected`, fields the Android backend never carries; and the Android helper's declared fidelity residues (no `checked`/`checkable`/`selected`/`long-clickable`, 5000-node cap before scoping, API-level cache-reset divergence) are recorded in `CONTEXT.md`. +- Android `snapshot --raw` is now the acquired accessibility tree (#1832 C3): the three regular-projection pruners — nodes Android marks invisible, stale application windows, and covered same-window surfaces — no longer run at parse time, so `--raw` keeps everything the helper serialized (normalization only) and pruned content is recoverable for diagnosis. Regular and `-i` output is unchanged (property-checked identical to the previous implementation over 12,000 random tree × projection pairs, hidden-content hints included). Also: Android blocking-dialog recovery now reads the same daemon presentation an agent's `snapshot` sees instead of a hand-rolled subset, and acts on its occlusion result — a stale "App isn't responding" surface left under the foreground one no longer triggers recovery, and a covered "Close app" is never tapped ahead of the visible one; the Android freshness route signature no longer keys on `role`/`selected`, fields the Android backend never carries; and the Android helper's declared fidelity residues (no `checked`/`checkable`/`selected`/`long-clickable`, 5000-node cap before scoping, API-level cache-reset divergence) are recorded in `CONTEXT.md`. - `agent-device mcp` now carries its own usage guidance, so MCP-only clients (Codex CLI, Cursor, custom agents) no longer depend on a separately installed skill (#1833). The handshake `instructions` — returned by both `server/discover` and, newly, the legacy `initialize` — is a compact (< 2 KB, the Claude Code truncation limit) workflow card: start with `open {app, foreground: true}` instead of probing, act with `settle: true` and continue from the diff, verify with `wait`/`is`/`get`/`find`, copy `@refs` byte-for-byte, recover from sparse/AX-unavailable, follow error hints, `close`. A new MCP-only `help` tool serves the full guides on demand: no `topic` returns the CLI's decision card; `topic` returns `agent-device help ` verbatim (workflow, gestures, scripting, tv, macos, web, remote, debugging, …, or any tool name for its complete flag reference), prefixed with the one-line CLI→tool-property mapping. `help` is router-owned rather than a command descriptor, so it appears in `tools/list` only — not in the CLI, Node client, or `batch` — and its description tells the model it is not a startup step. Legacy `initialize` gains the optional `instructions` field; no other legacy field changes. - Android `snapshot --scope` (and every selector command's `--scope`, e.g. `press "Save" --scope Panel`) now resolves scope exactly once, inside the Android projection, under the shared scope specification: the scope root is the first node **in document order** whose label, value, or identifier contains the scope text (case-insensitive) **and whose subtree still has content in the projection you asked for**, the result is that subtree re-rooted at depth 0, and no match returns an empty snapshot (#1832). That second clause is what makes `snapshot -i --scope panel` return the button inside a structural container `-i` drops, and stops a decorative heading that happens to match from emptying the snapshot. Before, Android ran two passes with contradictory rules — a breadth-first platform match that fell back to the full tree on a miss, then the daemon's document-order pass — so a shallower later container could win over an earlier match, and an interaction capture whose scope reached only the daemon layer was silently unscoped. `--depth` under `--scope` counts from the scope root, filtering the depths the response prints (a node shown at depth 0 is never hidden by `--depth 0`), and ancestor context above the scope root (a clickable row, a list) still shapes `-i` membership inside it. The rule is pinned by `contracts/fixtures/snapshot-scope-policy.json`, the same golden table the iOS runner will consume (#1797). Also new: `androidSnapshot.occlusionScanUnavailable: true` discloses an API 23 capture, where the helper cannot report `drawing-order` and covered same-window surfaces are therefore not pruned. - New `hover ` command for `--platform web` (#1783). It moves the pointer over the target without pressing, so hover-gated UI — a message row's `...` toolbar, a menu that opens on pointer enter — becomes reachable through agent-device the way it already was through the underlying `agent-browser` backend (`mouse move`). It is a member of the targeted-touch family: same `@ref`/selector/coordinate targeting, occlusion and off-screen guards, and `--settle` (the settled diff carries the revealed controls with fresh refs, e.g. `+ @e4 [button] "Delete"`), but no `--verify`, since hover reveals rather than activates. `hover @ref` publishes as a portable selector line in recorded scripts, and the Node client exposes `interactions.hover`. Hover is a pointer state that touch platforms do not have, so `capabilities` advertises it on web only and iOS/Android/Linux reject it during admission with `UNSUPPORTED_OPERATION` and a hint naming `--platform web`; `longpress` remains the mobile hold-gesture verb. diff --git a/src/daemon/__tests__/android-system-dialog-occlusion.test.ts b/src/daemon/__tests__/android-system-dialog-occlusion.test.ts new file mode 100644 index 0000000000..1e7ca0646b --- /dev/null +++ b/src/daemon/__tests__/android-system-dialog-occlusion.test.ts @@ -0,0 +1,114 @@ +import { test, expect, vi } from 'vitest'; + +// #1832: blocking-dialog recovery reads the daemon presentation, so it must also consume that +// presentation's OCCLUSION result. A stale "App isn't responding" surface left under the foreground +// one must not trigger recovery, and a covered "Close app" must not be tapped ahead of the visible +// one — the exact disagreement routing through buildSnapshotState exists to remove. +vi.mock('../../platforms/android/snapshot.ts', () => ({ snapshotAndroid: vi.fn() })); +vi.mock('../../platforms/android/adb.ts', () => ({ runAndroidAdb: vi.fn() })); + +import { snapshotAndroid } from '../../platforms/android/snapshot.ts'; +import { runAndroidAdb } from '../../platforms/android/adb.ts'; +import { recoverAndroidBlockingSystemDialog } from '../android-system-dialog.ts'; +import { makeAndroidSession } from '../../__tests__/test-utils/session-factories.ts'; +import { makeTestScreenRecordingResource } from '../../__tests__/test-utils/screen-recording-live-handle.ts'; + +const SCREEN = { x: 0, y: 0, width: 1080, height: 2400 }; + +/** A foreground surface drawn over everything below it — what makes the stale dialog unreachable. */ +const foregroundSheet = (index: number) => ({ + index, + type: 'com.google.android.material.bottomsheet.BottomSheetDialog', + identifier: 'com.example:id/bottom_sheet', + hittable: true, + rect: SCREEN, +}); + +function recordingSession(name: string) { + const session = makeAndroidSession(name); + session.screenRecording = makeTestScreenRecordingResource(session, { + backend: 'adb screenrecord', + outPath: '/tmp/anr.mp4', + startedAt: 0, + }); + return session; +} + +function tappedPoints(): Array<{ x: number; y: number }> { + return vi + .mocked(runAndroidAdb) + .mock.calls.filter((call) => (call[1] as string[]).includes('tap')) + .map((call) => { + const args = call[1] as string[]; + const at = args.indexOf('tap'); + return { x: Number(args[at + 1]), y: Number(args[at + 2]) }; + }); +} + +test('recovery taps the reachable Close app, not a covered one that comes first', async () => { + vi.mocked(runAndroidAdb).mockReset(); + // The live ANR dialog is on top; a stale "Close app" from an earlier one is still in the tree + // underneath the sheet. Document order puts the stale button FIRST, which is what the previous + // hand-rolled presentation selected: the first text match carrying a rect. + const staleCloseApp = { + index: 0, + type: 'Button', + label: 'Close app', + hittable: true, + rect: { x: 40, y: 300, width: 200, height: 80 }, + }; + const sheet = { ...foregroundSheet(1), rect: { x: 0, y: 0, width: 1080, height: 1000 } }; + const dialog = { + index: 2, + type: 'TextView', + label: "App isn't responding", + rect: { x: 0, y: 1700, width: 1080, height: 60 }, + }; + const visibleCloseApp = { + index: 3, + type: 'Button', + label: 'Close app', + hittable: true, + rect: { x: 40, y: 1800, width: 200, height: 80 }, + }; + vi.mocked(snapshotAndroid) + .mockResolvedValueOnce({ nodes: [staleCloseApp, sheet, dialog, visibleCloseApp] } as never) + .mockResolvedValue({ nodes: [] } as never); + vi.mocked(runAndroidAdb).mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' } as never); + + const result = await recoverAndroidBlockingSystemDialog({ + session: recordingSession('anr-covered-button'), + }); + + expect(result.status).toBe('recovered'); + expect(tappedPoints()).toEqual([{ x: 140, y: 1840 }]); +}); + +test('a fully covered dialog does not trigger recovery', async () => { + vi.mocked(runAndroidAdb).mockReset(); + const staleDialog = { + index: 0, + type: 'TextView', + label: "App isn't responding", + hittable: true, + rect: { x: 0, y: 100, width: 1080, height: 60 }, + }; + const staleCloseApp = { + index: 1, + type: 'Button', + label: 'Close app', + hittable: true, + rect: { x: 40, y: 300, width: 200, height: 80 }, + }; + vi.mocked(snapshotAndroid).mockResolvedValue({ + nodes: [staleDialog, staleCloseApp, foregroundSheet(2)], + } as never); + vi.mocked(runAndroidAdb).mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' } as never); + + const result = await recoverAndroidBlockingSystemDialog({ + session: recordingSession('anr-covered-dialog'), + }); + + expect(result.status).toBe('absent'); + expect(tappedPoints()).toEqual([]); +}); diff --git a/src/daemon/android-system-dialog.ts b/src/daemon/android-system-dialog.ts index 7685d75ffc..73f159cd11 100644 --- a/src/daemon/android-system-dialog.ts +++ b/src/daemon/android-system-dialog.ts @@ -11,6 +11,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { centerOfRect, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { sleep } from '../utils/timeouts.ts'; import { buildSnapshotState } from './snapshot-state.ts'; +import { isSnapshotNodeInteractionBlocked } from '../snapshot/snapshot-occlusion.ts'; import { expireRefFrame } from './ref-frame.ts'; import type { SessionState } from './types.ts'; @@ -302,6 +303,12 @@ async function tapAndroidDialogButton( return { ok: true, x, y }; } +/** + * Recovery acts on what a user can actually touch, so both decisions read the presentation's + * structured occlusion result rather than raw text: a stale "Close app" left under the foreground + * surface must neither trigger recovery nor be tapped ahead of the visible one (#1832 review). + * `buildSnapshotState` already annotates covered nodes; this is the consumer side of that. + */ function findCloseAppButton( nodes: SnapshotNode[], options: { requireDialogSignal?: boolean } = {}, @@ -309,13 +316,18 @@ function findCloseAppButton( if (options.requireDialogSignal !== false && !containsBlockingDialog(nodes)) { return undefined; } - return nodes.find((node) => { + return nodes.filter(isTouchableDialogNode).find((node) => { return ( readNodeTextParts(node).some((text) => ANDROID_CLOSE_APP_PATTERN.test(text)) && node.rect ); }); } +/** A node the recovery tap can actually reach: present in the tree and not covered by a surface above it. */ +function isTouchableDialogNode(node: SnapshotNode): boolean { + return !isSnapshotNodeInteractionBlocked(node); +} + async function waitForBlockingDialogToDismiss(session: SessionState): Promise { for (let attempt = 0; attempt < ANDROID_MODAL_POLL_ATTEMPTS; attempt += 1) { const nodes = await readAndroidSnapshotNodes(session); @@ -377,7 +389,7 @@ function readNodeTextParts(node: { } function containsBlockingDialog(nodes: SnapshotNode[]): boolean { - return nodes.some((node) => { + return nodes.filter(isTouchableDialogNode).some((node) => { const text = readNodeText(node); return text.length > 0 && ANDROID_BLOCKING_MODAL_PATTERN.test(text); }); diff --git a/src/daemon/screenshot-runtime.ts b/src/daemon/screenshot-runtime.ts index a5d6ef399f..40e63ebb4a 100644 --- a/src/daemon/screenshot-runtime.ts +++ b/src/daemon/screenshot-runtime.ts @@ -19,7 +19,8 @@ import { readScreenshotResultMetadata, } from '../utils/screenshot-density.ts'; import type { DaemonCommandContext } from './context.ts'; -import { buildSnapshotState, captureSnapshotData } from './handlers/snapshot-capture.ts'; +import { captureSnapshotData } from './handlers/snapshot-capture.ts'; +import { buildSnapshotState } from './snapshot-state.ts'; import type { RecordedGenericRequest, ResolvedGenericExecution,