diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index cd1831bf3..8e3aed178 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -163,6 +163,8 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsScopeAndRelativeDepth \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPresentationRefusesAnAcquisitionCapturedForTheOtherProjection \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCaptureHintIsTheOnlyAcquisitionViewOfARequest \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthFrontierSurvivesStructuralWrapperCollapse \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthFrontierKeepsVisibleIndependentChildPastClippedParent \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldClipsScrollOverflowReparentsAndBooksHints \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldKeepsWindowCarriersButNeverHittableOutsideClip \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldDropsSubPixelContentlessDecorationOnEveryBackend \ diff --git a/CHANGELOG.md b/CHANGELOG.md index dc4d9c713..e33626642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- iOS regular `snapshot --depth` now measures depth after structural accessibility wrappers + collapse. The recursive-tree backend follows a bounded presented-depth frontier, so controls + that fit the requested regular depth are no longer lost behind raw wrappers; raw `--depth` + remains a traversal-depth limit. Flat query recovery is limited to one presented level, and + private AX does not claim deeper regular-depth completeness until it has a hierarchy-aware + frontier (#1797). - iOS regular snapshot nodes now publish presentation-owned effective geometry through the existing `rect` field: backend-reported frames remain available to acquisition, while regular output uses the viewport and declared scroll-clip intersection. Raw snapshots and direct element reads retain diff --git a/CONTEXT.md b/CONTEXT.md index cf3fab182..1d23c319f 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -192,8 +192,12 @@ The policy input controlling how one snapshot acquisition becomes a public proje **Capture hint**: The acquisition-facing view of a snapshot request, derived once from presentation options. It names -the projection a backend must serve and may narrow acquisition only where that backend can prove the -narrowing complete. +the projection a backend must serve, keeps raw traversal depth separate from regular presented depth, +and may narrow acquisition only where that backend can prove the narrowing complete. + +**Regular presented-depth frontier**: +The acquisition boundary for an unscoped regular snapshot, measured against regular presented depth +after structural wrappers collapse. It is distinct from raw traversal depth. **Snapshot eligibility**: Membership in a presented snapshot projection, independent of whether a node is currently hittable. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index b24970654..b8092a25d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -109,10 +109,10 @@ extension RunnerTests { deadline: Date = .distantFuture ) -> SnapshotAcquisition? { #if os(iOS) && targetEnvironment(simulator) - let requestedDepth = hint.depth ?? 64 + let requestedDepth = hint.rawTraversalDepth ?? 64 // An explicit --depth request is honored as asked: no accepted-depth // memory, no frontier extension past it. - let exactDepthRequested = hint.depth != nil + let exactDepthRequested = hint.rawTraversalDepth != nil let rememberedDepth = exactDepthRequested ? nil @@ -645,7 +645,8 @@ extension RunnerTests { let nodes = privateAXAcquisition( rawRoot: tree, hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: false, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), viewport: CGRect(x: 0, y: 0, width: 390, height: 844) ) @@ -756,7 +757,8 @@ extension RunnerTests { ] let viewport = CGRect(x: 0, y: 0, width: 390, height: 844) let hint = CaptureHint( - projection: .regular, depth: nil, interactiveOnly: true, customActions: false) + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: true, customActions: false) let acquired = privateAXAcquisition(rawRoot: tree, hint: hint, viewport: viewport) // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index 81e0b3e62..24a4e1a4e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -34,7 +34,7 @@ extension RunnerTests { private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [RawAXNode], hint: CaptureHint, viewport: CGRect, depth: Int, parentIndex: Int?) { - if let limit = hint.depth, depth > limit { return } + if let limit = hint.rawTraversalDepth, depth > limit { return } let fields = privateAXFields(raw) let index = nodes.count nodes.append( @@ -134,7 +134,8 @@ extension RunnerTests { interactiveOnly: Bool = false ) throws -> [PresentedNode] { let hint = CaptureHint( - projection: .regular, depth: nil, interactiveOnly: interactiveOnly, customActions: false) + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: interactiveOnly, customActions: false) let acquired = privateAXAcquisition(rawRoot: rawRoot, hint: hint, viewport: viewport) return try SnapshotPresentation.presentRegular( SnapshotAcquisition( @@ -164,7 +165,9 @@ extension RunnerTests { let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport, interactiveOnly: true) let raw = privateAXAcquisition(rawRoot: root, - hint: CaptureHint(projection: .raw, depth: nil, interactiveOnly: false, customActions: false), + hint: CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), viewport: viewport) XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Image", "Button"]) @@ -190,7 +193,9 @@ extension RunnerTests { /// can prove complete. func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() { let raw = privateAXAcquisition(rawRoot: Self.privateAXScrolledFixture, - hint: CaptureHint(projection: .raw, depth: 2, interactiveOnly: false, customActions: false), + hint: CaptureHint( + projection: .raw, depth: 2, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Button"]) XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 2]) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 11e161119..bd892ccf0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -15,7 +15,6 @@ extension RunnerTests { let queryRoot: XCUIElement let rootSnapshot: XCUIElementSnapshot let viewport: CGRect - let maxDepth: Int } private struct SnapshotEvaluation { @@ -30,6 +29,8 @@ extension RunnerTests { let snapshot: XCUIElementSnapshot let depth: Int let parentIndex: Int? + let parentPresentedDepth: Int + let parentTraversal: SnapshotVisibilityFold.TraversalState } struct SnapshotCaptureFailure: Error { @@ -146,9 +147,9 @@ extension RunnerTests { // Acquisition serializes facts: every traversed node is emitted at raw traversal depth, and // the regular projection's clip fold runs once inside `SnapshotPresentation` (#1797). The two - // walks this backend keeps are budget and augmentation, never membership: the traversal-depth - // cut (declared residue -- regular presentation emits collapsed depth) and the collapsed-tab - // expansion, which needs live element handles. + // walks this backend keeps are the raw budget or regular presented-depth frontier, plus + // collapsed-tab augmentation which needs live element handles; neither walk publishes a + // presentation node. var nodes: [RawAXNode] = [] let rootEvaluation = evaluateSnapshot(context.rootSnapshot) nodes.append( @@ -161,7 +162,12 @@ extension RunnerTests { viewport: context.viewport ) ) - if context.maxDepth > 0 { + let shouldVisitRootChildren = SnapshotPresentation.shouldAcquireChildren( + for: hint, + rawDepth: 0, + regularPresentedDepth: 0 + ) + if shouldVisitRootChildren { appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: context.rootSnapshot, @@ -173,21 +179,34 @@ extension RunnerTests { } var seen = Set() - var stack: [SnapshotTraversalEntry] = context.rootSnapshot.children.map { - SnapshotTraversalEntry( - snapshot: $0, - depth: 1, - parentIndex: 0 - ) + var stack: [SnapshotTraversalEntry] = [] + if shouldVisitRootChildren { + stack = context.rootSnapshot.children.map { + SnapshotTraversalEntry( + snapshot: $0, + depth: 1, + parentIndex: 0, + parentPresentedDepth: 0, + parentTraversal: .root + ) + } } while let entry = stack.popLast() { let snapshot = entry.snapshot let depth = entry.depth let parentIndex = entry.parentIndex - if let limit = hint.depth, depth > limit { continue } + if let limit = hint.rawTraversalDepth, depth > limit { continue } let evaluation = evaluateSnapshot(snapshot) + let node = makeSnapshotNode( + snapshot: snapshot, + evaluation: evaluation, + depth: depth, + index: nodes.count, + parentIndex: parentIndex, + viewport: context.viewport + ) let key = Self.snapshotTraversalIdentity( elementType: snapshot.elementType, label: evaluation.label, @@ -200,13 +219,26 @@ extension RunnerTests { } let currentIndex = !isDuplicate ? nodes.count : parentIndex - if depth < context.maxDepth { + let transition = SnapshotPresentation.regularTraversalTransition( + for: node, + parentPresentedDepth: entry.parentPresentedDepth, + parentTraversal: entry.parentTraversal, + hint: hint, + rawDepth: depth, + viewport: context.viewport, + hasChildren: !snapshot.children.isEmpty, + isDuplicate: isDuplicate, + policy: .platformDefault + ) + if transition.shouldVisitChildren { for child in snapshot.children.reversed() { stack.append( SnapshotTraversalEntry( snapshot: child, depth: depth + 1, - parentIndex: currentIndex + parentIndex: currentIndex, + parentPresentedDepth: transition.presentedDepth, + parentTraversal: transition.traversal ) ) } @@ -214,24 +246,14 @@ extension RunnerTests { if isDuplicate { continue } - let index = nodes.count - nodes.append( - makeSnapshotNode( - snapshot: snapshot, - evaluation: evaluation, - depth: depth, - index: index, - parentIndex: parentIndex, - viewport: context.viewport - ) - ) - if depth < context.maxDepth { + nodes.append(node) + if transition.shouldVisitChildren { appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: snapshot, resolveElements: collapsedTabDescendants, depth: depth + 1, - parentIndex: index, + parentIndex: node.index, viewport: context.viewport ) } @@ -349,7 +371,7 @@ extension RunnerTests { var nodes: [RawAXNode] = [] func walk(_ snapshot: XCUIElementSnapshot, depth: Int, parentIndex: Int?) throws { - if let limit = hint.depth, depth > limit { return } + if let limit = hint.rawTraversalDepth, depth > limit { return } let evaluation = evaluateSnapshot(snapshot) if nodes.count >= Self.rawSnapshotMaxNodes { @@ -391,7 +413,7 @@ extension RunnerTests { var nodes: [RawAXNode] = [ interactiveRootNode(rect: .zero) ] - if hint.depth == 0 { + if hint.rawTraversalDepth == 0 || hint.regularPresentedDepth == 0 { return SnapshotAcquisition( hint: hint, nodes: nodes, @@ -831,8 +853,7 @@ extension RunnerTests { return SnapshotTraversalContext( queryRoot: app, rootSnapshot: rootSnapshot, - viewport: viewport, - maxDepth: hint.depth ?? Int.max + viewport: viewport ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift index 9baef4dcd..0acbe0294 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift @@ -5,6 +5,16 @@ enum SnapshotBackendEnvironment { case physicalDevice } +enum SnapshotRegularDepthCapability: String { + /// The backend can stop acquisition at the requested regular presented-depth frontier. + case presentedFrontier = "presented-frontier" + /// The backend is flat; it can answer the root and one presented level, but has no hierarchy + /// from which to prove deeper regular depth. + case flat + /// The backend can return raw traversal depth, but cannot prove regular presented depth. + case rawOnly = "raw-only" +} + enum SnapshotBackendKind: String, CaseIterable { case recursiveTree = "tree" case querySweep = "queries" @@ -45,6 +55,29 @@ enum SnapshotBackendKind: String, CaseIterable { } } + var regularDepthCapability: SnapshotRegularDepthCapability { + switch self { + case .recursiveTree: + return .presentedFrontier + case .querySweep: + return .flat + case .privateAX: + return .rawOnly + } + } + + func canServeRegularPresentedDepth(_ requestedDepth: Int?) -> Bool { + guard let requestedDepth else { return true } + switch regularDepthCapability { + case .presentedFrontier: + return true + case .flat: + return requestedDepth <= 1 + case .rawOnly: + return false + } + } + var isAvailableOnCurrentPlatform: Bool { #if os(iOS) && targetEnvironment(simulator) return isAvailable(on: .simulator) @@ -74,6 +107,7 @@ private struct SnapshotBackendParityFixture: Decodable { let name: String let forceable: Bool let supportsRawProjection: Bool + let regularDepth: String let hittable: String let availability: Availability } @@ -99,7 +133,7 @@ extension RunnerTests { } /// The JSON table is the cross-runtime declaration used by the TypeScript capability registry - /// and this runner. A backend case, forceability branch, raw projection claim, or availability + /// and this runner. A backend case, forceability branch, projection/depth claim, or availability /// change that is not classified in both implementations fails before an iOS smoke can drift. func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws { let fixture = try loadSnapshotBackendParityFixture() @@ -115,6 +149,11 @@ extension RunnerTests { } XCTAssertEqual(backend.isForceable, expected.forceable, expected.name) XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name) + XCTAssertEqual( + backend.regularDepthCapability.rawValue, + expected.regularDepth, + "regular depth capability: \(expected.name)" + ) XCTAssertEqual( backend.hittableSemantics, expected.hittable, @@ -132,5 +171,14 @@ extension RunnerTests { ) } } + + func testRegularDepthCapabilityDoesNotClaimFlatOrRawOnlyParity() { + XCTAssertTrue(SnapshotBackendKind.recursiveTree.canServeRegularPresentedDepth(8)) + XCTAssertTrue(SnapshotBackendKind.querySweep.canServeRegularPresentedDepth(0)) + XCTAssertTrue(SnapshotBackendKind.querySweep.canServeRegularPresentedDepth(1)) + XCTAssertFalse(SnapshotBackendKind.querySweep.canServeRegularPresentedDepth(2)) + XCTAssertFalse(SnapshotBackendKind.privateAX.canServeRegularPresentedDepth(1)) + XCTAssertTrue(SnapshotBackendKind.privateAX.canServeRegularPresentedDepth(nil)) + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 030a7e4b1..91dad856f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -422,6 +422,22 @@ extension RunnerTests { ) throws -> SnapshotBackendAttempt { let hint = SnapshotPresentation.captureHint(for: options) var timer = SnapshotPhaseTimer() + // Scoped depth is relative to a presentation-selected root, so its hint stays broad and the + // backend capability gate applies only to an unscoped regular frontier. + let requestedRegularDepth = options.raw || SnapshotScopePolicy.isActive(options.scope) + ? nil + : options.depth + guard kind.canServeRegularPresentedDepth(requestedRegularDepth) else { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_BACKEND_DEPTH_UNSUPPORTED backend=%@ depth=%ld", + kind.rawValue, + requestedRegularDepth ?? -1 + ) + return SnapshotBackendAttempt( + outcome: .noCapture, + timing: timer.timing + ) + } let acquisition: SnapshotAcquisition? do { acquisition = try timer.measure(.acquisition) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift index 24c96b6e9..09c5ce51f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift @@ -170,12 +170,103 @@ enum SnapshotPresentation { let projection: CaptureHint.Projection = options.raw ? .raw : .regular return CaptureHint( projection: projection, - depth: scoped ? nil : options.depth, + depth: scoped || projection == .regular ? nil : options.depth, + regularPresentedDepth: scoped || projection == .raw ? nil : options.depth, interactiveOnly: projection == .raw ? false : options.interactiveOnly, customActions: options.customActions ) } + /// Raw depth is an acquisition limit. Regular depth is a presentation limit, + /// so hierarchy-capable backends acquire through the structural-wrapper + /// frontier until this presented depth is complete. + static func shouldAcquireChildren( + for hint: CaptureHint, + rawDepth: Int, + regularPresentedDepth: Int + ) -> Bool { + if let rawLimit = hint.rawTraversalDepth { + return rawDepth < rawLimit + } + if let presentedLimit = hint.regularPresentedDepth { + return regularPresentedDepth < presentedLimit + } + return true + } + + /// Computes one regular-tree frontier transition for both acquisition and its tests. The fold + /// remains the sole visibility interpreter; this helper only carries its decision into the + /// presentation-owned depth budget and the next traversal state. + static func regularTraversalTransition( + for raw: RawAXNode, + parentPresentedDepth: Int, + parentTraversal: SnapshotVisibilityFold.TraversalState, + hint: CaptureHint, + rawDepth: Int, + viewport: CGRect, + hasChildren: Bool, + isDuplicate: Bool, + policy: SnapshotVisibilityFold.Policy = .platformDefault + ) -> ( + presentedDepth: Int, + traversal: SnapshotVisibilityFold.TraversalState, + shouldVisitChildren: Bool + ) { + guard hint.regularPresentedDepth != nil else { + return ( + parentPresentedDepth, + parentTraversal, + shouldAcquireChildren( + for: hint, + rawDepth: rawDepth, + regularPresentedDepth: parentPresentedDepth + ) + ) + } + + let visibility = SnapshotVisibilityFold.traversalDecision( + for: raw, + parent: parentTraversal, + viewport: viewport, + interactiveOnly: hint.interactiveOnly, + hasChildren: hasChildren, + policy: policy + ) + let presentedDepth = regularPresentedDepth( + for: raw, + parentPresentedDepth: parentPresentedDepth, + visibility: visibility + ) + let nextPresentedDepth = isDuplicate ? parentPresentedDepth : presentedDepth + let nextTraversal = isDuplicate ? parentTraversal : visibility.descendants + let descendantsMayBeVisible = isDuplicate + ? parentTraversal.descendantsMayBeVisible + : visibility.descendantsMayBeVisible + return ( + nextPresentedDepth, + nextTraversal, + shouldAcquireChildren( + for: hint, + rawDepth: rawDepth, + regularPresentedDepth: nextPresentedDepth + ) && descendantsMayBeVisible + ) + } + + /// Shared depth accounting for the acquisition frontier. The fold supplies the same visibility + /// decision used by regular presentation; this method adds only the presentation-owned semantic + /// eligibility predicate. + static func regularPresentedDepth( + for raw: RawAXNode, + parentPresentedDepth: Int, + visibility: SnapshotVisibilityFold.TraversalDecision + ) -> Int { + guard raw.parentIndex != nil else { return 0 } + return visibility.isIncluded && isEligibleForRegularPresentation(raw) + ? parentPresentedDepth + 1 + : parentPresentedDepth + } + private static func project( _ projectionNodes: [SnapshotPresentationNode], acquisition: SnapshotAcquisition, @@ -183,10 +274,14 @@ enum SnapshotPresentation { projection: CaptureHint.Projection ) -> SnapshotBackendCapture { let scopedRawNodes = applyScope(to: projectionNodes, options: options, projection: projection) - let nodes = presentedNodes(from: scopedRawNodes, projection: projection) + let nodes = presentedNodes( + from: scopedRawNodes, + projection: projection, + maximumDepth: projection == .regular ? options.depth : nil + ) let qualityPayload: DataPayload? = SnapshotScopePolicy.isActive(options.scope) ? DataPayload( - nodes: presentedNodes(from: projectionNodes, projection: projection), + nodes: presentedNodes(from: projectionNodes, projection: projection, maximumDepth: nil), truncated: acquisition.truncated ) : nil @@ -209,7 +304,8 @@ enum SnapshotPresentation { private static func presentedNodes( from rawNodes: [SnapshotPresentationNode], - projection: CaptureHint.Projection + projection: CaptureHint.Projection, + maximumDepth: Int? = nil ) -> [PresentedNode] { if projection == .raw { return rawNodes.map { PresentedNode(presenting: $0) } @@ -231,6 +327,12 @@ enum SnapshotPresentation { let presentedIndex = nodes.count let presentedDepth = presentedParent.map { $0.depth + 1 } ?? 0 + if let maximumDepth, presentedDepth > maximumDepth { + if let presentedParent { + nearestPresentedNodeByRawIndex[raw.index] = presentedParent + } + continue + } nearestPresentedNodeByRawIndex[raw.index] = (presentedIndex, presentedDepth) nodes.append( PresentedNode( @@ -272,10 +374,10 @@ enum SnapshotPresentation { depth: { $0.raw.depth } ) let maxDepth = options.depth ?? Int.max - return reindex( - Array(rawNodes[range]).filter { $0.raw.depth - startDepth <= maxDepth }, - depthOffset: startDepth - ) + let scopedNodes = projection == .raw + ? Array(rawNodes[range]).filter { $0.raw.depth - startDepth <= maxDepth } + : Array(rawNodes[range]) + return reindex(scopedNodes, depthOffset: startDepth) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift index d5e6142ec..9b49318b7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift @@ -44,14 +44,22 @@ struct CaptureHint { } let projection: Projection - /// Traversal-depth budget. Regular presentation's collapsed depth retains the visible-depth - /// frontier residue described by #1797; this cut stays cheap for depth probes. + /// Raw traversal-depth budget. This is populated only for raw captures, where + /// presented depth is raw acquisition depth by contract. let depth: Int? + /// Requested regular depth after presentation collapses structural wrappers. + /// It is populated only for unscoped regular captures; scoped depth is applied + /// after the scope root is selected and therefore cannot narrow acquisition. + let regularPresentedDepth: Int? /// Regular-projection acquisition budget. Raw projection is the acquired tree and never carries /// this narrowing. let interactiveOnly: Bool let customActions: Bool + var rawTraversalDepth: Int? { + projection == .raw ? depth : nil + } + var isRaw: Bool { projection == .raw } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift index d4b5bcecf..cb4a3b614 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift @@ -22,24 +22,24 @@ enum SnapshotVisibilityFold { /// Wire-name vocabulary corresponding to XCTest's scroll-container element types. static let scrollContainerTypeNames: Set = ["CollectionView", "ScrollView", "Table"] - private enum Geometry { + fileprivate enum Geometry { case geometryless case framed(intersectsClip: Bool) } - private enum DescendantVisibility { + fileprivate enum DescendantVisibility { case independent case owned } - private struct ProjectionCursor { + fileprivate struct ProjectionCursor { static let root = ProjectionCursor(ancestorProjectedOut: false) private let ancestorProjectedOut: Bool var isProjectedOut: Bool { ancestorProjectedOut } - func project( + fileprivate func project( geometry: Geometry, descendantVisibility: DescendantVisibility ) -> ProjectionDecision { @@ -60,16 +60,45 @@ enum SnapshotVisibilityFold { } } - private struct ProjectionDecision { + fileprivate struct ProjectionDecision { let presentationVisible: Bool let descendants: ProjectionCursor } - private struct ProjectionTransition { + fileprivate struct ProjectionTransition { let decision: ProjectionDecision let hiddenContentFrame: CGRect? } + + /// The fold's traversal state is also consumed by acquisition when a regular depth frontier is + /// requested. It carries only projection facts; presentation remains the owner of membership + /// and output construction. + struct TraversalState { + fileprivate let cursor: ProjectionCursor + fileprivate let ancestorClip: CGRect? + + fileprivate init(cursor: ProjectionCursor, ancestorClip: CGRect?) { + self.cursor = cursor + self.ancestorClip = ancestorClip + } + + static let root = TraversalState(cursor: .root, ancestorClip: nil) + + var descendantsMayBeVisible: Bool { !cursor.isProjectedOut } + } + + struct TraversalDecision { + /// Whether the shared fold would retain this raw node before regular semantic eligibility. + let isIncluded: Bool + /// Whether any descendant can remain visible under the same projection cursor. + let descendantsMayBeVisible: Bool + let descendants: TraversalState + fileprivate let effectiveFrame: CGRect + fileprivate let hiddenContentFrame: CGRect? + fileprivate let establishesScrollAnchor: Bool + } + private struct BranchState { - let cursor: ProjectionCursor + let traversal: TraversalState let anchor: (index: Int, rect: CGRect)? let keptIndex: Int? let keptDepth: Int @@ -78,6 +107,64 @@ enum SnapshotVisibilityFold { private static let negligibleDecorationTolerance = 1.0 private static let visibilityExemptCarrierTypes: Set = ["Application", "Window"] + static func traversalDecision( + for node: RawAXNode, + parent: TraversalState, + viewport: CGRect, + interactiveOnly: Bool, + hasChildren: Bool, + policy: Policy + ) -> TraversalDecision { + let ancestorClip = policy == .cursorProjected ? parent.ancestorClip : nil + let rect = CGRect( + x: node.rect.x, y: node.rect.y, width: node.rect.width, height: node.rect.height + ) + let effectiveFrame = SnapshotGeometry.effectiveFrame( + reportedFrame: rect, + viewport: viewport, + ancestorClip: ancestorClip + ) + let intersects = !effectiveFrame.isNull && !effectiveFrame.isEmpty + let transition = projectionTransition( + frame: rect, + intersectsClip: intersects, + typeName: node.type, + hasChildren: hasChildren, + cursor: parent.cursor, + policy: policy + ) + let negligibleDecoration = policy == .cursorProjected + && node.parentIndex != nil + && !node.hasSemanticContent + && (rect.isEmpty + || rect.width <= negligibleDecorationTolerance + || rect.height <= negligibleDecorationTolerance) + let visible = transition.decision.presentationVisible && !negligibleDecoration + let isIncluded = shouldInclude( + node, + visible: visible, + interactiveOnly: interactiveOnly, + policy: policy + ) + let establishesScrollAnchor = policy == .cursorProjected + && isIncluded + && intersects + && hasChildren + && scrollContainerTypeNames.contains(node.type) + let descendants = TraversalState( + cursor: transition.decision.descendants, + ancestorClip: establishesScrollAnchor ? effectiveFrame : ancestorClip + ) + return TraversalDecision( + isIncluded: isIncluded, + descendantsMayBeVisible: !transition.decision.descendants.isProjectedOut, + descendants: descendants, + effectiveFrame: effectiveFrame, + hiddenContentFrame: transition.hiddenContentFrame, + establishesScrollAnchor: establishesScrollAnchor + ) + } + static func fold( _ nodes: [RawAXNode], viewport: CGRect, @@ -97,47 +184,27 @@ enum SnapshotVisibilityFold { for (offset, node) in nodes.enumerated() { let parentState = node.parentIndex.flatMap { states[$0] } - let parentCursor = parentState?.cursor ?? .root + let parentTraversal = parentState?.traversal ?? .root let parentAnchor = policy == .cursorProjected ? parentState?.anchor : nil let rect = CGRect( x: node.rect.x, y: node.rect.y, width: node.rect.width, height: node.rect.height ) - let effectiveFrame = SnapshotGeometry.effectiveFrame( - reportedFrame: rect, + let decision = traversalDecision( + for: node, + parent: parentTraversal, viewport: viewport, - ancestorClip: parentAnchor?.rect - ) - let intersects = !effectiveFrame.isNull && !effectiveFrame.isEmpty - let transition = projectionTransition( - frame: rect, - intersectsClip: intersects, - typeName: node.type, - hasChildren: hasChildren[offset], - cursor: parentCursor, - policy: policy - ) - - let negligibleDecoration = policy == .cursorProjected - && node.parentIndex != nil - && !node.hasSemanticContent - && (rect.isEmpty - || rect.width <= negligibleDecorationTolerance - || rect.height <= negligibleDecorationTolerance) - let visible = transition.decision.presentationVisible && !negligibleDecoration - let include = shouldInclude( - node, - visible: visible, interactiveOnly: interactiveOnly, + hasChildren: hasChildren[offset], policy: policy ) - if let hiddenFrame = transition.hiddenContentFrame, let parentAnchor { + if let hiddenFrame = decision.hiddenContentFrame, let parentAnchor { rememberHiddenContentHint(for: hiddenFrame, relativeTo: parentAnchor, hints: &hints) } var keptIndex = parentState?.keptIndex var keptDepth = parentState?.keptDepth ?? -1 - if include { + if decision.isIncluded { let outIndex = kept.count let outDepth = keptDepth + 1 kept.append( @@ -154,7 +221,7 @@ enum SnapshotVisibilityFold { selected: node.selected, hittable: node.parentIndex != nil && SnapshotGeometry.isGeometricallyActionable( enabled: node.enabled, - frame: effectiveFrame, + frame: decision.effectiveFrame, viewport: viewport ), depth: outDepth, @@ -164,7 +231,9 @@ enum SnapshotVisibilityFold { actions: node.actions ), effectiveRect: SnapshotGeometry.snapshotRect( - from: effectiveFrame, reportedFrame: rect) + from: decision.effectiveFrame, + reportedFrame: rect + ) ) ) keptIndex = outIndex @@ -172,20 +241,11 @@ enum SnapshotVisibilityFold { } var anchor = parentAnchor - if policy == .cursorProjected, - include, - let newAnchor = scrollContainerAnchor( - forTypeName: node.type, - hasChildren: hasChildren[offset], - visible: intersects, - frame: effectiveFrame, - nodeIndex: keptIndex - ) - { - anchor = newAnchor + if decision.establishesScrollAnchor, let keptIndex { + anchor = (index: keptIndex, rect: decision.effectiveFrame) } states[offset] = BranchState( - cursor: transition.decision.descendants, + traversal: decision.descendants, anchor: anchor, keptIndex: keptIndex, keptDepth: keptDepth @@ -238,21 +298,6 @@ enum SnapshotVisibilityFold { return visibilityExemptCarrierTypes.contains(node.type) || visible } - private static func scrollContainerAnchor( - forTypeName typeName: String, - hasChildren: Bool, - visible: Bool, - frame: CGRect, - nodeIndex: Int? - ) -> (index: Int, rect: CGRect)? { - guard let nodeIndex, - visible, - hasChildren, - scrollContainerTypeNames.contains(typeName) - else { return nil } - return (nodeIndex, frame) - } - private static func rememberHiddenContentHint( for frame: CGRect, relativeTo scrollAnchor: (index: Int, rect: CGRect), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift index 5711834fd..f744bdf54 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift @@ -91,7 +91,8 @@ extension RunnerTests { return SnapshotAcquisition( hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: false, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, + customActions: false), nodes: nodes, truncated: false, effectiveDepth: nil, @@ -102,7 +103,8 @@ extension RunnerTests { func testRegularPresentationKeepsNestedClipGeometryCumulative() throws { let acquisition = SnapshotAcquisition( hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: false, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, + customActions: false), nodes: [ Self.invariantNode( 0, @@ -163,7 +165,8 @@ extension RunnerTests { func testRegularPresentationKeepsFramelessSemanticCarriersNonActionableAndNonClipping() throws { let acquisition = SnapshotAcquisition( hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: true, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: true, + customActions: false), nodes: [ Self.invariantNode( 0, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index 9996fb972..79874e634 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -28,7 +28,8 @@ extension RunnerTests { let capture = try XCTUnwrap(try SnapshotPresentation.present( SnapshotAcquisition( hint: CaptureHint( - projection: .raw, depth: nil, interactiveOnly: false, customActions: false), + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), nodes: [raw], truncated: true, effectiveDepth: 4, @@ -108,7 +109,8 @@ extension RunnerTests { try SnapshotPresentation.presentRegular( SnapshotAcquisition( hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: false, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), nodes: acquired, truncated: false, effectiveDepth: nil, viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)), options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) @@ -118,7 +120,8 @@ extension RunnerTests { try SnapshotPresentation.presentRegular( SnapshotAcquisition( hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: true, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: true, customActions: false), nodes: acquired, truncated: false, effectiveDepth: nil, viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)), options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false) @@ -141,7 +144,8 @@ extension RunnerTests { SnapshotPresentation.presentRaw( SnapshotAcquisition( hint: CaptureHint( - projection: .raw, depth: nil, interactiveOnly: false, customActions: false), + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), nodes: acquired, truncated: false, effectiveDepth: nil, viewport: .infinite), options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: true) ).payload.nodes @@ -166,7 +170,8 @@ extension RunnerTests { ] let acquisition = SnapshotAcquisition( hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: false, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), nodes: nodes, truncated: false, effectiveDepth: nil, @@ -267,7 +272,8 @@ extension RunnerTests { let acquisition = SnapshotAcquisition( hint: CaptureHint( - projection: .regular, depth: nil, interactiveOnly: true, customActions: false), + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: true, customActions: false), nodes: [ node(0, type: "Application", label: "App", depth: 0, parentIndex: nil), node(1, type: "Button", label: "Earlier sibling", depth: 1, parentIndex: 0), @@ -301,7 +307,8 @@ extension RunnerTests { SnapshotPresentation.presentRaw( SnapshotAcquisition( hint: CaptureHint( - projection: .raw, depth: nil, interactiveOnly: false, customActions: false), + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), nodes: acquisition.nodes, truncated: false, effectiveDepth: nil, @@ -388,18 +395,27 @@ extension RunnerTests { interactiveOnly: true, depth: 2, scope: "Settings", raw: false)) // Scope re-roots the tree and depth counts from that root: neither can narrow acquisition. XCTAssertNil(scoped.depth) + XCTAssertNil(scoped.regularPresentedDepth) XCTAssertEqual(scoped.projection, .regular) let depthOnly = SnapshotPresentation.captureHint( for: PresentationOptions(interactiveOnly: true, depth: 2, scope: nil, raw: false)) - XCTAssertEqual(depthOnly.depth, 2) + XCTAssertNil(depthOnly.depth) + XCTAssertEqual(depthOnly.regularPresentedDepth, 2) // The raw projection is the acquired tree, so `--raw -i` never narrows acquisition either. let raw = SnapshotPresentation.captureHint( for: PresentationOptions(interactiveOnly: true, depth: 3, scope: nil, raw: true)) XCTAssertEqual(raw.projection, .raw) XCTAssertFalse(raw.interactiveOnly) - XCTAssertEqual(raw.depth, 3) + XCTAssertEqual(raw.rawTraversalDepth, 3) + XCTAssertNil(raw.regularPresentedDepth) + XCTAssertTrue( + SnapshotPresentation.shouldAcquireChildren( + for: raw, rawDepth: 0, regularPresentedDepth: 0)) + XCTAssertFalse( + SnapshotPresentation.shouldAcquireChildren( + for: raw, rawDepth: 3, regularPresentedDepth: 0)) XCTAssertTrue(raw.isRaw) let actions = SnapshotPresentation.captureHint( @@ -407,5 +423,185 @@ extension RunnerTests { interactiveOnly: false, depth: nil, scope: nil, raw: false, customActions: true)) XCTAssertTrue(actions.customActions) } + + /// #1797 visible-depth frontier: a regular depth is measured after structural + /// wrappers collapse, so the acquisition hint cannot present it as a raw + /// traversal cap. The root -> wrapper -> button fixture is the smallest tree + /// that distinguishes those two meanings at the public boundary. + func testRegularDepthFrontierSurvivesStructuralWrapperCollapse() throws { + func node( + _ index: Int, + type: String, + label: String?, + depth: Int, + parentIndex: Int? + ) -> RawAXNode { + RawAXNode( + index: index, + type: type, + label: label, + identifier: nil, + value: nil, + rect: SnapshotRect(x: 0, y: Double(index * 20), width: 100, height: 20), + enabled: true, + focused: nil, + selected: nil, + hittable: type == "Button", + depth: depth, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + let options = PresentationOptions( + interactiveOnly: false, depth: 1, scope: nil, raw: false) + let hint = SnapshotPresentation.captureHint(for: options) + let capture = try XCTUnwrap( + SnapshotPresentation.present( + SnapshotAcquisition( + hint: hint, + nodes: [ + node(0, type: "Application", label: "App", depth: 0, parentIndex: nil), + node(1, type: "Other", label: nil, depth: 1, parentIndex: 0), + node(2, type: "Button", label: "Save", depth: 2, parentIndex: 1), + node(3, type: "Button", label: "More", depth: 3, parentIndex: 2), + ], + truncated: false, + effectiveDepth: nil, + viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + ), + options: options + ) + ) + + XCTAssertEqual(hint.regularPresentedDepth, 1) + XCTAssertNil(hint.rawTraversalDepth) + XCTAssertTrue( + SnapshotPresentation.shouldAcquireChildren( + for: hint, rawDepth: 0, regularPresentedDepth: 0)) + XCTAssertTrue( + SnapshotPresentation.shouldAcquireChildren( + for: hint, rawDepth: 1, regularPresentedDepth: 0)) + XCTAssertFalse( + SnapshotPresentation.shouldAcquireChildren( + for: hint, rawDepth: 2, regularPresentedDepth: 1)) + XCTAssertEqual(capture.payload.nodes?.map(\.label), ["App", "Save"]) + XCTAssertEqual(capture.payload.nodes?.map(\.depth), [0, 1]) + } + + /// #1797 P1: an eligible parent outside the viewport is removed by the shared visibility fold, + /// while an independently projected child remains visible and must occupy the requested depth. + /// The public presentation/capture-hint boundary must not let the removed parent stop acquisition. + func testRegularDepthFrontierKeepsVisibleIndependentChildPastClippedParent() throws { + func node( + _ index: Int, + type: String, + label: String?, + rect: SnapshotRect, + depth: Int, + parentIndex: Int? + ) -> RawAXNode { + RawAXNode( + index: index, + type: type, + label: label, + identifier: nil, + value: nil, + rect: rect, + enabled: true, + focused: nil, + selected: nil, + hittable: type == "Button", + depth: depth, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + let viewport = CGRect(x: 0, y: 0, width: 100, height: 100) + let options = PresentationOptions( + interactiveOnly: false, depth: 1, scope: nil, raw: false) + let hint = SnapshotPresentation.captureHint(for: options) + let nodes = [ + node( + 0, + type: "Application", + label: "App", + rect: SnapshotRect(x: 0, y: 0, width: 100, height: 100), + depth: 0, + parentIndex: nil + ), + node( + 1, + type: "Other", + label: "Clipped parent", + rect: SnapshotRect(x: 200, y: 20, width: 40, height: 40), + depth: 1, + parentIndex: 0 + ), + node( + 2, + type: "Button", + label: "Projected child", + rect: SnapshotRect(x: 20, y: 20, width: 40, height: 40), + depth: 2, + parentIndex: 1 + ), + ] + let acquisition = SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: false, + effectiveDepth: nil, + viewport: viewport + ) + let presented = try XCTUnwrap( + SnapshotPresentation.present(acquisition, options: options)?.payload.nodes) + + XCTAssertEqual(presented.map(\.label), ["App", "Projected child"]) + XCTAssertEqual(presented.map(\.depth), [0, 1]) + XCTAssertEqual(presented.map(\.parentIndex), [nil, 0]) + + let clippedParentTransition = SnapshotPresentation.regularTraversalTransition( + for: nodes[1], + parentPresentedDepth: 0, + parentTraversal: .root, + hint: hint, + rawDepth: 1, + viewport: viewport, + hasChildren: true, + isDuplicate: false, + policy: .cursorProjected + ) + XCTAssertEqual(clippedParentTransition.presentedDepth, 0) + XCTAssertTrue(clippedParentTransition.traversal.descendantsMayBeVisible) + XCTAssertTrue(clippedParentTransition.shouldVisitChildren) + + let childTransition = SnapshotPresentation.regularTraversalTransition( + for: nodes[2], + parentPresentedDepth: clippedParentTransition.presentedDepth, + parentTraversal: clippedParentTransition.traversal, + hint: hint, + rawDepth: 2, + viewport: viewport, + hasChildren: false, + isDuplicate: false, + policy: .cursorProjected + ) + XCTAssertEqual(childTransition.presentedDepth, 1) + XCTAssertFalse(childTransition.shouldVisitChildren) + + let rawHint = SnapshotPresentation.captureHint( + for: PresentationOptions(interactiveOnly: false, depth: 1, scope: nil, raw: true)) + XCTAssertFalse( + SnapshotPresentation.shouldAcquireChildren( + for: rawHint, + rawDepth: 1, + regularPresentedDepth: 0 + ) + ) + } } #endif diff --git a/contracts/fixtures/ios-snapshot-backends.json b/contracts/fixtures/ios-snapshot-backends.json index 24a5fa9e3..129e55a6d 100644 --- a/contracts/fixtures/ios-snapshot-backends.json +++ b/contracts/fixtures/ios-snapshot-backends.json @@ -4,6 +4,7 @@ "name": "tree", "forceable": true, "supportsRawProjection": true, + "regularDepth": "presented-frontier", "hittable": "geometric-actionability", "deepExtension": "no", "depthLadder": "n/a", @@ -24,6 +25,7 @@ "name": "queries", "forceable": false, "supportsRawProjection": false, + "regularDepth": "flat", "hittable": "geometric-actionability", "deepExtension": "n/a", "depthLadder": "n/a", @@ -37,6 +39,7 @@ "name": "private-ax", "forceable": true, "supportsRawProjection": true, + "regularDepth": "raw-only", "hittable": "geometric-actionability", "deepExtension": "yes", "depthLadder": "yes", diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 2376d68cc..0f0f312c7 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -162,16 +162,26 @@ degenerate semantic carriers stay eligible but are never actionable, while raw projection remains exempt by contract. A violation is a typed `IOS_SNAPSHOT_PRESENTATION_FAILED` capture failure with the named `presentation-failed` snapshot-quality reason, preserved through recovery and the existing TypeScript verdict/warning -contract. The remaining acquisition-side narrowings are declared: -traversal-depth budget cut, the flat query sweep's frameless-element drop (a flat query has no -hierarchy for geometryless semantics to attach to), and the private-AX bridge's device-side node -cap. - -Declared residue: a regular-projection `--depth` request still cuts the traversal at that depth, -while regular presentation emits collapsed depth. A node whose presented depth would be within the -limit can therefore be dropped when structural wrappers put it deeper in the raw tree. The cut is -what keeps `--depth 1` probes cheap; making it complete is the outstanding visible-depth frontier -obligation (#1797), not a property of the current output. +contract. + +The visible-depth frontier completes that migration for unscoped regular captures. `CaptureHint` +keeps raw traversal depth (`--raw --depth`) separate from regular presented depth. A +hierarchy-capable tree capture walks through structural wrappers until each branch ends or reaches +the requested presented depth; regular presentation then applies the depth limit after the shared +fold and eligibility collapse. This keeps shallow probes bounded by the requested presented +frontier without inventing a raw-depth multiplier. Scoped captures remain broad because depth is +relative to the scope root selected in presentation. + +Backend capability declarations are part of the contract: recursive tree supports the presented +frontier, the flat query sweep supports only its root and one presented level, and private AX is +raw-depth-only for regular depth requests until it has an equivalent hierarchy-aware frontier. +The capture plan does not claim deeper regular-depth completeness from a backend that cannot prove +it. Raw depth remains acquisition depth for every backend. + +Acquisition-side limits remain explicit: raw private-AX captures still disclose their bridge-side +node cap, the flat query sweep still drops frameless elements because it has no hierarchy to attach +geometryless semantics to, and the recursive tree still has no raw-depth extension for deep XCTest +trees. Presentation cannot repair any of those acquisition limits. When adding new iOS snapshot behavior, maintainers should first decide which strategy owns it. If a change tries to make regular snapshots fast by dropping visible controls behind a node budget, or diff --git a/examples/test-app/app/_layout.tsx b/examples/test-app/app/_layout.tsx index 8ea767a7b..d49b81755 100644 --- a/examples/test-app/app/_layout.tsx +++ b/examples/test-app/app/_layout.tsx @@ -26,6 +26,7 @@ function RootLayoutContent() { + diff --git a/examples/test-app/app/snapshot-depth.tsx b/examples/test-app/app/snapshot-depth.tsx new file mode 100644 index 000000000..6f4cf290f --- /dev/null +++ b/examples/test-app/app/snapshot-depth.tsx @@ -0,0 +1,5 @@ +import { VisibleDepthScreen } from '../src/screens/VisibleDepthScreen'; + +export default function SnapshotDepthRoute() { + return ; +} diff --git a/examples/test-app/src/screens/VisibleDepthScreen.tsx b/examples/test-app/src/screens/VisibleDepthScreen.tsx new file mode 100644 index 000000000..2a0332c5a --- /dev/null +++ b/examples/test-app/src/screens/VisibleDepthScreen.tsx @@ -0,0 +1,40 @@ +import { Pressable, StyleSheet, View } from 'react-native'; + +import { useAppColors, type AppColors } from '../theme'; + +export function VisibleDepthScreen() { + const colors = useAppColors(); + const styles = createStyles(colors); + + return ( + + + + ); +} + +function createStyles(colors: AppColors) { + return StyleSheet.create({ + frame: { + flex: 1, + paddingHorizontal: 18, + paddingTop: 24, + }, + projectedChild: { + alignItems: 'center', + backgroundColor: colors.accent, + borderRadius: 6, + left: 16, + minHeight: 52, + paddingHorizontal: 18, + paddingVertical: 14, + position: 'absolute', + top: 164, + width: 220, + }, + }); +} diff --git a/src/commands/capture/snapshot.ts b/src/commands/capture/snapshot.ts index e6dd74010..30505c838 100644 --- a/src/commands/capture/snapshot.ts +++ b/src/commands/capture/snapshot.ts @@ -21,7 +21,7 @@ const snapshotCommandDescription = const snapshotBackendCapabilityHelp = Object.entries(SNAPSHOT_BACKEND_CAPABILITIES) .map(([backend, capability]) => { const gaps = capability.knownGaps.map((gap) => `known gap ${gap}`); - return `${backend}: hittable=${capability.hittable}, deep-extension=${capability.deepExtension}, depth-ladder=${capability.depthLadder}${gaps.length > 0 ? `, ${gaps.join(', ')}` : ''}`; + return `${backend}: hittable=${capability.hittable}, regular-depth=${capability.regularDepth}, deep-extension=${capability.deepExtension}, depth-ladder=${capability.depthLadder}${gaps.length > 0 ? `, ${gaps.join(', ')}` : ''}`; }) .join('; '); diff --git a/src/snapshot-quality/backend-capabilities.test.ts b/src/snapshot-quality/backend-capabilities.test.ts index 91c936cbd..907365f94 100644 --- a/src/snapshot-quality/backend-capabilities.test.ts +++ b/src/snapshot-quality/backend-capabilities.test.ts @@ -10,6 +10,7 @@ type SnapshotBackendParityFixture = { name: string; forceable: boolean; supportsRawProjection: boolean; + regularDepth: string; hittable: string; deepExtension: string; depthLadder: string; @@ -87,6 +88,7 @@ test('iOS snapshot registry classifies every backend and conformance target', () expect(SNAPSHOT_BACKEND_CAPABILITIES.tree).toMatchObject({ forceable: true, supportsRawProjection: true, + regularDepth: 'presented-frontier', hittable: 'geometric-actionability', deepExtension: 'no', depthLadder: 'n/a', @@ -94,6 +96,7 @@ test('iOS snapshot registry classifies every backend and conformance target', () expect(SNAPSHOT_BACKEND_CAPABILITIES['private-ax']).toMatchObject({ forceable: true, supportsRawProjection: true, + regularDepth: 'raw-only', hittable: 'geometric-actionability', deepExtension: 'yes', depthLadder: 'yes', @@ -101,6 +104,7 @@ test('iOS snapshot registry classifies every backend and conformance target', () expect(SNAPSHOT_BACKEND_CAPABILITIES.queries).toMatchObject({ forceable: false, supportsRawProjection: false, + regularDepth: 'flat', hittable: 'geometric-actionability', }); }); @@ -116,6 +120,7 @@ test('Swift and TypeScript snapshot backend declarations match the parity table' expect(capability).toMatchObject({ forceable: backend.forceable, supportsRawProjection: backend.supportsRawProjection, + regularDepth: backend.regularDepth, hittable: backend.hittable, deepExtension: backend.deepExtension, depthLadder: backend.depthLadder, diff --git a/src/snapshot-quality/backend-capabilities.ts b/src/snapshot-quality/backend-capabilities.ts index 84efabd4c..16c741d8d 100644 --- a/src/snapshot-quality/backend-capabilities.ts +++ b/src/snapshot-quality/backend-capabilities.ts @@ -6,9 +6,11 @@ import type { /** #1933: every iOS backend publishes this shared predicate in the wire `hittable` field. */ type SnapshotBackendHittable = 'geometric-actionability'; type SnapshotBackendSupport = 'yes' | 'no' | 'n/a'; +type SnapshotRegularDepthCapability = 'presented-frontier' | 'flat' | 'raw-only'; type SnapshotBackendCapability = { supportsRawProjection: boolean; + regularDepth: SnapshotRegularDepthCapability; hittable: SnapshotBackendHittable; deepExtension: SnapshotBackendSupport; depthLadder: SnapshotBackendSupport; @@ -29,6 +31,7 @@ export const SNAPSHOT_BACKEND_CAPABILITIES = { tree: { forceable: true, supportsRawProjection: true, + regularDepth: 'presented-frontier', hittable: 'geometric-actionability', deepExtension: 'no', depthLadder: 'n/a', @@ -37,6 +40,7 @@ export const SNAPSHOT_BACKEND_CAPABILITIES = { queries: { forceable: false, supportsRawProjection: false, + regularDepth: 'flat', hittable: 'geometric-actionability', deepExtension: 'n/a', depthLadder: 'n/a', @@ -45,6 +49,7 @@ export const SNAPSHOT_BACKEND_CAPABILITIES = { 'private-ax': { forceable: true, supportsRawProjection: true, + regularDepth: 'raw-only', hittable: 'geometric-actionability', deepExtension: 'yes', depthLadder: 'yes', diff --git a/test/integration/ios-simulator-e2e/behavior-coverage.ts b/test/integration/ios-simulator-e2e/behavior-coverage.ts index 64ef54f72..6c89a1f44 100644 --- a/test/integration/ios-simulator-e2e/behavior-coverage.ts +++ b/test/integration/ios-simulator-e2e/behavior-coverage.ts @@ -6,6 +6,7 @@ export type IosSimulatorBehaviorId = | 'long-list-scroll-recovery' | 'modal-open-close' | 'permission-state-recovery' + | 'regular-visible-depth-frontier' | 'text-entry-keyboard-lifecycle'; type BehaviorCoverageEntry = @@ -54,6 +55,12 @@ export const IOS_SIMULATOR_BEHAVIOR_COVERAGE = { level: 'live', owner: 'full:lifecycle-system', }, + 'regular-visible-depth-frontier': { + assertion: + 'regular depth 1 retains an independently projected child after its clipped structural parent is removed, while raw depth remains traversal-bounded', + level: 'live', + owner: 'smoke:regular-visible-depth-frontier', + }, 'interrupted-system-ui-flow': { assertion: 'Home and app switcher expose distinct system pixels before fixture restoration', level: 'live', diff --git a/test/integration/ios-simulator-e2e/live-assertions.ts b/test/integration/ios-simulator-e2e/live-assertions.ts index 0a289f8bc..0df1b882b 100644 --- a/test/integration/ios-simulator-e2e/live-assertions.ts +++ b/test/integration/ios-simulator-e2e/live-assertions.ts @@ -14,10 +14,12 @@ import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; export { assertFilesDiffer, assertJsonContains, assertMp4File, assertNonEmptyFile }; -export const { assertElementText, assertWaitText, capturePng } = createLiveDeviceAssertions< - IosSimulatorBehaviorId, - LiveContext ->(runStep, verifyCommand, PUBLIC_COMMANDS.wait); +export const { assertElementText, assertWaitSelector, assertWaitText, capturePng } = + createLiveDeviceAssertions( + runStep, + verifyCommand, + PUBLIC_COMMANDS.wait, + ); const SCROLL_SEARCH_ATTEMPTS = 4; // A stalled capture says nothing about where the element is, so it must not consume the scroll diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts index 09e521f80..87ca2bb22 100644 --- a/test/integration/ios-simulator-e2e/live-automation-scenario.ts +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -237,7 +237,7 @@ async function assertClearStateLaunchUrl(context: LiveContext): Promise { await assertElementText(context, 'id="automation-event-payload"', '{"source":"deep-link"}'); } -async function acceptDeepLinkConfirmationIfPresent(context: LiveContext): Promise { +export async function acceptDeepLinkConfirmationIfPresent(context: LiveContext): Promise { const destination = await runStep( context, 'wait for deep-link destination before inspecting system UI', @@ -246,7 +246,10 @@ async function acceptDeepLinkConfirmationIfPresent(context: LiveContext): Promis ); if (destination.status === 0) return; - const alert = await runStep(context, 'inspect delayed deep-link system alert', ['alert', 'get']); + const alert = await runStep(context, 'inspect delayed deep-link system alert', ['alert', 'get'], { + allowFailure: true, + }); + if (alert.status !== 0) return; const alertInfo = alert.json?.data; assert.match(String(alertInfo?.message), /^Open in\b/, JSON.stringify(alert.json)); assert.ok( diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index 44d8f41b8..a88d60dcc 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -16,6 +16,7 @@ import { } from './live-assertions.ts'; import { assertAutomationInput } from './live-automation-scenario.ts'; import { assertDeviceLifecycle } from './live-device-lifecycle.ts'; +import { assertRegularVisibleDepthFrontier } from './live-snapshot-depth-frontier.ts'; import { assertLifecycleAndSystem, assertObservabilityAndArtifacts, @@ -75,6 +76,7 @@ const LIVE_SCENARIOS = bindIosSimulatorScenarios({ inventoryInstall: assertInventoryAndInstall, lifecycleSystem: assertLifecycleAndSystem, observabilityArtifacts: assertObservabilityAndArtifacts, + snapshotDepthFrontier: assertRegularVisibleDepthFrontier, }); export async function runIosSimulatorE2E(): Promise { diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts new file mode 100644 index 000000000..864033c9e --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -0,0 +1,136 @@ +import assert from 'node:assert/strict'; + +import { assertWaitSelector, assertWaitText } from './live-assertions.ts'; +import { acceptDeepLinkConfirmationIfPresent } from './live-automation-scenario.ts'; +import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; + +const VISIBLE_DEPTH_DEEP_LINK = 'agent-device-test-app:///snapshot-depth'; +const CHILD_ID = 'visible-depth-projected-child'; + +type SnapshotNode = { + depth?: unknown; + identifier?: unknown; + index?: unknown; + label?: unknown; + parentIndex?: unknown; +}; + +export async function assertRegularVisibleDepthFrontier(context: LiveContext): Promise { + await runStep(context, 'open regular visible-depth fixture', [ + 'open', + context.appId, + '--relaunch', + '--launch-url', + VISIBLE_DEPTH_DEEP_LINK, + ]); + await acceptDeepLinkConfirmationIfPresent(context); + // Wait for the target itself so the depth assertion is about the frontier, not route readiness. + await assertWaitSelector(context, `id="${CHILD_ID}"`); + + const regular = await runStep(context, 'capture regular visible-depth frontier', [ + 'snapshot', + '--depth', + '1', + ]); + assertSnapshotBackend(regular, 'regular depth-1 snapshot'); + const regularNodes = snapshotNodes(regular); + const regularRoot = requireRoot(regularNodes, 'regular depth-1 snapshot'); + const projectedChild = requireIdentifier(regularNodes, CHILD_ID, 'regular depth-1 snapshot'); + assert.equal( + projectedChild.depth, + 1, + `projected child should occupy presented depth 1: ${JSON.stringify(regular)}`, + ); + assert.equal( + projectedChild.parentIndex, + regularRoot.index, + `projected child should be reparented to the presented root: ${JSON.stringify(regular)}`, + ); + assert.ok( + regularNodes.every((node) => numericDepth(node) <= 1), + `regular --depth 1 exceeded the presented frontier: ${JSON.stringify(regular)}`, + ); + + const rawFull = await runStep(context, 'capture full raw visible-depth tree', [ + 'snapshot', + '--raw', + ]); + assertSnapshotBackend(rawFull, 'full raw visible-depth snapshot'); + const rawFullNodes = snapshotNodes(rawFull); + const rawChild = requireIdentifier(rawFullNodes, CHILD_ID, 'full raw visible-depth snapshot'); + assert.ok( + numericDepth(rawChild) > 1, + `raw projected child must remain below traversal depth 1: ${JSON.stringify(rawFull)}`, + ); + + const rawDepthOne = await runStep(context, 'capture raw depth-bounded visible-depth tree', [ + 'snapshot', + '--raw', + '--depth', + '1', + ]); + assertSnapshotBackend(rawDepthOne, 'raw depth-1 visible-depth snapshot'); + const rawDepthOneNodes = snapshotNodes(rawDepthOne); + assert.equal( + rawDepthOneNodes.some((node) => node.identifier === CHILD_ID), + false, + `raw --depth 1 must omit the raw depth-2 child: ${JSON.stringify(rawDepthOne)}`, + ); + assert.ok( + rawDepthOneNodes.every((node) => numericDepth(node) <= 1), + `raw --depth 1 exceeded the acquisition frontier: ${JSON.stringify(rawDepthOne)}`, + ); + + await runStep(context, 'restore fixture home after visible-depth capture', [ + 'open', + context.appId, + '--relaunch', + ]); + await assertWaitText(context, 'Agent Device Tester'); + verifyBehavior( + context, + 'regular-visible-depth-frontier', + 'public regular depth 1 keeps a raw-deep visible child at presented depth 1 while raw depth remains traversal-bounded', + ); +} + +function snapshotNodes(result: { json?: any }): SnapshotNode[] { + const nodes = result.json?.data?.nodes; + assert.ok( + Array.isArray(nodes), + `snapshot response did not contain nodes: ${JSON.stringify(result)}`, + ); + return nodes as SnapshotNode[]; +} + +function requireIdentifier(nodes: SnapshotNode[], identifier: string, description: string) { + const node = nodes.find((candidate) => candidate.identifier === identifier); + assert.ok(node, `${description} missing ${identifier}: ${JSON.stringify(nodes)}`); + return node; +} + +function requireRoot(nodes: SnapshotNode[], description: string) { + const root = nodes.find( + (node) => + numericDepth(node) === 0 && (node.parentIndex === undefined || node.parentIndex === null), + ); + assert.ok(root, `${description} missing a presented root: ${JSON.stringify(nodes)}`); + return root; +} + +function numericDepth(node: SnapshotNode): number { + assert.equal( + typeof node.depth, + 'number', + `snapshot node has no numeric depth: ${JSON.stringify(node)}`, + ); + return node.depth as number; +} + +function assertSnapshotBackend(result: { json?: any }, description: string): void { + assert.equal( + result.json?.data?.snapshotQuality?.backend, + 'tree', + `${description} must exercise the recursive tree backend: ${JSON.stringify(result)}`, + ); +} diff --git a/test/integration/ios-simulator-e2e/scenarios.ts b/test/integration/ios-simulator-e2e/scenarios.ts index a55ce3b3f..da101d578 100644 --- a/test/integration/ios-simulator-e2e/scenarios.ts +++ b/test/integration/ios-simulator-e2e/scenarios.ts @@ -11,7 +11,8 @@ type ScenarioRunnerKey = | 'formInput' | 'inventoryInstall' | 'lifecycleSystem' - | 'observabilityArtifacts'; + | 'observabilityArtifacts' + | 'snapshotDepthFrontier'; type ScenarioDefinition = IosSimulatorScenario & { runner: ScenarioRunnerKey; @@ -21,6 +22,11 @@ const SCENARIO_DEFINITIONS: readonly ScenarioDefinition[] = [ { id: 'smoke:inventory-install', runner: 'inventoryInstall', tier: 'smoke' }, { id: 'smoke:automation-input', runner: 'automationInput', tier: 'smoke' }, { id: 'smoke:form-input', runner: 'formInput', tier: 'smoke' }, + { + id: 'smoke:regular-visible-depth-frontier', + runner: 'snapshotDepthFrontier', + tier: 'smoke', + }, { id: 'smoke:capture-close', runner: 'captureClose', tier: 'smoke' }, { id: 'full:lifecycle-system', runner: 'lifecycleSystem', tier: 'full' }, {