test: classify the runner XCTests — pure decisions to a macOS host lane, simulator semantics gated os(iOS) (#1781 A7)

.github/workflows/ci.yml

@@ -55,14 +55,41 @@ jobs:
             exit 1
           fi
 
-  swift-runner-unit-compile:
-    name: Swift Runner Unit Compile
+  # The runner's pure-decision XCTests, on the macOS host with no simulator (#1781 A7). This
+  # job already compiled the macOS runner with the unit-test flag on every PR (as "Swift
+  # Runner Unit Compile") and threw the bundle away; running it costs one
+  # `test-without-building` on the host. Which tests that reaches is not a list: it is
+  # everything the macOS build compiles, and the simulator-only tests keep themselves out
+  # with an `os(iOS)` guard (see the classification note in RunnerTests.swift). The
+  # `-skip-testing:` entry is validated by `pnpm check:xctest-selection` for the same
+  # reason the nightly's is: `RunnerTests/testCommand` is the runner's 24-hour server
+  # entry point, and a typo re-arms the hang. The lane's reporter asserts the executed
+  # count equals the reach the check derives from source, so a build without the compile
+  # flag or a guard that compiles a file out reads as red rather than as a smaller green.
+  #
+  #   - Catches: regressions in the ~130 host-runnable tests on every PR, in about a
+  #     minute, independent of ios.yml — which is fifteen minutes long, cancelled ~19% of
+  #     the time, and names only a subset of them.
+  #   - Evidence: the nightly's first execution found #1812 in exactly this class of test
+  #     (a geometry guard); the same tests pass on this lane's platform.
+  #   - Cost: ~1 minute more on a job that already occupies a macOS slot for the build;
+  #     no simulator boot; no new job.
+  #   - Kill criterion: if the host run proves flaky where the simulator run of the same
+  #     tests is not (a platform-branch difference this classification missed), gate the
+  #     offender `os(iOS)` and it returns to the simulator lanes; if that happens to more
+  #     than a handful, the host lane is the wrong tool and goes.
+  swift-runner-host-tests:
+    name: Swift Runner Host XCTests
     runs-on: macos-26
     timeout-minutes: 20
     # Was an inline `VAR=1 pnpm gate …` prefix on the action's build-command input. The
     # input is a gate id now, so the variable lives where ios.yml already puts it: the job.
     env:
       AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
+      AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/swift-runner-unit-derived
+      RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-host/RunnerTests.xcresult
+      RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-host/test-results-summary.json
+      XCTEST_LANE: host
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -73,12 +100,58 @@ jobs:
       - name: Restore and compile Swift runner unit-test surface
         uses: ./.github/actions/setup-apple-runner-build
         with:
-          derived-path: ${{ github.workspace }}/.tmp/swift-runner-unit-derived
+          derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
           cache-key-prefix: swift-runner-unit
           gate: swift-runner-macos
           xcuitest-platform: macos
           xcuitest-destination: platform=macOS,arch=arm64
 
+      - name: Run the host-runnable runner XCTests
+        run: |
+          set -euo pipefail
+          XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
+          test -n "$XCTESTRUN_PATH"
+          mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")"
+          xcodebuild test-without-building \
+            -xctestrun "$XCTESTRUN_PATH" \
+            -destination 'platform=macOS,arch=arm64' \
+            -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \
+            -resultBundlePath "$RESULT_BUNDLE_PATH"
+
+      # Same shape as xctest-nightly.yml: extract is best-effort, the report step asserts.
+      - name: Extract the test-results summary
+        if: always()
+        run: |
+          set -uo pipefail
+          [ -d "$RESULT_BUNDLE_PATH" ] || exit 0
+          xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" --compact \
+            > "$RESULT_SUMMARY_PATH" 2>/dev/null ||
+            xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" \
+              > "$RESULT_SUMMARY_PATH" 2>/dev/null ||
+            rm -f "$RESULT_SUMMARY_PATH"
+          exit 0
+
+      - name: Report the run and assert it executed the source-derived set
+        if: always()
+        run: |
+          set -euo pipefail
+          if [ ! -s "$RESULT_SUMMARY_PATH" ]; then
+            echo 'No usable test-results summary was produced, so nothing here can show a test ran.'
+            echo 'The suite step above failed before or during the run; read its log first.'
+            echo '### iOS runner host XCTest lane' >> "$GITHUB_STEP_SUMMARY"
+            echo 'No result-bundle summary; see the run log and the uploaded bundle.' >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+          node --experimental-strip-types scripts/xctest-run-summary.ts
+
+      - name: Upload result bundle
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: xctest-host-results-${{ github.run_id }}-${{ github.run_attempt }}
+          path: .tmp/xctest-host
+          if-no-files-found: warn
+
   lint:
     name: Lint & Format
     runs-on: ubuntu-latest

.github/workflows/xctest-nightly.yml

@@ -15,22 +15,27 @@ name: XCTest Nightly
 #
 # Under the lane rule in #1781:
 #
-#   - Catches: regressions in the runner-semantics tests the PR list leaves out — selector
-#     matching, snapshot traversal, command journal, sequence execution, tv remote, keyboard,
-#     scroll/gesture policy. Nothing else executes them. Note the reachable set is smaller
-#     than the declared set: `testCommand` is skipped, and the `…OnTvOS` tests sit behind
-#     `#if os(tvOS)` and cannot execute on this lane's iOS Simulator destination.
-#   - Evidence: the list has never been re-derived since it was written by hand; the tests it
-#     omits have been dark for their whole life, so their current state is unmeasured. The
-#     first red night is the evidence this lane exists to produce.
+#   - Catches: regressions in the simulator-only tests (guard `… && os(iOS)`: they launch
+#     the host app, route through SpringBoard, or assert an iOS-only branch) that ios.yml's
+#     PR list leaves out. Since the A7 classification the pure-decision tests also run on
+#     the macOS host lane on every PR (ci.yml "Swift Runner Host XCTests"), so this lane's
+#     unique reach is the simulator-only remainder — `pnpm check:xctest-selection` prints
+#     both numbers. `testCommand` is skipped, and the tests guarded to macOS/tvOS only
+#     cannot execute on this lane's iOS Simulator destination (the host lane runs them).
+#   - Evidence: the first execution found #1812 in the dark set (2 of 155 red, both real);
+#     the other ~114 never-run tests all passed.
 #   - Cost: this is a public repository, so nothing here is billed. The real cost is one macOS
 #     concurrency slot per night. The Apple runner build shares ios.yml's cache (identical
 #     inputs and build variant), but scheduled runs only see DEFAULT-BRANCH caches, so the hit
 #     depends on ios.yml's `push: main` run having populated the cache at the same source
 #     hash — on a miss this lane also pays a full runner build. Adds nothing to any PR.
-#   - Kill criterion: when A7's classification lands — pure Swift decision tests moved to a
-#     fast unit target, stale tests deleted, the rest kept on the simulator — this lane goes
-#     if the surviving set is small enough to run on every PR.
+#   - Kill criterion: what ONLY this lane reaches is 11 tests / ~42 s of simulator time
+#     (measured from the first night's xcresult) — the app-launching, SpringBoard-probing
+#     class. Once ios.yml runs the whole iOS-compiled bundle instead of its hand-written
+#     list — the follow-up stacked on the classification — every test reaches a PR lane and
+#     this lane goes. That trade is not free: those 11 are the flakiest tests in the suite,
+#     and they would land on a blocking 15-minute gate that is already cancelled ~19% of the
+#     time, so measure their flake rate here before moving them.
 #
 # Scheduled + manual only. It is deliberately NOT a PR gate: ios.yml is already over half of
 # this repo's CI spend (#1781 A9), and moving the unrun set onto it would make that worse for
@@ -79,6 +84,7 @@ jobs:
       AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
       RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/RunnerTests.xcresult
       RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/test-results-summary.json
+      XCTEST_LANE: nightly
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -155,10 +161,12 @@ jobs:
       # Reports the run AND is the lane's own liveness check. `xcodebuild` exits 0 when a
       # selection matches nothing, so "green" and "ran no tests" are the same log tail: a
       # build variant without the unit-test compile flag, an empty test plan, or a renamed
-      # target would all read as a healthy night. Asserting the run executed tests is what
-      # tells those apart. The failure list is capped so a badly red night cannot blow the
-      # 1 MiB job-summary limit and lose the headline with it.
-      - name: Report the run and assert it executed tests
+      # target would all read as a healthy night. The reporter asserts the executed count
+      # equals what `pnpm check:xctest-selection` derives this lane reaches from the `#if`
+      # guards (XCTEST_LANE above), so a guard that compiles a file of tests out of iOS reads
+      # as red too. The failure list is capped so a badly red night cannot blow the 1 MiB
+      # job-summary limit and lose the headline with it.
+      - name: Report the run and assert it executed the source-derived set
         if: always()
         run: |
           set -euo pipefail

AGENTS.md

@@ -90,6 +90,9 @@ the new thing — never to suppress or allowlist it.
 - every command declares a timeout policy on its descriptor (timeout-policy completeness test)
 - TS/Swift rule parity: golden tables under `contracts/fixtures/`, consumed by vitest and the gated
   XCTest — change the rule only via the table
+- runner XCTest reach: `check:xctest-selection` evaluates each test's `#if` guard per platform and
+  fails when a declared method reaches no lane (host/PR/nightly) — classify with the guard, do not
+  list by hand
 - cross-command apple-leak guard; folder DAG/import lint (zero value-import cycles, zero target-spine
   back-edges); fallow (dead code, duplication, complexity)
 

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift

@@ -95,19 +95,23 @@ extension RunnerTests {
     XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown))
   }
 
-  // tvOS has no SpringBoard host, so both the snapshot and alert-resolution paths
-  // must resolve without probing com.apple.springboard (#1351).
-  #if os(tvOS)
-  func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardOnTvOS() {
+  // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return
+  // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane
+  // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same
+  // flag on macOS, so the host lane runs the real branch on every PR.
+  //
+  // Its former sibling `testBlockingSystemAlertSnapshotIsNilOnTvOS` is deleted rather than
+  // widened: `blockingSystemAlertSnapshot` is `#if os(macOS) return nil`, so on the only lane
+  // that could run it the assertion would pin a compile-time literal — a green that no change
+  // to the runner could turn red. The runtime gate it meant to cover is this test's subject,
+  // and the nil it returns on macOS is the compiler's business, not a test's.
+  #if os(tvOS) || os(macOS)
+  func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardHost() {
     guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else {
-      XCTFail("tvOS blocking system-modal resolution must be .absent")
+      XCTFail("blocking system-modal resolution must be .absent without a SpringBoard host")
       return
     }
   }
-
-  func testBlockingSystemAlertSnapshotIsNilOnTvOS() {
-    XCTAssertNil(blockingSystemAlertSnapshot(deadline: .distantFuture))
-  }
   #endif
 }
 #endif

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -481,6 +481,11 @@ extension RunnerTests {
     XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse))
   }
 
+  // Simulator-only from here to the matching #endif: these launch the host app, route through
+  // SpringBoard, or assert the iOS-only alert/system-modal branches. Tests outside the
+  // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host
+  // lane (ci.yml) — see the classification convention in RunnerTests.swift.
+#if os(iOS)
   func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws {
     app.launch()
     currentApp = app
@@ -681,6 +686,10 @@ extension RunnerTests {
     XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence))
   }
 
+  // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is
+  // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time
+  // literal and no edit to the iOS body could make it red. Its five siblings above and below
+  // are gated for the same reason.
   func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws {
     currentApp = nil
     currentBundleId = nil
@@ -727,6 +736,7 @@ extension RunnerTests {
 
     XCTAssertTrue(shouldSkipAppActivationPreflight(alert))
   }
+#endif
 
   func testExecuteDispatchedReturnsBusyBeforeMainThreadFastPath() throws {
     let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#)
@@ -760,6 +770,7 @@ extension RunnerTests {
     XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true)
   }
 
+#if os(iOS)
   func testAlertResolutionCannotBypassRequestedDeadline() throws {
     final class ResultBox {
       var error: Error?
@@ -821,6 +832,7 @@ extension RunnerTests {
     releaseResolution.signal()
     wait(for: [resolutionExited], timeout: 1)
   }
+#endif
 
   func testRunMainThreadWorkExecutesOffMainCallerOnMainThread() {
     final class ResultBox {

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -616,6 +616,11 @@ extension RunnerTests {
     XCTAssertEqual(payload.truncated, true)
     XCTAssertEqual(payload.runnerFatal, true)
     XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason)
+    // The planned terminal result carries the structured verdict like every other planned
+    // snapshot — downstream sparse handling keys off it, not off node shapes.
+    XCTAssertEqual(payload.snapshotQuality?.state, "sparse")
+    XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected")
+    XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage)
     XCTAssertNil(currentApp)
     XCTAssertNil(currentBundleId)
   }
@@ -651,6 +656,9 @@ extension RunnerTests {
     XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0)
   }
 
+  // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the
+  // timeout/penalty/drain machinery below only exists on the iOS branch.
+#if os(iOS)
   /// Regression for #1244/#1248: drives the bounded system-modal probe through a real,
   /// production-only command entry point (`snapshotFast` or `snapshotRaw` -- see the two test
   /// methods below), not `boundedBlockingSystemAlertSnapshot` directly, with
@@ -783,6 +791,7 @@ extension RunnerTests {
       try self.snapshotRaw(app: target, options: options)
     }
   }
+#endif
 
   func testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied() {
     // The #1244 recovery shape: the modal probe abandoned an XCTest query that is still grinding on

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

@@ -966,24 +966,5 @@ extension RunnerTests {
       )
     )
   }
-
-  func testSnapshotAccessibilityUnavailableCarriesSparseVerdict() {
-    currentApp = app
-    currentBundleId = "com.example.app"
-    defer {
-      currentApp = nil
-      currentBundleId = nil
-    }
-    let payload = snapshotAccessibilityUnavailable(
-      failure: SnapshotCaptureFailure(
-        code: "IOS_AX_SNAPSHOT_FAILED",
-        message: "kAXErrorIllegalArgument",
-        hint: "use screenshot"
-      )
-    )
-    XCTAssertEqual(payload.runnerFatal, true)
-    XCTAssertEqual(payload.snapshotQuality?.state, "sparse")
-    XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected")
-  }
 }
 #endif

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -119,6 +119,17 @@ final class RunnerTests: XCTestCase {
   // Bounds the pre-plan SpringBoard system-modal probe, which can otherwise grind for tens of
   // seconds on remote-hosted consent dialogs and bypass the plan budget (#1244).
   let systemModalProbeBudget: TimeInterval = 4
+  // In-bundle unit tests (every `func test…` except `testCommand` below) compile only under
+  // `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` and are classified by their `#if` guard (#1781 A7):
+  //   - `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone: a pure runner decision (rule table,
+  //     geometry, parser, policy, journal, dispatch bookkeeping) that needs no launched app.
+  //     Runs on the macOS host lane on every PR (ci.yml, no simulator) and on the iOS lanes.
+  //   - `… && os(iOS)` (or a nested `#if os(iOS)`): runner/XCTest semantics — launches the
+  //     host app, routes through SpringBoard, swizzles XCUIApplication, or asserts an
+  //     iOS-only branch. Simulator lanes only (ios.yml PR list, xctest-nightly.yml).
+  // `pnpm check:xctest-selection` derives each lane's reachable set from these guards and
+  // fails when a declared test is reachable by no lane, so a test gated to a platform nothing
+  // runs (the old tvOS-only pair) cannot go dark silently.
   #if AGENT_DEVICE_RUNNER_UNIT_TESTS
   // #1605 merge gate: deterministic live reproduction of the field ambiguity —
   // a tap whose coordinate activation LANDS while XCTest bookkeeping records a