ci: shard the test run and deflake timing-sensitive suites

.github/workflows/ci.yml

@@ -15,13 +15,18 @@ concurrency:
 
 jobs:
   tests:
+    name: tests (shard ${{ matrix.shard }})
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     # The hooks target macOS /bin/bash 3.2; macos-15 ships it as /bin/bash.
     runs-on: macos-15
-    timeout-minutes: 30
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v5
       - name: Run test suites
         # /bin first so run.sh's unqualified `bash` resolves to 3.2, not brew bash 5.
         run: |
           export PATH="/bin:$PATH"
-          /bin/bash tests/run.sh
+          /bin/bash tests/run.sh --shard ${{ matrix.shard }}/4

tests/lease.sh

@@ -1628,7 +1628,7 @@ t58_reclaimer_diagnostics_include_wait_budget() (
   : > "$state.release"
   wait "$holder" || return 1
   [ "$rc" -eq 11 ] || { echo "reclaimer contention rc=$rc want 11"; return 1; }
-  printf '%s\n' "$out" | grep -q 'wait_budget=0s wait_elapsed=0s' ||
+  printf '%s\n' "$out" | grep -Eq 'wait_budget=0s wait_elapsed=[0-9]+s' ||
     { echo "reclaimer contention omitted wait accounting: $out"; return 1; }
 )
 

tests/liveness.sh

@@ -638,8 +638,8 @@ t8_verifier_boundaries() {
   ) > "$state/output" 2>&1 &
   pid=$!
   set +m
-  wait_bounded "$pid" 10
-  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "verifier exceeded 10s bound"; return 1; }
+  wait_bounded "$pid" 30
+  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "verifier exceeded 30s bound"; return 1; }
   [ "$WAIT_RC" -eq 12 ] || { echo "rc=$WAIT_RC want 12"; return 1; }
   grep -q 'verification timed out after 3s' "$state/output" ||
     { echo "attempt evidence omitted verifier timeout"; return 1; }
@@ -670,8 +670,8 @@ t8_verifier_boundaries() {
   ) > "$state/output" 2>&1 &
   pid=$!
   set +m
-  wait_bounded "$pid" 7
-  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "verifier lease check exceeded 7s bound"; return 1; }
+  wait_bounded "$pid" 25
+  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "verifier lease check exceeded 25s bound"; return 1; }
   ownership=$(sed -n '1p' "$state/ownership" 2>/dev/null)
   [ "$ownership" = 1 ] ||
     { echo "verifier inherited Lease interval capability state=${ownership:-no result}"; return 1; }
@@ -696,8 +696,8 @@ t8_verifier_boundaries() {
   ) > "$state/output" 2>&1 &
   pid=$!
   set +m
-  wait_bounded "$pid" 7
-  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "successful verifier reap exceeded 7s bound"; return 1; }
+  wait_bounded "$pid" 25
+  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "successful verifier reap exceeded 25s bound"; return 1; }
   [ "$WAIT_RC" -eq 0 ] || { echo "successful verifier reap rc=$WAIT_RC want 0"; return 1; }
   child=$(sed -n '1p' "$state/child.pid" 2>/dev/null)
   [ -n "$child" ] || { echo "successful verifier child pid missing"; return 1; }
@@ -725,8 +725,8 @@ t8_verifier_boundaries() {
   ) > "$state/output" 2>&1 &
   pid=$!
   set +m
-  wait_bounded "$pid" 7
-  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "verifier root cwd exceeded 7s bound"; return 1; }
+  wait_bounded "$pid" 25
+  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "verifier root cwd exceeded 25s bound"; return 1; }
   [ "$WAIT_RC" -eq 0 ] ||
     { echo "verifier root cwd rc=$WAIT_RC want 0: $(tr '\n' ' ' < "$state/output")"; return 1; }
   grep -q '^MAESTRO_FINAL: LOOP VERIFIED_DONE rc=0$' "$state/output" ||
@@ -775,8 +775,8 @@ t8_verifier_boundaries() {
     { echo "verification heartbeat did not advance: first=$first second=$second"; return 1; }
   kill -0 "$pid" 2>/dev/null ||
     { echo "verifier ended before heartbeat observation"; return 1; }
-  wait_bounded "$pid" 10
-  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "heartbeat verifier exceeded 10s"; return 1; }
+  wait_bounded "$pid" 30
+  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "heartbeat verifier exceeded 30s"; return 1; }
   [ "$WAIT_RC" -eq 0 ] || { echo "heartbeat verifier rc=$WAIT_RC"; return 1; }
 }
 
@@ -800,8 +800,8 @@ t9_terminal_at_deadline_harvests() {
   ) > "$state/output" 2>&1 &
   pid=$!
   set +m
-  wait_bounded "$pid" 7
-  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "completed dispatch exceeded 7s bound"; return 1; }
+  wait_bounded "$pid" 25
+  [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "completed dispatch exceeded 25s bound"; return 1; }
   if grep -q '^cancel task-fake0000-aaaaaa$' "$state/calls.log"; then
     echo "completed job was cancelled"
     return 1

tests/run.sh

@@ -56,30 +56,146 @@ terminate_suite_tree() { # process-group leader
   for descendant in $descendants; do kill -KILL "$descendant" 2>/dev/null || :; done
 }
 
-if [ "$#" -gt 0 ]; then
-  suites="$*"
+suite_inventory() {
+    local suite_path suite
+    for suite_path in "$TEST_DIR"/*.sh; do
+        suite="${suite_path##*/}"
+        [ "$suite" = run.sh ] || printf '%s\n' "$suite"
+    done | LC_ALL=C sort
+}
+
+suite_cost() {
+    case "$1" in
+        liveness.sh) printf '%s\n' 166 ;;
+        stop-report.sh) printf '%s\n' 141 ;;
+        lease.sh) printf '%s\n' 83 ;;
+        bounded-calls.sh) printf '%s\n' 58 ;;
+        detection.sh) printf '%s\n' 39 ;;
+        job-lock.sh) printf '%s\n' 30 ;;
+        scout.sh) printf '%s\n' 25 ;;
+        discussion.sh) printf '%s\n' 22 ;;
+        commit-invariance.sh) printf '%s\n' 9 ;;
+        provenance-edge.sh) printf '%s\n' 8 ;;
+        orphan-lifecycle.sh) printf '%s\n' 8 ;;
+        gate.sh) printf '%s\n' 7 ;;
+        shared-git-dir.sh) printf '%s\n' 4 ;;
+        preflight.sh) printf '%s\n' 4 ;;
+        manual-check-and-submodules.sh) printf '%s\n' 4 ;;
+        install.sh) printf '%s\n' 4 ;;
+        runner-timeout.sh) printf '%s\n' 2 ;;
+        model-selector.sh) printf '%s\n' 2 ;;
+        *) printf '%s\n' 60 ;;
+    esac
+}
+
+usage() {
+    printf 'usage: %s [--list | --shard I/N | suite ...]\n' "$0" >&2
+}
+
+inventory=$(suite_inventory)
+inventory_suites=()
+while IFS= read -r suite; do
+    [ -n "$suite" ] || continue
+    inventory_suites[${#inventory_suites[@]}]="$suite"
+done <<EOF
+$inventory
+EOF
+
+suites=()
+shard_index=''
+shard_count=''
+if [ "$#" -eq 0 ]; then
+    suites=("${inventory_suites[@]}")
 else
-  suites="gate.sh
-install.sh
-model-selector.sh
-preflight.sh
-lease.sh
-job-lock.sh
-scout.sh
-liveness.sh
-stop-report.sh
-bounded-calls.sh
-runner-timeout.sh
-shared-git-dir.sh
-provenance-edge.sh
-discussion.sh
-detection.sh
-orphan-lifecycle.sh
-commit-invariance.sh
-manual-check-and-submodules.sh"
+    case "$1" in
+        --list)
+            [ "$#" -eq 1 ] || { usage; exit 2; }
+            printf '%s\n' "${inventory_suites[@]}"
+            exit 0
+            ;;
+        --shard)
+            if [ "$#" -ne 2 ]; then
+                usage
+                exit 2
+            fi
+            shard_spec="$2"
+            case "$shard_spec" in
+                */*) ;;
+                *)
+                    usage
+                    exit 2
+                    ;;
+            esac
+            case "$shard_spec" in
+                ''|*/*/*|/*|*/|*[!0-9/]*)
+                    usage
+                    exit 2
+                    ;;
+            esac
+            shard_index_text="${shard_spec%%/*}"
+            shard_count_text="${shard_spec#*/}"
+            shard_index=$((10#$shard_index_text))
+            shard_count=$((10#$shard_count_text))
+            if [ "$shard_index" -lt 1 ] ||
+                [ "$shard_count" -lt 1 ] ||
+                [ "$shard_index" -gt "$shard_count" ]; then
+                usage
+                exit 2
+            fi
+            ;;
+        *)
+            suites=("$@")
+            ;;
+    esac
+fi
+
+if [ -n "$shard_index" ]; then
+    ordered_suites=()
+    ordered_costs=()
+    for suite in "${inventory_suites[@]}"; do
+        cost=$(suite_cost "$suite")
+        position=0
+        while [ "$position" -lt "${#ordered_suites[@]}" ] &&
+            [ "${ordered_costs[$position]}" -ge "$cost" ]; do
+            position=$((position + 1))
+        done
+        index=${#ordered_suites[@]}
+        while [ "$index" -gt "$position" ]; do
+            previous=$((index - 1))
+            ordered_suites[$index]="${ordered_suites[$previous]}"
+            ordered_costs[$index]="${ordered_costs[$previous]}"
+            index=$previous
+        done
+        ordered_suites[$position]="$suite"
+        ordered_costs[$position]="$cost"
+    done
+
+    shard_loads=()
+    assigned_shards=()
+    for ((shard = 1; shard <= shard_count; shard++)); do
+        shard_loads[$shard]=0
+    done
+    for ((suite_index = 0; suite_index < ${#ordered_suites[@]}; suite_index++)); do
+        target=1
+        cost="${ordered_costs[$suite_index]}"
+        for ((shard = 2; shard <= shard_count; shard++)); do
+            if [ "${shard_loads[$shard]}" -lt "${shard_loads[$target]}" ]; then
+                target=$shard
+            fi
+        done
+        assigned_shards[$suite_index]=$target
+        shard_loads[$target]=$((shard_loads[$target] + cost))
+    done
+
+    suites=()
+    for ((suite_index = 0; suite_index < ${#ordered_suites[@]}; suite_index++)); do
+        if [ "${assigned_shards[$suite_index]}" -eq "$shard_index" ]; then
+            suites[${#suites[@]}]="${ordered_suites[$suite_index]}"
+        fi
+    done
 fi
 
-for suite in $suites; do
+for suite in "${suites[@]}"; do
   set -m
   bash "$TEST_DIR/$suite" &
   pid=$!