tests: 8814AU validation harnesses + rail_check SDR-degraded arbitration (#110)

tests/au8814_5g_baseline.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Issue #110 re-baseline on current master: RTL8814AU 5 GHz TX health in ONE
+# cold session — (a) computed TXAGC index (DEVOURER_LOG_TXPWR), (b) CU-ground
+# decode (frames/EVM/RSSI), (c) a single SDR duty read (first read only — a
+# second back-to-back read can fail to reacquire).
+#   sudo tests/au8814_5g_baseline.sh [channel] [rate]
+set -u
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+CH="${1:-36}"
+RATE="${2:-MCS7}"
+FREQ=$((5000 + 5*CH))e6
+OUT=/tmp/au8814-baseline
+mkdir -p "$OUT"
+cleanup(){ sudo -n pkill -x rxdemo 2>/dev/null; sudo -n pkill -x txdemo 2>/dev/null; true; }
+trap cleanup EXIT INT TERM
+cleanup; sleep 1
+
+# VBUS cold the 8814 (rail-sag history on its hub branch).
+SYS=""
+for d in /sys/bus/usb/devices/*/idProduct; do
+    [ "$(cat "$d" 2>/dev/null)" = "8813" ] && { SYS=$(basename "$(dirname "$d")"); break; }
+done
+[ -n "$SYS" ] || { echo "ERROR: 8813 not on USB"; exit 1; }
+case "$SYS" in
+    *.*) sudo -n uhubctl -l "${SYS%.*}" -p "${SYS##*.}" -a cycle -d 3 >/dev/null 2>&1 || \
+             echo "WARN: uhubctl cycle failed";;
+    *)   echo "WARN: 8814 on a root port — skipping VBUS cycle";;
+esac
+sleep 4; lsusb -d 0bda:8813 >/dev/null || { echo "ERROR: 8814 gone after cycle"; exit 1; }
+
+echo "== CU ground up (ch$CH) =="
+: >"$OUT/ground.log"
+sudo -n env DEVOURER_PID=0xc812 DEVOURER_VID=0x0bda DEVOURER_CHANNEL="$CH" \
+    DEVOURER_STREAM_OUT=1 timeout 60 "$ROOT/build/rxdemo" \
+    >"$OUT/ground.log" 2>/dev/null &
+GJ=$!; sleep 12
+
+echo "== 8814 TX ($RATE ch$CH, max duty) =="
+sudo -n env DEVOURER_PID=0x8813 DEVOURER_VID=0x0bda DEVOURER_CHANNEL="$CH" \
+    DEVOURER_TX_RATE="$RATE" DEVOURER_TX_GAP_US=0 DEVOURER_LOG_TXPWR=1 \
+    timeout 35 "$ROOT/build/txdemo" >/dev/null 2>"$OUT/tx.err" &
+TJ=$!; sleep 8
+
+echo "== single SDR duty read =="
+sudo -n python3 "$ROOT/tests/sdr_duty.py" --freq "$FREQ" --rate 25e6 --secs 5 \
+    --gain 60 --mcs 7 --bw 20 2>&1 | grep -iE "duty=" | head -1
+wait "$TJ" 2>/dev/null; sleep 1
+cleanup; wait "$GJ" 2>/dev/null
+
+echo "== computed TXAGC (from DEVOURER_LOG_TXPWR) =="
+grep -iE "txpwr|power.*index|idx" "$OUT/tx.err" | grep -vE "bulk|ready" | head -8
+python3 - "$OUT/ground.log" <<'PYEOF'
+import json, statistics as st
+n=0; evm=[]; rssi=[]
+for line in open(sys_argv1 := __import__('sys').argv[1], errors="replace"):
+    if '"ev":"rx.frame"' not in line: continue
+    try: ev=json.loads(line)
+    except ValueError: continue
+    if ev.get("rate")==19:
+        n+=1; evm.append(ev["evm"][0]); rssi.append(ev["rssi"][0])
+print(f"ground: MCS7 n={n} medEVM={st.median(evm) if evm else None} medRSSI={st.median(rssi) if rssi else None}")
+PYEOF

tests/au8814_dual_soak.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Issue #110 disambiguation soak: 8814AU max-duty 5 GHz TX, measured
+# CONTINUOUSLY by the 8822CU ground and PERIODICALLY by the B210 — when a
+# low SDR reading appears, the per-window ground tallies say whether the
+# transmitter actually collapsed (both drop) or the SDR read is bad (ground
+# stays strong). Ground = truth anchor; SDR = the instrument under suspicion.
+#   sudo tests/au8814_dual_soak.sh [rounds] [round_secs]
+set -u
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+ROUNDS="${1:-10}"
+RSECS="${2:-45}"
+OUT=/tmp/au8814-dualsoak
+mkdir -p "$OUT"
+cleanup(){ sudo -n pkill -x rxdemo 2>/dev/null; sudo -n pkill -x txdemo 2>/dev/null; true; }
+trap cleanup EXIT INT TERM
+cleanup; sleep 1
+
+TOTAL=$((ROUNDS*RSECS + 40))
+: >"$OUT/ground.log"
+sudo -n env DEVOURER_PID=0xc812 DEVOURER_VID=0x0bda DEVOURER_CHANNEL=36 \
+    DEVOURER_STREAM_OUT=1 timeout "$TOTAL" "$ROOT/build/rxdemo" \
+    | while IFS= read -r line; do printf '%s %s\n' "$(date +%s)" "$line"; done \
+    >>"$OUT/ground.log" 2>/dev/null &
+GJ=$!; sleep 12
+sudo -n env DEVOURER_PID=0x8813 DEVOURER_VID=0x0bda DEVOURER_CHANNEL=36 \
+    DEVOURER_TX_RATE=MCS7 DEVOURER_TX_GAP_US=0 \
+    timeout $((TOTAL-15)) "$ROOT/build/txdemo" >/dev/null 2>&1 &
+TJ=$!; sleep 6
+
+: >"$OUT/rounds.txt"
+for i in $(seq 1 "$ROUNDS"); do
+    t0=$(date +%s)
+    duty=$(sudo -n python3 "$ROOT/tests/sdr_duty.py" --freq 5180e6 --rate 25e6 \
+        --secs 4 --gain 60 --mcs 7 --bw 20 2>&1 | grep -oE "duty=[0-9.]+" | head -1)
+    t1=$(date +%s)
+    echo "$i $t0 $t1 ${duty:-duty=NA}" >>"$OUT/rounds.txt"
+    sleep $((RSECS-8))
+done
+cleanup; wait "$GJ" "$TJ" 2>/dev/null
+
+python3 - "$OUT/ground.log" "$OUT/rounds.txt" <<'PYEOF'
+import json, statistics, sys
+frames=[]
+for line in open(sys.argv[1], errors="replace"):
+    if '"ev":"rx.frame"' not in line: continue
+    ts,_,js=line.partition(" ")
+    try: ev=json.loads(js)
+    except ValueError: continue
+    if ev.get("rate")==19:
+        frames.append((int(ts), ev["rssi"][0]))
+print(f"{'round':>5} {'sdr':>10} {'ground fps':>10} {'medRSSI':>8}")
+for line in open(sys.argv[2]):
+    i, t0, t1, duty = line.split()
+    t0, t1 = int(t0), int(t1)
+    sel=[r for t,r in frames if t0<=t<=t1]
+    fps=len(sel)/max(1,(t1-t0))
+    med=statistics.median(sel) if sel else None
+    print(f"{i:>5} {duty:>10} {fps:>10.0f} {str(med):>8}")
+PYEOF

tests/bench_onair.py

@@ -140,6 +140,17 @@ def main() -> int:
     # mis-read as a per-chip 5 GHz deficit.
     if not args.skip_rail_check:
         rc = subprocess.run(["bash", str(HERE / "rail_check.sh")])
+        if rc.returncode == 2:
+            # rail_check's ground arbitration: the control airs strongly (the
+            # ground hears it) but the B210 reads low — the SDR itself is in
+            # its degraded-read mode, so EVERY duty number this session is
+            # unreliable. Abort rather than emit a garbage table.
+            print("\n" + "!" * 70)
+            print("!! SDR DEGRADED — the B210's readings can't be trusted this session")
+            print("!! (the ground hears the control at strong RSSI while the SDR reads")
+            print("!! low). Reset/reseat the B210 or power-cycle its port, then re-run.")
+            print("!" * 70 + "\n")
+            return 2
         if rc.returncode != 0:
             print("\n" + "!" * 70)
             print("!! RAIL SAG DETECTED — 5 GHz duty numbers below are NOT trustworthy.")

tests/rail_check.sh

@@ -1,32 +1,44 @@
 #!/usr/bin/env bash
-# Rail-sag guard — CLAUDE.md defence #1: re-check the known-good control adapter's
-# 5 GHz on-air duty EACH SESSION before trusting any 5 GHz TX measurement.
+# Rail-sag guard — re-check the known-good control adapter's 5 GHz on-air duty
+# EACH SESSION before trusting any 5 GHz TX measurement.
 #
 # 5 GHz TX draws far more PA current than 2.4 GHz; fed through a bus-powered hub
 # chain the rail browns out the PA, collapsing on-air power *intermittently* and
 # worst on the most deeply-nested adapter — while frames still submit fine (0
 # send-fails) and 2.4 GHz keeps working. The trap is to mis-diagnose this as a
-# per-chip dead PA / a 5 GHz code gate / an EFUSE TX-power bug. This guard catches
-# it: it floods the control adapter at 5 GHz and 2.4 GHz via devourer and compares
-# the SDR duty. A healthy rail gives comparable 5 GHz and 2.4 GHz duty; a sagging
-# rail gives 5 GHz << 2.4 GHz (and 5 GHz below the absolute floor).
+# per-chip dead PA / a 5 GHz code gate / an EFUSE TX-power bug.
 #
-#   sudo tests/rail_check.sh                  # default control = CU c812
-#   sudo CTRL_PID=0xc812 RAIL_MIN_DUTY=50 RAIL_MIN_RATIO=0.6 tests/rail_check.sh
+# The guard itself has a trap: the B210 has a degraded-read mode on back-to-back
+# sessions (reads ~1.5% duty with a clean-looking noise floor while the air is
+# in fact saturated), which used to false-trigger this check as "RAIL SAG". The
+# check therefore now (a) takes the 5 GHz reading in the trustworthy
+# first-read-of-session slot with the noise floor pinned from an idle
+# measurement, (b) on a low reading: cools down, usbreset's the SDR and retries
+# once, and (c) arbitrates a persistent low reading against an independent
+# devourer GROUND adapter — the ground hearing the control at strong RSSI means
+# the SDR is degraded, not the rail.
 #
-# Exit 0 = rail OK (5 GHz measurements trustworthy); exit 1 = rail sag (do NOT
-# trust 5 GHz numbers — power-cycle the hub / move the control to a powered hub or
-# direct root port and re-run).
+#   sudo tests/rail_check.sh
+#   sudo CTRL_PID=0xc812 GROUND_PID=0xa81a RAIL_MIN_DUTY=50 tests/rail_check.sh
+#
+# Exit 0 = rail OK (5 GHz measurements trustworthy)
+# Exit 1 = RAIL SAG (5 GHz on-air genuinely collapsing — fix the power feed)
+# Exit 2 = SDR DEGRADED (rail is fine per the ground, but the B210 cannot be
+#          trusted this session — reset/reseat it; bench numbers unreliable)
 set -u
 PY=${PY:-python3}
 CTRL_VID=${CTRL_VID:-0x0bda}
 CTRL_PID=${CTRL_PID:-0xc812}          # CU (RTL8812CU) — the validated 5 GHz control
+GROUND_VID=${GROUND_VID:-0x0bda}
+GROUND_PID=${GROUND_PID:-0xa81a}      # EU as the independent decode sensor
 SECS=${SECS:-5}
 RAIL_MIN_DUTY=${RAIL_MIN_DUTY:-50}    # absolute 5 GHz duty floor (%), tune per SDR setup
-RAIL_MIN_RATIO=${RAIL_MIN_RATIO:-0.6} # min 5GHz/2.4GHz duty ratio on a healthy rail
+RAIL_MIN_RATIO=${RAIL_MIN_RATIO:-0.6} # 5GHz/2.4GHz duty ratio: warning-only
+SDR_USB_ID=${SDR_USB_ID:-2500:0020}   # B210, for the usbreset recovery attempt
 TXBIN=${TXBIN:-build/txdemo}
+RXBIN=${RXBIN:-build/rxdemo}
 
-cleanup(){ sudo pkill -9 -x txdemo 2>/dev/null; }
+cleanup(){ sudo pkill -9 -x txdemo 2>/dev/null; sudo pkill -x rxdemo 2>/dev/null; }
 trap cleanup EXIT
 
 # free the control from any bound kernel driver
@@ -42,40 +54,97 @@ free_ctrl() {
   done
 }
 
-measure() { # $1=channel $2=freq -> echoes duty%
+idle_floor() { # $1=freq -> echoes noise floor dB (idle, no TX)
+  sudo $PY tests/sdr_duty.py --freq "$1" --rate 25e6 --secs 2 --gain 60 \
+      --mcs 7 --bw 20 2>&1 | grep -oiE "noise=-?[0-9.]+" | head -1 | cut -d= -f2
+}
+
+duty_read() { # $1=freq $2=floor -> echoes duty%
+  local args=(--freq "$1" --rate 25e6 --secs "$SECS" --gain 60 --mcs 7 --bw 20)
+  [ -n "${2:-}" ] && args+=(--noise-db "$2")
+  sudo $PY tests/sdr_duty.py "${args[@]}" 2>&1 | \
+      grep -oiE "duty=[0-9.]+%" | head -1 | sed 's/duty=//;s/%//'
+}
+
+flood() { # $1=channel — start the control flooding (backgrounded)
   cleanup; sleep 2
   sudo env DEVOURER_VID=$CTRL_VID DEVOURER_PID=$CTRL_PID DEVOURER_CHANNEL=$1 \
        DEVOURER_TX_RATE=MCS7 DEVOURER_TX_GAP_US=0 \
-       stdbuf -oL timeout -k 5 $((SECS+5)) "$TXBIN" >/tmp/railchk_$1.log 2>&1 &
+       stdbuf -oL timeout -k 5 60 "$TXBIN" >/tmp/railchk_$1.log 2>&1 &
   sleep 5
-  local out
-  out=$(sudo $PY tests/sdr_duty.py --freq "$2" --rate 25e6 --secs "$SECS" \
-        --gain 60 --mcs 7 --bw 20 2>&1 | grep -oiE "duty=[0-9.]+%" | head -1)
-  cleanup; sleep 1
-  echo "${out#duty=}" | tr -d '%'
 }
 
 echo "# rail_check: control $CTRL_VID:$CTRL_PID"
 sudo rmmod rtl88x2eu_ohd 2>/dev/null
 free_ctrl
-D24=$(measure 6   2437e6)
-D5=$(measure  36  5180e6)
-D24=${D24:-0}; D5=${D5:-0}
-echo "  2.4 GHz duty = ${D24}%   5 GHz duty = ${D5}%"
 
-# ratio in integer math (avoid bc dependency): 100*D5/D24 vs 100*RATIO
-ratio_ok=1
-if awk "BEGIN{exit !($D24 > 0 && $D5/$D24 < $RAIL_MIN_RATIO)}"; then ratio_ok=0; fi
-floor_ok=1
-if awk "BEGIN{exit !($D5 < $RAIL_MIN_DUTY)}"; then floor_ok=0; fi
+# Idle noise floors first (also serves as the SDR warm-up read).
+F5=$(idle_floor 5180e6); sleep 5
+F24=$(idle_floor 2437e6); sleep 8
+echo "  idle floors: 5 GHz ${F5:-?} dB, 2.4 GHz ${F24:-?} dB"
+
+# 5 GHz reading in the fresh slot, floor pinned.
+flood 36
+D5=$(duty_read 5180e6 "$F5"); D5=${D5:-0}
 
-if [ "$floor_ok" = 0 ] || [ "$ratio_ok" = 0 ]; then
+if awk "BEGIN{exit !($D5 < $RAIL_MIN_DUTY)}"; then
+  echo "  5 GHz duty low (${D5}%) — cooling down, resetting the SDR, retrying once"
+  cleanup; sleep 15
+  command -v usbreset >/dev/null && sudo usbreset "$SDR_USB_ID" >/dev/null 2>&1
+  sleep 5
+  flood 36
+  D5=$(duty_read 5180e6 "$F5"); D5=${D5:-0}
+fi
+
+if awk "BEGIN{exit !($D5 < $RAIL_MIN_DUTY)}"; then
+  # Persistent low reading: arbitrate against an independent devourer ground.
+  if lsusb -d "${GROUND_VID#0x}:${GROUND_PID#0x}" >/dev/null 2>&1 && \
+     [ "$GROUND_PID" != "$CTRL_PID" ]; then
+    echo "  arbitrating with ground $GROUND_VID:$GROUND_PID ..."
+    sudo env DEVOURER_VID=$GROUND_VID DEVOURER_PID=$GROUND_PID \
+         DEVOURER_CHANNEL=36 DEVOURER_STREAM_OUT=1 \
+         timeout 25 "$RXBIN" >/tmp/railchk_ground.log 2>/dev/null &
+    sleep 22
+    G=$(python3 - /tmp/railchk_ground.log <<'PYEOF'
+import json, statistics, sys
+n=0; rssi=[]
+for line in open(sys.argv[1], errors="replace"):
+    if '"ev":"rx.frame"' not in line: continue
+    try: ev=json.loads(line)
+    except ValueError: continue
+    if ev.get("rate")==19:
+        n+=1; rssi.append(ev["rssi"][0])
+print(f"{n} {int(statistics.median(rssi)) if rssi else 0}")
+PYEOF
+)
+    GN=${G% *}; GR=${G#* }
+    echo "  ground heard n=$GN medRSSI=$GR"
+    if [ "${GN:-0}" -gt 500 ] && [ "${GR:-0}" -gt 50 ]; then
+      echo "  RESULT: SDR DEGRADED — the ground hears the control at strong RSSI"
+      echo "          while the B210 reads ${D5}%. The rail is fine; the SDR is"
+      echo "          not trustworthy this session. Reset/reseat the B210 (or"
+      echo "          power-cycle its port) and re-run."
+      exit 2
+    fi
+  fi
   echo "  RESULT: RAIL SAG — 5 GHz on-air power is collapsing on the control."
   echo "          5 GHz measurements are NOT trustworthy. Do not chase a 5 GHz"
   echo "          'code gate' / PA-bias / EFUSE bug until this is fixed."
   echo "          Fix: power-cycle the hub (uhubctl -l <hub> -a cycle), or move"
   echo "          the adapter to a powered USB hub / direct root port, then re-run."
   exit 1
 fi
+
+# 2.4 GHz reading (floor pinned — an unpinned floor on a busy 2.4 GHz channel
+# under-reads duty and used to poison the ratio heuristic). Ratio is
+# warning-only: the absolute 5 GHz floor above is the real guard.
+cleanup; sleep 10
+flood 6
+D24=$(duty_read 2437e6 "$F24"); D24=${D24:-0}
+cleanup
+echo "  2.4 GHz duty = ${D24}%   5 GHz duty = ${D5}%"
+if awk "BEGIN{exit !($D24 > 0 && $D5/$D24 < $RAIL_MIN_RATIO)}"; then
+  echo "  WARN: 5GHz/2.4GHz duty ratio below $RAIL_MIN_RATIO — watch for partial sag."
+fi
 echo "  RESULT: rail OK — 5 GHz measurements are trustworthy."
 exit 0