feat(p1-parity): honest live entry — deterministic task family, harness oracle, driver protocol

examples/p1-parity/arms.ts

@@ -197,6 +197,9 @@ export type GraphArmBackend =
       /** Transport-only Router substrate; the reviewer profile owns the model. */
       readonly router: RouterTransportConfig
       readonly shotPassed: (workerOutText: string) => boolean
+      /** Driver tool-loop turn cap, forwarded to `runGraph`. Live runs bound a confused driver
+       *  brain; the delegates cap still owns the SHOT budget. */
+      readonly maxTurns?: number
     }
 
 // ── The graph topology (exported so tests can assert on the exact inputs) ──────
@@ -449,6 +452,7 @@ export async function runGraphArm(cell: CellSpec, backend: GraphArmBackend): Pro
           router: backend.router,
           analysts: parityAnalysts(),
           hooks: infraHooks,
+          ...(backend.maxTurns !== undefined ? { maxTurns: backend.maxTurns } : {}),
         }
   const startedAt = Date.now()
   try {

examples/p1-parity/run-parity.ts

@@ -4,17 +4,22 @@
  *
  *   pnpm tsx examples/p1-parity/run-parity.ts --backend offline --cells 2 --shots 3
  *
- *   VB_CLI_BRIDGE_URL=http://127.0.0.1:3344/v1 VB_CLI_BRIDGE_BEARER=... \
- *   VB_PARITY_MODEL=pi/deepseek \
+ *   VB_CLI_BRIDGE_URL=https://router.example/v1 VB_CLI_BRIDGE_BEARER=... \
+ *   VB_PARITY_MODEL=glm-5.3 \
  *   VB_PARITY_ROUTER_URL=https://router.example/v1 VB_PARITY_ROUTER_KEY=... \
- *   VB_PARITY_DRIVER_MODEL=deepseek/deepseek-chat \
+ *   VB_PARITY_DRIVER_MODEL=glm-5.3 \
  *   pnpm tsx examples/p1-parity/run-parity.ts --backend cli-bridge --cells 1 --shots 3
  *
  * offline    — scripted seams (mirrors examples/graphs/shared.ts): zero network, zero env, $0.
  *              Shot script per cell: fail × (shots−1), then pass — so the graph arm settles on
  *              the final shot while the multishot arm burns its whole budget, and the paired records
  *              show exactly that. This mode is CI-safe and is what the vitest suite exercises.
- * cli-bridge — the LIVE entry. Nothing in this repo's gates ever executes it.
+ * cli-bridge — the LIVE entry. Nothing in this repo's gates ever executes it. Cells come from
+ *              {@link LIVE_TASK_FAMILY}: count-and-positions tasks whose ground truth the harness
+ *              computes, so the deliverable check is a deployable text oracle (exactly one
+ *              matching ANSWER line), identical in both arms. `VB_CLI_BRIDGE_URL` names ANY
+ *              OpenAI-compatible `/v1` both arms' coders share — a cli-bridge or the router
+ *              itself; substrate symmetry needs only that it is the SAME endpoint + model.
  *
  * LIVE SUBSTRATE SYMMETRY (the validity invariant, per #710/#721). Both arms' coders are a conversation
  * on the SAME bare OpenAI-compatible `/v1/chat/completions` endpoint (`VB_CLI_BRIDGE_URL`) with
@@ -29,6 +34,7 @@
  * explicit — a missing env var fails loud; there is no silent fallback id.
  */
 
+import { randomBytes } from 'node:crypto'
 import { parseArgs } from 'node:util'
 import type { MultishotTransport } from '@tangle-network/agent-eval/multishot'
 import type { AgentProfile } from '@tangle-network/agent-interface'
@@ -102,23 +108,91 @@ function parityCell(index: number, shots: number): CellSpec {
   }
 }
 
-/** The live cell: same shape, with BOTH models from the environment and the marker contract the
- *  live completion check reads (see {@link LIVE_PASS_MARKER}). */
-function liveParityCell(index: number, shots: number, env: LiveEnv): CellSpec {
+// ── The live task family: deterministic count-and-positions cells ──────────────
+//
+// The deliverable check must be a deployable oracle read off the coder's text, never the model
+// judging itself. Each cell asks for the occurrence count AND every 1-based position of one
+// character in one exact string; the harness computes the ground truth and the check demands
+// exactly ONE well-formed answer line that matches it. Character indexing is genuinely
+// error-prone for chat models, so shot 1 fails often enough to exercise the multi-shot path —
+// the regime where the two orchestration forms can actually diverge.
+
+interface LiveTask {
+  readonly text: string
+  readonly ch: string
+}
+
+/** Fixed family; `--cells N` runs the first N. Mixed length/density for a spread of difficulty. */
+const LIVE_TASK_FAMILY: ReadonlyArray<LiveTask> = [
+  { text: 'strawberry', ch: 'r' },
+  { text: 'parallelizable', ch: 'l' },
+  { text: 'mississippi riverbanks', ch: 's' },
+  { text: 'indivisibility', ch: 'i' },
+  { text: 'bookkeeping paraphernalia', ch: 'a' },
+  { text: 'overengineering', ch: 'e' },
+  { text: 'circumnavigation of the archipelago', ch: 'c' },
+  { text: 'unconstitutionally', ch: 'n' },
+]
+
+/** Ground truth, computed by the harness — the one line a passing reply must contain. */
+export function expectedAnswerLine(task: LiveTask): string {
+  const positions: number[] = []
+  for (let i = 0; i < task.text.length; i += 1) {
+    if (task.text[i] === task.ch) positions.push(i + 1)
+  }
+  return `ANSWER: ${positions.length}@[${positions.join(',')}]`
+}
+
+/** Exactly one `ANSWER:` line, equal to the expected line — several candidate lines cannot
+ *  shotgun the check, and prose around the line does not defeat it. */
+export function liveShotPassed(expected: string): (text: string) => boolean {
+  return (text) => {
+    const answers = text
+      .split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.startsWith('ANSWER:'))
+    return answers.length === 1 && answers[0] === expected
+  }
+}
+
+function liveTaskText(index: number, task: LiveTask, nonce: string): string {
+  return (
+    `counting cell ${index + 1} [run ${nonce}]: in the exact string '${task.text}', count the ` +
+    `occurrences of the character '${task.ch}' and list every occurrence's 1-based position. ` +
+    'Case-sensitive; every character including spaces occupies one position. End your reply ' +
+    'with one line of the exact form: ANSWER: <count>@[<p1>,<p2>,...]'
+  )
+}
+
+/** The live cell: the family task above, BOTH models from the environment, and the reviewer
+ *  authored for both forms — its `systemPrompt` is the stance both arms share (the multishot
+ *  driver leg reads only this), and its `instructions` carry the graph arm's tool protocol
+ *  (`resolveSupervisorSystemPrompt` appends them for the root brain; `runMultishot` never sees
+ *  them — orchestration in the loop form is code, in the graph form it is the supervisor). */
+function liveParityCell(index: number, shots: number, env: LiveEnv, nonce: string): CellSpec {
   const base = parityCell(index, shots)
+  const task = LIVE_TASK_FAMILY[index]
+  if (task === undefined) {
+    throw new Error(
+      `--cells ${index + 1} exceeds the live task family (${LIVE_TASK_FAMILY.length} cells)`,
+    )
+  }
   return {
     ...base,
+    task: liveTaskText(index, task, nonce),
     coderProfile: {
       ...base.coderProfile,
       model: {
         ...base.coderProfile.model,
-        provider: 'cli-bridge',
+        provider: 'openai-compat',
         default: env.coderModel,
       },
       prompt: {
         systemPrompt:
-          'Make tests pass. Print the exact line ' +
-          `'${LIVE_PASS_MARKER}' when and ONLY when the full suite genuinely passes.`,
+          'Solve the counting task exactly. Verify character by character, indexing every ' +
+          'position, before you answer. Your reply must contain exactly one line starting with ' +
+          "'ANSWER:', of the exact form ANSWER: <count>@[<p1>,<p2>,...] — no other line may " +
+          'start with ANSWER:.',
       },
     },
     reviewerProfile: {
@@ -128,6 +202,25 @@ function liveParityCell(index: number, shots: number, env: LiveEnv): CellSpec {
         provider: 'tangle-router',
         default: env.driverModel,
       },
+      prompt: {
+        systemPrompt:
+          'You are the reviewer driving a coder through repeated shots at a counting task. You ' +
+          'never solve the task or state an answer yourself. Each round, issue one short ' +
+          'corrective instruction: tell the coder to recount character by character, index ' +
+          'every position, and restate its full reply ending with its final ANSWER line.',
+        instructions: [
+          "You supervise exactly ONE pinned worker node named 'coder'; the graph pins its full " +
+            'profile. To run a shot call spawn_agent with arguments ' +
+            '{"profile":{"name":"coder"},"task":"<brief>"} — never author any other profile.',
+          "Shot 1's task must be the root task COPIED VERBATIM. After each spawn, call " +
+            'await_event until that worker settles; the settle reports valid:true only when ' +
+            'the deliverable check passed.',
+          "valid:false → spawn 'coder' again with a short corrective brief (the session " +
+            'resumes automatically). valid:true → stop calling tools and reply with a one-line ' +
+            'summary.',
+          `You may spawn 'coder' at most ${shots} times in total.`,
+        ],
+      },
     },
   }
 }
@@ -236,14 +329,10 @@ function completionsTransport(
   }
 }
 
-/** LIVE completion check: a marker oracle, pending a real verifier lens. The coder profile must
- *  instruct the harness to print this marker only when the suite genuinely passes. */
-const LIVE_PASS_MARKER = 'ALL TESTS PASS'
-const livePassed = (text: string): boolean => text.includes(LIVE_PASS_MARKER)
-
 function liveBackends(
   env: LiveEnv,
   cell: CellSpec,
+  shotPassed: (text: string) => boolean,
 ): {
   multishot: MultishotArmBackend
   graph: GraphArmBackend
@@ -254,7 +343,7 @@ function liveBackends(
       // substrate — each leg matching its graph-arm counterpart, including the driver model.
       agentTransport: completionsTransport(cell.coderProfile, env.url, env.bearer),
       driverTransport: completionsTransport(cell.reviewerProfile, env.routerUrl, env.routerKey),
-      shotPassed: livePassed,
+      shotPassed,
       apiKey: env.bearer,
       baseUrl: env.url,
     },
@@ -263,7 +352,10 @@ function liveBackends(
       url: env.url,
       bearer: env.bearer,
       router: { routerBaseUrl: env.routerUrl, routerKey: env.routerKey },
-      shotPassed: livePassed,
+      shotPassed,
+      // Spawns + awaits + a final reply for the shot budget, with slack for re-reads; the
+      // delegates cap, not this, owns the shot budget.
+      maxTurns: cell.shots * 6 + 6,
     },
   }
 }
@@ -299,6 +391,10 @@ function printRecord(row: PairedRow): void {
 
 export async function main(): Promise<void> {
   const cli = parseCli(process.argv.slice(2))
+  // One nonce per run, embedded in every live task text: the SAME text still reaches both arms
+  // (input equivalence within a cell) while no prompt is byte-identical across runs, so an
+  // upstream response cache cannot serve one run's completion to another.
+  const runNonce = randomBytes(4).toString('hex')
   const rows: PairedRow[] = []
   for (let i = 0; i < cli.cells; i += 1) {
     let multishotBackend: MultishotArmBackend
@@ -312,8 +408,14 @@ export async function main(): Promise<void> {
       graphBackend = offlineGraphBackend(cell, script).backend
     } else {
       const env = requireLiveEnv()
-      cell = liveParityCell(i, cli.shots, env)
-      const backends = liveBackends(env, cell)
+      const task = LIVE_TASK_FAMILY[i]
+      if (task === undefined) {
+        throw new Error(
+          `--cells ${i + 1} exceeds the live task family (${LIVE_TASK_FAMILY.length})`,
+        )
+      }
+      cell = liveParityCell(i, cli.shots, env, runNonce)
+      const backends = liveBackends(env, cell, liveShotPassed(expectedAnswerLine(task)))
       multishotBackend = backends.multishot
       graphBackend = backends.graph
     }
@@ -324,10 +426,40 @@ export async function main(): Promise<void> {
   }
   console.log(`p1-parity — backend=${cli.backend} cells=${cli.cells} shots=${cli.shots}\n`)
   for (const row of rows) printRecord(row)
+  printParitySummary(rows, cli)
   console.log('\nfull records (JSON):')
   console.log(JSON.stringify(rows, null, 2))
 }
 
+/** Paired verdict per cell + the ledger audit: does the graph arm's edge ledger account for
+ *  every traversal it claims (bound worker on every delivered delegates row, shot budget
+ *  respected, shotsUsed = distinct bound workers)? */
+function printParitySummary(rows: ReadonlyArray<PairedRow>, cli: CliOptions): void {
+  console.log('\nparity summary (per cell):')
+  for (let cell = 1; cell <= cli.cells; cell += 1) {
+    const multishot = rows.find((r) => r.cell === cell && r.arm === 'multishot')?.record
+    const graph = rows.find((r) => r.cell === cell && r.arm === 'graph')?.record
+    if (!multishot || !graph) continue
+    const agree = multishot.converged === graph.converged
+    const ledger = graph.ledger ?? []
+    const delegates = ledger.filter((t) => t.kind === 'delegates')
+    const delivered = delegates.filter((t) => t.outcome === 'delivered')
+    const boundWorkers = new Set(
+      delivered.map((t) => t.workerId).filter((id): id is string => id !== undefined),
+    )
+    const unbound = delivered.filter((t) => t.workerId === undefined).length
+    const ledgerAccounts =
+      unbound === 0 && boundWorkers.size === graph.shotsUsed && boundWorkers.size <= cli.shots
+    console.log(
+      `cell ${cell}: converged multishot=${multishot.converged} graph=${graph.converged} ` +
+        `${agree ? 'AGREE' : 'DIVERGE'} | shots ${multishot.shotsUsed} vs ${graph.shotsUsed} | ` +
+        `ledger delegates=${delegates.length} delivered=${delivered.length} ` +
+        `other=${delegates.filter((t) => t.outcome !== 'delivered').length} ` +
+        `boundWorkers=${boundWorkers.size} accounts=${ledgerAccounts}`,
+    )
+  }
+}
+
 if (import.meta.url === `file://${process.argv[1]}`) {
   main().catch((err) => {
     console.error(err)