From 380dfc2819b8b4cca1f053980e5cd5c0dd6181d8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 15 Aug 2026 03:25:18 -0600 Subject: [PATCH] feat(examples): real live task family + driver protocol for the P1 parity harness The live entry gets a deterministic count-and-positions task family whose ground truth the harness computes, so the deliverable check is a deployable text oracle identical in both arms. The reviewer profile carries the graph driver protocol in prompt.instructions; the shared systemPrompt stays the multishot driver stance. A per-run nonce keeps prompts unique across runs so an upstream cache cannot cross-serve completions. The chat backend gains a runGraph maxTurns bound, and the CLI prints a per-cell parity summary with an edge-ledger accounting audit. --- examples/p1-parity/arms.ts | 4 + examples/p1-parity/run-parity.ts | 170 +++++++++++++++++++++++++++---- 2 files changed, 155 insertions(+), 19 deletions(-) diff --git a/examples/p1-parity/arms.ts b/examples/p1-parity/arms.ts index e99a8087..8f2118f6 100644 --- a/examples/p1-parity/arms.ts +++ b/examples/p1-parity/arms.ts @@ -197,6 +197,9 @@ export type GraphArmBackend = /** Transport-only Router substrate; the reviewer profile owns the model. */ readonly router: RouterTransportConfig readonly shotPassed: (workerOutText: string) => boolean + /** Driver tool-loop turn cap, forwarded to `runGraph`. Live runs bound a confused driver + * brain; the delegates cap still owns the SHOT budget. */ + readonly maxTurns?: number } // ── The graph topology (exported so tests can assert on the exact inputs) ────── @@ -449,6 +452,7 @@ export async function runGraphArm(cell: CellSpec, backend: GraphArmBackend): Pro router: backend.router, analysts: parityAnalysts(), hooks: infraHooks, + ...(backend.maxTurns !== undefined ? { maxTurns: backend.maxTurns } : {}), } const startedAt = Date.now() try { diff --git a/examples/p1-parity/run-parity.ts b/examples/p1-parity/run-parity.ts index a9bf79fd..ec68df8a 100644 --- a/examples/p1-parity/run-parity.ts +++ b/examples/p1-parity/run-parity.ts @@ -4,17 +4,22 @@ * * pnpm tsx examples/p1-parity/run-parity.ts --backend offline --cells 2 --shots 3 * - * VB_CLI_BRIDGE_URL=http://127.0.0.1:3344/v1 VB_CLI_BRIDGE_BEARER=... \ - * VB_PARITY_MODEL=pi/deepseek \ + * VB_CLI_BRIDGE_URL=https://router.example/v1 VB_CLI_BRIDGE_BEARER=... \ + * VB_PARITY_MODEL=glm-5.3 \ * VB_PARITY_ROUTER_URL=https://router.example/v1 VB_PARITY_ROUTER_KEY=... \ - * VB_PARITY_DRIVER_MODEL=deepseek/deepseek-chat \ + * VB_PARITY_DRIVER_MODEL=glm-5.3 \ * pnpm tsx examples/p1-parity/run-parity.ts --backend cli-bridge --cells 1 --shots 3 * * offline — scripted seams (mirrors examples/graphs/shared.ts): zero network, zero env, $0. * Shot script per cell: fail × (shots−1), then pass — so the graph arm settles on * the final shot while the multishot arm burns its whole budget, and the paired records * show exactly that. This mode is CI-safe and is what the vitest suite exercises. - * cli-bridge — the LIVE entry. Nothing in this repo's gates ever executes it. + * cli-bridge — the LIVE entry. Nothing in this repo's gates ever executes it. Cells come from + * {@link LIVE_TASK_FAMILY}: count-and-positions tasks whose ground truth the harness + * computes, so the deliverable check is a deployable text oracle (exactly one + * matching ANSWER line), identical in both arms. `VB_CLI_BRIDGE_URL` names ANY + * OpenAI-compatible `/v1` both arms' coders share — a cli-bridge or the router + * itself; substrate symmetry needs only that it is the SAME endpoint + model. * * LIVE SUBSTRATE SYMMETRY (the validity invariant, per #710/#721). Both arms' coders are a conversation * on the SAME bare OpenAI-compatible `/v1/chat/completions` endpoint (`VB_CLI_BRIDGE_URL`) with @@ -29,6 +34,7 @@ * explicit — a missing env var fails loud; there is no silent fallback id. */ +import { randomBytes } from 'node:crypto' import { parseArgs } from 'node:util' import type { MultishotTransport } from '@tangle-network/agent-eval/multishot' import type { AgentProfile } from '@tangle-network/agent-interface' @@ -102,23 +108,91 @@ function parityCell(index: number, shots: number): CellSpec { } } -/** The live cell: same shape, with BOTH models from the environment and the marker contract the - * live completion check reads (see {@link LIVE_PASS_MARKER}). */ -function liveParityCell(index: number, shots: number, env: LiveEnv): CellSpec { +// ── The live task family: deterministic count-and-positions cells ────────────── +// +// The deliverable check must be a deployable oracle read off the coder's text, never the model +// judging itself. Each cell asks for the occurrence count AND every 1-based position of one +// character in one exact string; the harness computes the ground truth and the check demands +// exactly ONE well-formed answer line that matches it. Character indexing is genuinely +// error-prone for chat models, so shot 1 fails often enough to exercise the multi-shot path — +// the regime where the two orchestration forms can actually diverge. + +interface LiveTask { + readonly text: string + readonly ch: string +} + +/** Fixed family; `--cells N` runs the first N. Mixed length/density for a spread of difficulty. */ +const LIVE_TASK_FAMILY: ReadonlyArray = [ + { text: 'strawberry', ch: 'r' }, + { text: 'parallelizable', ch: 'l' }, + { text: 'mississippi riverbanks', ch: 's' }, + { text: 'indivisibility', ch: 'i' }, + { text: 'bookkeeping paraphernalia', ch: 'a' }, + { text: 'overengineering', ch: 'e' }, + { text: 'circumnavigation of the archipelago', ch: 'c' }, + { text: 'unconstitutionally', ch: 'n' }, +] + +/** Ground truth, computed by the harness — the one line a passing reply must contain. */ +export function expectedAnswerLine(task: LiveTask): string { + const positions: number[] = [] + for (let i = 0; i < task.text.length; i += 1) { + if (task.text[i] === task.ch) positions.push(i + 1) + } + return `ANSWER: ${positions.length}@[${positions.join(',')}]` +} + +/** Exactly one `ANSWER:` line, equal to the expected line — several candidate lines cannot + * shotgun the check, and prose around the line does not defeat it. */ +export function liveShotPassed(expected: string): (text: string) => boolean { + return (text) => { + const answers = text + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith('ANSWER:')) + return answers.length === 1 && answers[0] === expected + } +} + +function liveTaskText(index: number, task: LiveTask, nonce: string): string { + return ( + `counting cell ${index + 1} [run ${nonce}]: in the exact string '${task.text}', count the ` + + `occurrences of the character '${task.ch}' and list every occurrence's 1-based position. ` + + 'Case-sensitive; every character including spaces occupies one position. End your reply ' + + 'with one line of the exact form: ANSWER: @[,,...]' + ) +} + +/** The live cell: the family task above, BOTH models from the environment, and the reviewer + * authored for both forms — its `systemPrompt` is the stance both arms share (the multishot + * driver leg reads only this), and its `instructions` carry the graph arm's tool protocol + * (`resolveSupervisorSystemPrompt` appends them for the root brain; `runMultishot` never sees + * them — orchestration in the loop form is code, in the graph form it is the supervisor). */ +function liveParityCell(index: number, shots: number, env: LiveEnv, nonce: string): CellSpec { const base = parityCell(index, shots) + const task = LIVE_TASK_FAMILY[index] + if (task === undefined) { + throw new Error( + `--cells ${index + 1} exceeds the live task family (${LIVE_TASK_FAMILY.length} cells)`, + ) + } return { ...base, + task: liveTaskText(index, task, nonce), coderProfile: { ...base.coderProfile, model: { ...base.coderProfile.model, - provider: 'cli-bridge', + provider: 'openai-compat', default: env.coderModel, }, prompt: { systemPrompt: - 'Make tests pass. Print the exact line ' + - `'${LIVE_PASS_MARKER}' when and ONLY when the full suite genuinely passes.`, + 'Solve the counting task exactly. Verify character by character, indexing every ' + + 'position, before you answer. Your reply must contain exactly one line starting with ' + + "'ANSWER:', of the exact form ANSWER: @[,,...] — no other line may " + + 'start with ANSWER:.', }, }, reviewerProfile: { @@ -128,6 +202,25 @@ function liveParityCell(index: number, shots: number, env: LiveEnv): CellSpec { provider: 'tangle-router', default: env.driverModel, }, + prompt: { + systemPrompt: + 'You are the reviewer driving a coder through repeated shots at a counting task. You ' + + 'never solve the task or state an answer yourself. Each round, issue one short ' + + 'corrective instruction: tell the coder to recount character by character, index ' + + 'every position, and restate its full reply ending with its final ANSWER line.', + instructions: [ + "You supervise exactly ONE pinned worker node named 'coder'; the graph pins its full " + + 'profile. To run a shot call spawn_agent with arguments ' + + '{"profile":{"name":"coder"},"task":""} — never author any other profile.', + "Shot 1's task must be the root task COPIED VERBATIM. After each spawn, call " + + 'await_event until that worker settles; the settle reports valid:true only when ' + + 'the deliverable check passed.', + "valid:false → spawn 'coder' again with a short corrective brief (the session " + + 'resumes automatically). valid:true → stop calling tools and reply with a one-line ' + + 'summary.', + `You may spawn 'coder' at most ${shots} times in total.`, + ], + }, }, } } @@ -236,14 +329,10 @@ function completionsTransport( } } -/** LIVE completion check: a marker oracle, pending a real verifier lens. The coder profile must - * instruct the harness to print this marker only when the suite genuinely passes. */ -const LIVE_PASS_MARKER = 'ALL TESTS PASS' -const livePassed = (text: string): boolean => text.includes(LIVE_PASS_MARKER) - function liveBackends( env: LiveEnv, cell: CellSpec, + shotPassed: (text: string) => boolean, ): { multishot: MultishotArmBackend graph: GraphArmBackend @@ -254,7 +343,7 @@ function liveBackends( // substrate — each leg matching its graph-arm counterpart, including the driver model. agentTransport: completionsTransport(cell.coderProfile, env.url, env.bearer), driverTransport: completionsTransport(cell.reviewerProfile, env.routerUrl, env.routerKey), - shotPassed: livePassed, + shotPassed, apiKey: env.bearer, baseUrl: env.url, }, @@ -263,7 +352,10 @@ function liveBackends( url: env.url, bearer: env.bearer, router: { routerBaseUrl: env.routerUrl, routerKey: env.routerKey }, - shotPassed: livePassed, + shotPassed, + // Spawns + awaits + a final reply for the shot budget, with slack for re-reads; the + // delegates cap, not this, owns the shot budget. + maxTurns: cell.shots * 6 + 6, }, } } @@ -299,6 +391,10 @@ function printRecord(row: PairedRow): void { export async function main(): Promise { const cli = parseCli(process.argv.slice(2)) + // One nonce per run, embedded in every live task text: the SAME text still reaches both arms + // (input equivalence within a cell) while no prompt is byte-identical across runs, so an + // upstream response cache cannot serve one run's completion to another. + const runNonce = randomBytes(4).toString('hex') const rows: PairedRow[] = [] for (let i = 0; i < cli.cells; i += 1) { let multishotBackend: MultishotArmBackend @@ -312,8 +408,14 @@ export async function main(): Promise { graphBackend = offlineGraphBackend(cell, script).backend } else { const env = requireLiveEnv() - cell = liveParityCell(i, cli.shots, env) - const backends = liveBackends(env, cell) + const task = LIVE_TASK_FAMILY[i] + if (task === undefined) { + throw new Error( + `--cells ${i + 1} exceeds the live task family (${LIVE_TASK_FAMILY.length})`, + ) + } + cell = liveParityCell(i, cli.shots, env, runNonce) + const backends = liveBackends(env, cell, liveShotPassed(expectedAnswerLine(task))) multishotBackend = backends.multishot graphBackend = backends.graph } @@ -324,10 +426,40 @@ export async function main(): Promise { } console.log(`p1-parity — backend=${cli.backend} cells=${cli.cells} shots=${cli.shots}\n`) for (const row of rows) printRecord(row) + printParitySummary(rows, cli) console.log('\nfull records (JSON):') console.log(JSON.stringify(rows, null, 2)) } +/** Paired verdict per cell + the ledger audit: does the graph arm's edge ledger account for + * every traversal it claims (bound worker on every delivered delegates row, shot budget + * respected, shotsUsed = distinct bound workers)? */ +function printParitySummary(rows: ReadonlyArray, cli: CliOptions): void { + console.log('\nparity summary (per cell):') + for (let cell = 1; cell <= cli.cells; cell += 1) { + const multishot = rows.find((r) => r.cell === cell && r.arm === 'multishot')?.record + const graph = rows.find((r) => r.cell === cell && r.arm === 'graph')?.record + if (!multishot || !graph) continue + const agree = multishot.converged === graph.converged + const ledger = graph.ledger ?? [] + const delegates = ledger.filter((t) => t.kind === 'delegates') + const delivered = delegates.filter((t) => t.outcome === 'delivered') + const boundWorkers = new Set( + delivered.map((t) => t.workerId).filter((id): id is string => id !== undefined), + ) + const unbound = delivered.filter((t) => t.workerId === undefined).length + const ledgerAccounts = + unbound === 0 && boundWorkers.size === graph.shotsUsed && boundWorkers.size <= cli.shots + console.log( + `cell ${cell}: converged multishot=${multishot.converged} graph=${graph.converged} ` + + `${agree ? 'AGREE' : 'DIVERGE'} | shots ${multishot.shotsUsed} vs ${graph.shotsUsed} | ` + + `ledger delegates=${delegates.length} delivered=${delivered.length} ` + + `other=${delegates.filter((t) => t.outcome !== 'delivered').length} ` + + `boundWorkers=${boundWorkers.size} accounts=${ledgerAccounts}`, + ) + } +} + if (import.meta.url === `file://${process.argv[1]}`) { main().catch((err) => { console.error(err)