diff --git a/benchmarks/trace-repair/tools/assay-tbench-corpus.mjs b/benchmarks/trace-repair/tools/assay-tbench-corpus.mjs index d032d548..0320ab45 100644 --- a/benchmarks/trace-repair/tools/assay-tbench-corpus.mjs +++ b/benchmarks/trace-repair/tools/assay-tbench-corpus.mjs @@ -512,6 +512,13 @@ function verifyFailToPass(tb2Dir, task) { return null } } + // A run that dies before test.sh writes the reward must read as null, not + // as the previous run's verdict, so the file is removed before every run. + const clearReward = () => + sh(['exec', cid, 'rm', '-f', '/logs/verifier/reward.txt', '/logs/verifier/ctrf.json'], { + stdio: 'ignore', + }) + clearReward() const before = timed(() => sh(['exec', cid, 'bash', '/tests/test.sh'], { stdio: 'ignore' })) result.baselineSeconds = before.seconds result.baselineReward = readReward() @@ -521,6 +528,7 @@ function verifyFailToPass(tb2Dir, task) { result.solutionSeconds = solve.seconds result.solutionOk = solve.ok + clearReward() const after = timed(() => sh(['exec', cid, 'bash', '/tests/test.sh'], { stdio: 'ignore' })) result.regradeSeconds = after.seconds result.repairedReward = readReward() @@ -531,6 +539,7 @@ function verifyFailToPass(tb2Dir, task) { // whose verdict moves on identical bytes cannot price an intervention. const replicates = [] for (let index = 0; index < REGRADE_REPLICATES; index++) { + clearReward() // Captured to a file rather than discarded, because pytest's `-rA` summary is where // the per-parameter verdicts are and a base name is a conjunction over them. const run = timed(() => diff --git a/docs/trace-repair-free-lunch.md b/docs/trace-repair-free-lunch.md new file mode 100644 index 00000000..fd08b9e8 --- /dev/null +++ b/docs/trace-repair-free-lunch.md @@ -0,0 +1,119 @@ +# What unconditional continuation rescues + +TB-Repair's admission condition 3 asks whether a row is rescued by continuing from the recorded end state with no intervention. +Both milestone runs answered it under a control pinned to **zero model calls**. +A rollout that makes no model call executes no command, so the container the control graded held the same bytes the end-state check had already graded as failing: on the 32 rows measured here that control returned **0 passes in 96 rollouts**, the only answer it could return. + +This is the same question asked with a budget. + +Source: [`scripts/tb-repair-freelunch.ts`](../scripts/tb-repair-freelunch.ts). +The control contract is in [trace-repair-admission.md](./trace-repair-admission.md); the policy is in [trace-repair-continuation.md](./trace-repair-continuation.md). + +## The answer + +**3 of 64 rollouts, 4.7 %.** Two of 32 rows were rescued at least once, 6.2 %. + +The 64 rollouts are not 64 independent draws. +A seed-derivation defect (threat 8) made the second pass repeat the first pass's seed, and 14 of the 32 second-pass rollouts are byte-identical action repeats of their first-pass rollout. +Over the 50 distinct rollouts the rate is **3 of 50, 6.0 %**. +All three rescues come from pairs whose two rollouts differ, so no rescue is a repeat counted twice. + +| interval, 95 % | rescue rate | +| --- | --- | +| task-clustered bootstrap (3 clusters, 10 000 resamples, seed 7) | 0.0 % – 10.0 % | +| row-clustered bootstrap (32 clusters) | 0.0 % – 12.5 % | +| exact Clopper-Pearson on rollouts | 1.0 % – 13.1 % | + +Three clusters cannot carry a stable interval; the row-clustered and exact intervals are reported beside it for that reason. + +**The rate is not zero, and it is not noise.** One row, `count-dataset-tokens__HL3ZzrX`, was rescued in **both** of its rollouts on a suite that returned the same verdict in all 16 certification replicates. + +## What a rescue looked like + +`count-dataset-tokens__HL3ZzrX`, 13 steps, submitted. The recorded agent had computed the right answer and written it in a form the grader rejected. The continuation recomputed the count offline, then found and removed the formatting defect: + +``` +10. cat /app/answer.txt +11. printf '%s' "$(cat /app/answer.txt)" > /app/answer.txt && cat /app/answer.txt | xxd | head -5 +12. printf '%s' "$(cat /app/answer.txt)" > /app/answer.txt && od -c /app/answer.txt +13. echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT +``` + +`password-recovery__oDL7kv9`, 19 steps, submitted, rescued in 1 of 2 rollouts. This one is not a formatting fix — it is the task being solved: the continuation searched the disk image, hexdumped it, carved an embedded archive, and wrote the recovered password. + +## By task, and the asymmetry that explains it + +| task | rows | rollouts | passes | rate | recorded steps that used the network | +| --- | --- | --- | --- | --- | --- | +| `count-dataset-tokens` | 10 | 20 | 2 | 10.0 % | 24.3 % | +| `password-recovery` | 6 | 12 | 1 | 8.3 % | 0 % | +| `sanitize-git-repo` | 16 | 32 | 0 | 0 % | 0 % | + +The pinned policy disables the network the recorded agents had. Measured on the continuations' own actions, `count-dataset-tokens` rollouts spent **30.2 % of their steps reaching for a network that was not there** (114 of 377), against 0.8 % for `sanitize-git-repo` and 0 % for `password-recovery`. +`password-recovery` is the clean sub-population: its recordings never used the network, so its 8.3 % is unconfounded by the policy. + +## By exit status + +| exit status | rollouts | passes | rate | +| --- | --- | --- | --- | +| `submitted` | 15 | 3 | 20.0 % | +| `step-budget-exhausted` | 46 | 0 | 0 % | +| `model-error` | 3 | 0 | 0 % | + +Every rescue came from a rollout that decided it was finished. No rollout that burned all 20 steps ever passed. +The three `model-error` rollouts were ended by provider 503s after four retries; they are recorded and graded, never dropped, so the rate is a lower bound by at most those three. + +## What it cost + +64 rollouts, priced from the token counts the provider reported for this run's own calls at the router's published `glm-5.2` rate. + +| quantity | min | median | p90 | max | sum | +| --- | --- | --- | --- | --- | --- | +| prompt tokens | 15 146 | 170 939 | 260 088 | 321 288 | 10 734 764 | +| completion tokens | 151 | 6 663 | 14 961 | 23 866 | 546 346 | +| continuation steps | 1 | 20 | 20 | 20 | 1 143 | +| cost, USD | 0.0138 | 0.1506 | 0.2281 | 0.2821 | **9.4806** | + +**$0.1481 per rollout, $0.2963 per row at n = 2.** A paired study can budget-match against those two numbers directly. + +Cost is attributed per call, not from the router's account counter: `GET /v1/credits` covers the whole key, and 18 other processes on this host were calling it during the run. The counter's delta over the two passes was $8.60, which is neither this run's cost nor an upper bound on it once other traffic is in both directions. + +## What the number means + +A **high** rate would have meant unconditional continuation captures most of the available headroom, killing the gated-stop thesis. It did not. + +A **low but non-zero** rate means the headroom exists and a gate could claim it — which licenses a paired study without proving it. That is where this lands, with two qualifications that matter more than the point estimate: + +- The rescues are **not free**. Each cost $0.148 and up to 20 model calls. A gate that fires on every failed row pays that on every row. +- **Condition 3 is now calibrated.** It has a real screen rate to compare against: 4.7 % of rollouts and 6.2 % of rows, against the 0 % a zero-step control was structurally obliged to report. + +## Threats + +1. **Reconstructed assistant messages.** The corpus stores each recorded assistant turn as an elided placeholder, so the continuation inherits the bash block without the reasoning that produced it. +2. **Network asymmetry.** The recorded agents had internet; the pinned policy does not. On `count-dataset-tokens` that consumed 30.2 % of continuation steps, so the overall rate is a lower bound for a networked continuation. +3. **Three clusters.** A task-clustered interval over three tasks is coarse by construction. +4. **One model, `glm-5.2` at temperature 0.** Not a statement about continuation in general. +5. **n = 2.** Within-row variance is measured on two draws. One row rescued twice, one rescued once of two. +6. **Wall-time is not clean.** For part of the first pass the measurement seat was not held, because a killed sibling wrapper's exit trap removed a lock it no longer owned. Verdicts and token counts are unaffected; latency and throughput are not clean. +7. **Snapshot boundary.** State moves between container generations as a committed image, so a process the recording left running does not survive. +8. **The two passes shared one seed.** Each pass called `runContinuation` with `rollouts: 1`, and the seed derived from that call's internal index, which is always 0. Shifting `ROLLOUT_BASE` therefore never reached the seed, and both passes sent the provider the index-0 seed on an identical prompt. Measured on the records: 14 of 32 row pairs are byte-identical action sequences, and 31 of 32 share their first action. The runner now forwards the pass's base index through `rolloutBase`, so a future pass draws its own seed. + +## Reproducing + +```bash +# containers only, no model calls, no seat needed +npx tsx scripts/tb-repair-freelunch.ts --stop-points-only + +# one uniform pass over every row, under the measurement seat +TBR_FL_ROLLOUTS=1 TBR_FL_ROLLOUT_BASE=0 TBR_FL_OUT=freelunch-pass1.json \ + npx tsx scripts/tb-repair-freelunch.ts +``` + +`--plan` prints the denominator chain and the selected rows without opening a container. +The pre-registration, its amendments, and the raw per-rollout records are in `~/bench-cache/freelunch-20260810/`. + +## The raw records stay local + +`freelunch.json` holds every continuation's actions and observations, which are container state. GitHub push protection refused an earlier commit of it because a container carried a **Hugging Face user access token** in its cached credentials, at `free-lunch-n2.json:1883`. + +Raw per-rollout records are therefore kept out of the repository. What is committed is the runner, this report, and the numbers derived from the records. Anyone re-running the campaign should treat the artifact directory as credential-bearing. diff --git a/scripts/tb-repair-freelunch-model.ts b/scripts/tb-repair-freelunch-model.ts new file mode 100644 index 00000000..9a8a9894 --- /dev/null +++ b/scripts/tb-repair-freelunch-model.ts @@ -0,0 +1,204 @@ +/** + * The continuation's model call: one chat completion on router.tangle.tools, + * with the served id, the token usage and a priced cost recorded per call. + * + * Two rules keep a partial measurement from reading as a complete one: + * + * - a response that reports no usage returns `usage: null`, so the rollout's + * `captured` flag clears rather than being filled with zeros + * - a model with no local price returns `costUsd: null`, which makes the + * whole rollout's cost `uncaptured` rather than a sum that is too small + * + * The router's published per-token rates are the price source. They are read + * once from `/v1/models` at start-up rather than hard-coded, so a rate change + * cannot silently misprice a run. + */ + +import type { RunTokenUsage } from '../src/run-record' +import type { + ContinuationModel, + ContinuationModelRequest, + ContinuationModelResponse, +} from '../src/trace-repair' + +const ROUTER = process.env.TBR_FL_ROUTER ?? 'https://router.tangle.tools/v1' +const REQUEST_TIMEOUT_MS = Number(process.env.TBR_FL_TIMEOUT_MS ?? '180000') +const MAX_ATTEMPTS = Number(process.env.TBR_FL_ATTEMPTS ?? '4') + +export interface ModelPricing { + /** USD per prompt token. */ + prompt: number + /** USD per completion token, reasoning tokens included. */ + completion: number +} + +interface RouterUsage { + prompt_tokens?: number + completion_tokens?: number + total_tokens?: number + completion_tokens_details?: { reasoning_tokens?: number } + prompt_tokens_details?: { cached_tokens?: number } +} + +interface RouterChoice { + message?: { content?: string | null } + finish_reason?: string | null +} + +interface RouterResponse { + model?: string + choices?: RouterChoice[] + usage?: RouterUsage + error?: { message?: string; code?: string } +} + +const pricingCache = new Map() + +/** + * Whether the served id is the model the policy asked for. + * + * The router lists ids both bare and vendor-prefixed (`glm-5.2` and + * `z-ai/glm-5.2`) and answers with one form or the other, so the comparison is + * on the final path segment. It is not a fuzzy match: `deepseek-v4-flash` does + * not satisfy a request for `deepseek-v3.2` under this rule. + */ +export function servedMatches(requested: string, served: string): boolean { + const tail = (id: string): string => id.split('/').pop()!.toLowerCase() + return tail(requested) === tail(served) +} + +function apiKey(): string { + const key = process.env.TANGLE_API_KEY + if (!key) throw new Error('TANGLE_API_KEY is required for the continuation model') + return key +} + +/** Per-token rates the router publishes for `model`. Throws when it publishes none. */ +export async function fetchPricing(model: string): Promise { + const cached = pricingCache.get(model) + if (cached) return cached + const response = await fetch(`${ROUTER}/models`, { + headers: { authorization: `Bearer ${apiKey()}` }, + }) + if (!response.ok) throw new Error(`router model list failed: HTTP ${response.status}`) + const body = (await response.json()) as { + data: { id: string; pricing?: { prompt?: string; completion?: string } }[] + } + for (const entry of body.data) { + const prompt = Number(entry.pricing?.prompt) + const completion = Number(entry.pricing?.completion) + if (Number.isFinite(prompt) && Number.isFinite(completion)) { + pricingCache.set(entry.id, { prompt, completion }) + } + } + const found = pricingCache.get(model) + if (!found) throw new Error(`router publishes no per-token price for ${model}`) + return found +} + +function toRunTokenUsage(usage: RouterUsage | undefined): RunTokenUsage | null { + if (!usage) return null + const input = usage.prompt_tokens + const output = usage.completion_tokens + if (typeof input !== 'number' || typeof output !== 'number') return null + const record: RunTokenUsage = { input, output } + const reasoning = usage.completion_tokens_details?.reasoning_tokens + if (typeof reasoning === 'number') record.reasoning = reasoning + const cached = usage.prompt_tokens_details?.cached_tokens + if (typeof cached === 'number') record.cached = cached + return record +} + +/** + * Retries on transport failure and on the router's own 5xx, which it returns + * for transient upstream capacity. A request that never succeeds throws, and + * `runContinuation` records the rollout as `model-error` with the message. + */ +async function post(request: ContinuationModelRequest): Promise { + let lastError = '' + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS) + try { + const response = await fetch(`${ROUTER}/chat/completions`, { + method: 'POST', + signal: controller.signal, + headers: { + authorization: `Bearer ${apiKey()}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: request.model, + messages: request.messages, + temperature: request.temperature, + max_tokens: request.maxTokens, + seed: request.seed, + }), + }) + const text = await response.text() + if (!response.ok) { + lastError = `HTTP ${response.status}: ${text.slice(0, 300)}` + if (response.status >= 500 || response.status === 429) { + await new Promise((resolve) => setTimeout(resolve, attempt * 5_000)) + continue + } + throw new Error(lastError) + } + return JSON.parse(text) as RouterResponse + } catch (error) { + lastError = (error as Error).message + if (attempt === MAX_ATTEMPTS) break + await new Promise((resolve) => setTimeout(resolve, attempt * 5_000)) + } finally { + clearTimeout(timer) + } + } + throw new Error(`router call failed after ${MAX_ATTEMPTS} attempts: ${lastError}`) +} + +/** The router answered with a different model than the policy pinned. */ +export class ModelSubstitutionError extends Error {} + +export function routerContinuationModel(label = ''): ContinuationModel { + return async (request: ContinuationModelRequest): Promise => { + const startedMs = Date.now() + const body = await post(request) + if (body.error) throw new Error(`router error: ${body.error.code}: ${body.error.message}`) + const choice = body.choices?.[0] + const content = choice?.message?.content + if (typeof content !== 'string') { + throw new Error(`router returned no message content for ${request.model}`) + } + const served = body.model + if (typeof served !== 'string' || served.length === 0) { + throw new Error('router returned no served model id; provenance cannot be recorded') + } + // The router can answer a request for one model with another. Measured: + // `deepseek/deepseek-v3.2` was served by `deepseek-v4-flash`. A pinned + // policy that silently ran a substitute would name the wrong model in every + // number it produced, so a substitution stops the rollout. + if (!servedMatches(request.model, served)) { + throw new ModelSubstitutionError( + `policy pinned ${request.model}, router served ${served}`, + ) + } + // Priced on what was served, never on what was asked for. + const pricing = await fetchPricing(served) + const usage = toRunTokenUsage(body.usage) + // One line per call, so a run that takes hours shows where it is rather + // than going dark between rollouts. + process.stderr.write( + `${new Date().toISOString()} call${label ? ` ${label}` : ''} served=${served} ` + + `${Date.now() - startedMs}ms in=${usage?.input ?? 'null'} out=${usage?.output ?? 'null'} ` + + `finish=${choice?.finish_reason ?? 'null'}\n`, + ) + return { + content, + servedModel: served, + usage, + costUsd: + usage === null ? null : usage.input * pricing.prompt + usage.output * pricing.completion, + finishReason: choice?.finish_reason ?? null, + } + } +} diff --git a/scripts/tb-repair-freelunch.ts b/scripts/tb-repair-freelunch.ts new file mode 100644 index 00000000..7afa5c92 --- /dev/null +++ b/scripts/tb-repair-freelunch.ts @@ -0,0 +1,794 @@ +/** + * The free-lunch measurement: what fraction of failed runs does unconditional + * continuation rescue when it is given a real budget? + * + * One arm. No analyst, no hint, no gate, no injected action. A row is replayed + * to the stop point its recording ended at, and the pinned mini-swe-agent + * policy runs forward from there under a real model. + * + * This is the control TB-Repair's admission condition 3 always meant and never + * ran: both milestone runs screened rows with a control pinned to zero model + * calls, which executes no command and therefore grades the same bytes the + * end-state check already graded as failing. + * + * Three container generations per rollout, because two constraints point in + * opposite directions: + * + * replay default network, as the recording had and the milestones used + * continue `--network none`, which the pinned policy requires + * grade network again, because every task's `test.sh` runs apt-get, + * curl and uvx before it can run a single assertion + * + * A container created in `none` mode cannot be attached to a network, so the + * state moves between generations as a committed image rather than as a + * re-networked container. + */ + +import { execFile } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { + type ContinuationMessage, + type ContinuationRollout, + createDockerContinuationEnvironment, + definePinnedContinuationPolicy, + injectedTestOracle, + MINI_SWE_SYSTEM_MESSAGE, + nodeProcessRunner, + type OracleDeterminismVerdict, + parseTaskOracleRegistry, + type PinnedContinuationPolicy, + type RepairSession, + renderInstanceMessage, + renderObservation, + runContinuation, + type TaskOracleRegistry, + type TestOracle, + type TestSuiteFile, + testSuiteDigest, +} from '../src/trace-repair' +import type { RecordedTrajectoryStep } from '../src/trajectory-replay/steps' +import { routerContinuationModel } from './tb-repair-freelunch-model' + +const run = promisify(execFile) + +const REPO = '/home/drew/bench-cache/terminal-bench-2' +const CORPUS = '/home/drew/bench-cache/t8-milestone2' +const WORK = process.env.TBR_FL_WORK ?? '/home/drew/bench-cache/freelunch-20260810' + +/** Rollouts per row in this invocation. */ +const ROLLOUTS = Number(process.env.TBR_FL_ROLLOUTS ?? '3') +/** + * First rollout index this invocation produces. + * + * A campaign that adds rollouts in uniform passes runs one invocation per pass + * and shifts the base, so pass 2 draws the seeds of index 1 rather than + * redrawing index 0. The seed derives from the row and the index, so an index + * that repeats is a rollout that repeats. + */ +const ROLLOUT_BASE = Number(process.env.TBR_FL_ROLLOUT_BASE ?? '0') +const SEED = Number(process.env.TBR_FL_SEED ?? '20260810') +const MODEL = process.env.TBR_FL_MODEL ?? 'deepseek/deepseek-v3.2' + +const STEP_TIMEOUT_MS = 300_000 +const VERIFIER_TIMEOUT_MS = 900_000 +/** + * Bound for a step the recording itself killed for running too long. The + * scaffold renders such a step with no returncode, so waiting the full bound + * buys no information. This changes what the run costs, never what it counts. + */ +const RECORDED_TIMEOUT_STEP_MS = 60_000 +const RECORDED_TIMEOUT_MARKER = 'timed out and has been killed' + +const TASK_ORACLES_PATH = join( + import.meta.dirname, + '..', + 'benchmarks', + 'trace-repair', + 'task-oracles.json', +) + +const CONTINUATION_POLICY: PinnedContinuationPolicy = definePinnedContinuationPolicy({ + model: MODEL, + seed: SEED, +}) + +interface CorpusRow { + rowId: string + taskName: string + recordedModel: string + recordedCommands: number + finalReturncode: number | null + steps: { step_id: number; action: string; observation: string | null }[] +} + +interface AdmitRecord { + rowId: string + taskName: string + stratum: string + admitted: boolean + rejection: string | null + prefixDivergenceRatio: number | null + evidence: { endStatePassed: boolean } | null +} + +interface TaskFixture { + name: string + image: string + cwd: string + suite: readonly TestSuiteFile[] + suiteDigest: string + instruction: string +} + +function stepTimeoutMs(observation: string | null): number { + return observation !== null && observation.includes(RECORDED_TIMEOUT_MARKER) + ? RECORDED_TIMEOUT_STEP_MS + : STEP_TIMEOUT_MS +} + +function recordedReturncode(observation: string | null): number | null { + if (!observation) return null + const m = /(-?\d+)<\/returncode>/.exec(observation) + return m ? Number(m[1]) : null +} + +function loadSuite(task: string): TestSuiteFile[] { + const dir = join(REPO, task, 'tests') + const files: TestSuiteFile[] = [] + const walk = (rel: string): void => { + for (const entry of readdirSync(join(dir, rel))) { + const relPath = rel ? `${rel}/${entry}` : entry + if (statSync(join(dir, relPath)).isDirectory()) walk(relPath) + else + files.push({ + path: `/tests/${relPath}`, + contents: readFileSync(join(dir, relPath), 'utf8'), + }) + } + } + walk('') + return files +} + +async function loadTask(name: string): Promise { + const tag = `alexgshaw/${name}:20251031` + const { stdout: workdir } = await run('docker', [ + 'image', + 'inspect', + tag, + '--format', + '{{.Config.WorkingDir}}', + ]) + const { stdout: digest } = await run('docker', [ + 'image', + 'inspect', + tag, + '--format', + '{{index .RepoDigests 0}}', + ]) + const suite = loadSuite(name) + return { + name, + image: digest.trim(), + cwd: workdir.trim() || '/app', + suite, + suiteDigest: testSuiteDigest(suite), + instruction: readFileSync(join(REPO, name, 'instruction.md'), 'utf8'), + } +} + +/** + * The held-out suite, uploaded from outside the container at grade time and + * digest-checked after upload. `test.sh` always exits 0 and writes its verdict + * to `/logs/verifier/reward.txt`, so the command reads that file. + */ +function taskOracle(task: TaskFixture): TestOracle { + return injectedTestOracle({ + files: task.suite, + command: + 'chmod +x /tests/test.sh; rm -f /logs/verifier/reward.txt; ' + + '(/tests/test.sh) > /logs/verifier/test-stdout.txt 2>&1; ' + + 'grep -qx 1 /logs/verifier/reward.txt', + purge: ['/tests'], + commandTimeoutMs: VERIFIER_TIMEOUT_MS, + }) +} + +/** A networked container at the task image or a snapshot of one. */ +async function startNetworkedContainer(image: string, name: string): Promise { + await run('docker', [ + 'run', + '-d', + '--name', + name, + '--entrypoint', + '', + '--memory', + '2g', + '--cpus', + '1', + image, + 'sleep', + 'infinity', + ]) + await run('docker', ['exec', name, 'mkdir', '-p', '/logs/verifier', '/logs/agent', '/logs/artifacts']) + return name +} + +function dockerSession(name: string, cwd: string): RepairSession { + return { + ref: name, + async exec(command: string, timeoutMs: number) { + const seconds = Math.ceil(timeoutMs / 1000) + try { + const { stdout, stderr } = await run( + 'docker', + [ + 'exec', + '-e', + 'DEBIAN_FRONTEND=noninteractive', + '-w', + cwd, + name, + 'timeout', + '--kill-after=5', + String(seconds), + 'bash', + '-lc', + command, + ], + { maxBuffer: 64 * 1024 * 1024, timeout: timeoutMs + 30_000 }, + ) + return { exitCode: 0, stdout, stderr, timedOut: false } + } catch (error) { + const err = error as { code?: number; stdout?: string; stderr?: string } + const exitCode = typeof err.code === 'number' ? err.code : 1 + return { + exitCode, + stdout: err.stdout ?? '', + stderr: err.stderr ?? '', + timedOut: exitCode === 124 || exitCode === 137, + } + } + }, + async close() { + await run('docker', ['rm', '-f', name]).catch(() => undefined) + }, + } +} + +interface ReplayResult { + divergences: number + replayed: number + /** The message list the continuation inherits, ending on the last observation. */ + prefix: ContinuationMessage[] +} + +/** + * Replay every recorded command and build the message list the continuation + * inherits. + * + * The corpus stores each recorded assistant message as an elided placeholder, + * so the reasoning text does not exist. A turn is rendered as the bash block + * alone rather than with invented prose, and every observation is the one this + * replay produced. + */ +async function replayPrefix( + session: RepairSession, + task: TaskFixture, + steps: readonly RecordedTrajectoryStep[], + systemInformation: string, +): Promise { + const prefix: ContinuationMessage[] = [ + { role: 'system', content: MINI_SWE_SYSTEM_MESSAGE }, + { + role: 'user', + content: renderInstanceMessage({ task: task.instruction, systemInformation }), + }, + ] + let divergences = 0 + for (const step of steps) { + const result = await session.exec(step.action, stepTimeoutMs(step.observation)) + const recorded = recordedReturncode(step.observation) + if (recorded === null || recorded !== result.exitCode) divergences += 1 + prefix.push({ role: 'assistant', content: `\`\`\`bash\n${step.action}\n\`\`\`` }) + prefix.push({ + role: 'user', + content: renderObservation({ + returncode: result.exitCode, + output: `${result.stdout}${result.stderr}`, + }), + }) + } + return { divergences, replayed: steps.length, prefix } +} + +interface RolloutOutcome { + rowId: string + taskName: string + rolloutIndex: number + /** Divergences of the row's replay against the recording, measured once. */ + divergences: number + replayed: number + passed: boolean + gradeExitCode: number + gradeTimedOut: boolean + suiteDigest: string + continuationSteps: number + exitStatus: string + submission: string | null + terminalError: string | null + usage: ContinuationRollout['usage'] + costProvenance: ContinuationRollout['costProvenance'] + servedModels: string[] + networkMode: string + continuationWallMs: number + totalWallMs: number + actions: string[] +} + +/** Docker-safe fragment of a row id. */ +function slug(rowId: string): string { + return rowId.replace(/[^a-zA-Z0-9]+/g, '-').slice(-40).replace(/^-+/, '') +} + +interface StopPoint { + /** Image holding the state the recording stopped at. */ + image: string + replay: ReplayResult + buildWallMs: number + /** True when this invocation found the stop point already built. */ + reused: boolean +} + +/** + * Where a built stop point is recorded so a later pass reuses it. + * + * A campaign that adds rollouts in passes must hand every pass the same stop + * point, or the passes differ by their replay as well as by their rollout. The + * image carries the container state and this file carries the message list and + * the divergence count the replay produced, which cannot be recovered from the + * image. + */ +function stopPointRecordPath(rowId: string): string { + return join(WORK, 'stop-points', `${slug(rowId)}.json`) +} + +/** + * Reconstruct the recorded stop point once for a row and snapshot it. + * + * Every rollout of the row starts from this one image, so the stop point is + * held constant and the only thing that varies within a row is the + * continuation. It also pays the expensive commit once rather than per rollout: + * a recording that installed packages writes a multi-gigabyte layer. + */ +async function buildStopPoint( + row: CorpusRow, + task: TaskFixture, + systemInformation: string, +): Promise { + const startedMs = Date.now() + const base = `tbfl-${slug(row.rowId)}`.toLowerCase() + const image = `tbfl/stop:${base}` + const recordPath = stopPointRecordPath(row.rowId) + + const built = await run('docker', ['image', 'inspect', image, '--format', '{{.Id}}']).then( + () => true, + () => false, + ) + if (built) { + const record = JSON.parse(readFileSync(recordPath, 'utf8')) as { + image: string + replay: ReplayResult + buildWallMs: number + } + if (record.image !== image) { + throw new Error(`stop-point record for ${row.rowId} names image ${record.image}, not ${image}`) + } + return { ...record, reused: true } + } + + const steps: RecordedTrajectoryStep[] = row.steps.map((s) => ({ + step_id: s.step_id, + action: s.action, + observation: s.observation, + })) + const container = await startNetworkedContainer(task.image, `${base}-replay-${randomUUID().slice(0, 6)}`) + try { + const replay = await replayPrefix( + dockerSession(container, task.cwd), + task, + steps, + systemInformation, + ) + await run('docker', ['commit', container, image]) + const record = { image, replay, buildWallMs: Date.now() - startedMs } + mkdirSync(join(WORK, 'stop-points'), { recursive: true }) + writeFileSync(recordPath, JSON.stringify(record)) + return { ...record, reused: false } + } finally { + await run('docker', ['rm', '-f', container]).catch(() => undefined) + } +} + +async function runRollout( + row: CorpusRow, + task: TaskFixture, + stopPoint: StopPoint, + rolloutIndex: number, + log: (m: string) => void, +): Promise { + const startedMs = Date.now() + const base = `tbfl-${slug(row.rowId)}-${rolloutIndex}-${randomUUID().slice(0, 6)}`.toLowerCase() + const replayImage = stopPoint.image + const continuedImage = `tbfl/continued:${base}` + const replay = stopPoint.replay + + const continuationStartedMs = Date.now() + const continuationContainer = `${base}-cont` + let rollout: ContinuationRollout + try { + const rollouts = await runContinuation({ + policy: CONTINUATION_POLICY, + arm: 'no-fix-control', + rowId: row.rowId, + prefix: replay.prefix, + rollouts: 1, + // The seed derives from the global rollout index, so a pass that shifts + // ROLLOUT_BASE draws new seeds instead of redrawing index 0. + rolloutBase: rolloutIndex, + model: routerContinuationModel(`${row.rowId.split('::')[1] ?? row.rowId}#${rolloutIndex}`), + environments: { + id: 'docker-network-none', + async create() { + await run('docker', [ + 'run', + '-d', + '--name', + continuationContainer, + '--entrypoint', + '', + '--network', + 'none', + '--memory', + '2g', + '--cpus', + '1', + '-w', + task.cwd, + replayImage, + 'sleep', + '4h', + ]) + return createDockerContinuationEnvironment({ + containerRef: continuationContainer, + cwd: task.cwd, + runProcess: nodeProcessRunner, + // The container is committed after the rollout, so this owns it. + removeOnDispose: false, + }) + }, + }, + }) + const only = rollouts[0] + if (only === undefined) throw new Error(`no rollout recorded for ${row.rowId}#${rolloutIndex}`) + rollout = only + await run('docker', ['commit', continuationContainer, continuedImage]) + } finally { + // The environment factory keeps the container alive so it can be committed, + // so the container is removed here whether the commit happened or a throw + // skipped it. The stop-point image outlives the rollout and is removed by + // the row that owns it. + await run('docker', ['rm', '-f', continuationContainer]).catch(() => undefined) + } + const continuationWallMs = Date.now() - continuationStartedMs + + const gradeContainer = await startNetworkedContainer(continuedImage, `${base}-grade`) + const gradeSession = dockerSession(gradeContainer, task.cwd) + try { + const graded = await taskOracle(task).grade(gradeSession, { + rowId: row.rowId, + arm: 'no-fix-control', + rolloutIndex, + }) + const outcome: RolloutOutcome = { + rowId: row.rowId, + taskName: row.taskName, + rolloutIndex, + divergences: replay.divergences, + replayed: replay.replayed, + passed: graded.passed, + gradeExitCode: graded.exitCode, + gradeTimedOut: graded.timedOut, + suiteDigest: graded.suiteDigest, + continuationSteps: rollout.steps.length, + exitStatus: rollout.exitStatus, + submission: rollout.submission, + terminalError: rollout.terminalError ?? null, + usage: rollout.usage, + costProvenance: rollout.costProvenance, + servedModels: [...new Set(rollout.steps.map((s) => s.model.servedModel))], + networkMode: rollout.environment.networkMode, + continuationWallMs, + totalWallMs: Date.now() - startedMs, + actions: rollout.steps.map((s) => s.action ?? ''), + } + log( + `${row.rowId}#${rolloutIndex} passed=${outcome.passed} exit=${outcome.exitStatus} ` + + `steps=${outcome.continuationSteps} div=${replay.divergences}/${replay.replayed} ` + + `in=${outcome.usage.input} out=${outcome.usage.output} ` + + `cost=${outcome.costProvenance.usd === null ? 'uncaptured' : `$${outcome.costProvenance.usd.toFixed(4)}`} ` + + `${outcome.totalWallMs}ms`, + ) + writeFileSync(join(WORK, 'rollouts.jsonl'), `${JSON.stringify(outcome)}\n`, { flag: 'a' }) + return outcome + } finally { + await gradeSession.close() + await run('docker', ['rmi', '-f', continuedImage]).catch(() => undefined) + } +} + +async function mapLimit( + items: readonly T[], + limit: number, + fn: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length) + let next = 0 + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next + next += 1 + if (index >= items.length) return + results[index] = await fn(items[index]!, index) + } + }), + ) + return results +} + +function loadRows(): { primary: CorpusRow[]; divergent: CorpusRow[]; chain: Record } { + const rows: CorpusRow[] = JSON.parse(readFileSync(join(CORPUS, 'rows-all.json'), 'utf8')) + const admit: { records: AdmitRecord[] } = JSON.parse( + readFileSync(join(CORPUS, 'out', 'admit.json'), 'utf8'), + ) + const byRow = new Map(admit.records.map((r) => [r.rowId, r])) + const oracles = loadTaskOracles() + + const chain: Record = { evaluated: rows.length } + const certified = rows.filter((r) => oracles.get(r.taskName)?.stable === true) + chain.deterministicOracle = certified.length + const cleanExit = certified.filter((r) => r.finalReturncode === 0) + chain.cleanExit = cleanExit.length + const failed = cleanExit.filter((r) => byRow.get(r.rowId)?.evidence?.endStatePassed === false) + chain.failedEndState = failed.length + const primary = failed.filter((r) => byRow.get(r.rowId)?.rejection === null) + const divergent = failed.filter((r) => byRow.get(r.rowId)?.rejection !== null) + chain.prefixFidelityOk = primary.length + chain.prefixDivergent = divergent.length + if (failed.length !== cleanExit.length) { + throw new Error( + `expected every clean-exit row to have failed its end state; ${cleanExit.length - failed.length} did not`, + ) + } + return { primary, divergent, chain } +} + +function loadTaskOracles(): TaskOracleRegistry { + return parseTaskOracleRegistry(JSON.parse(readFileSync(TASK_ORACLES_PATH, 'utf8'))) +} + +async function routerSpend(): Promise<{ totalUsage: number; totalCredits: number }> { + const key = process.env.TANGLE_API_KEY + if (!key) throw new Error('TANGLE_API_KEY is required to read the router spend counter') + const response = await fetch('https://router.tangle.tools/v1/credits', { + headers: { authorization: `Bearer ${key}` }, + }) + if (!response.ok) throw new Error(`router credits read failed: HTTP ${response.status}`) + const body = (await response.json()) as { data: { total_credits: number; total_usage: number } } + return { totalUsage: body.data.total_usage, totalCredits: body.data.total_credits } +} + +async function main(): Promise { + mkdirSync(WORK, { recursive: true }) + const logPath = join(WORK, 'run.log') + const log = (message: string): void => { + const line = `${new Date().toISOString()} ${message}\n` + process.stdout.write(line) + writeFileSync(logPath, line, { flag: 'a' }) + } + + const { primary, divergent, chain } = loadRows() + const only = process.env.TBR_FL_ONLY?.split(',').map((s) => s.trim()).filter(Boolean) + const includeDivergent = process.env.TBR_FL_SET === 'divergent' + const selected = (includeDivergent ? divergent : primary).filter( + (r) => !only || only.includes(r.rowId), + ) + const limit = Number(process.env.TBR_FL_LIMIT ?? '0') + const rows = limit > 0 ? selected.slice(0, limit) : selected + + // A stop point is addressed by the row's slug, so two rows sharing one would + // silently share a container state. Checked before anything is built. + const bySlug = new Map() + for (const row of rows) { + const key = slug(row.rowId) + const other = bySlug.get(key) + if (other !== undefined) { + throw new Error(`rows ${other} and ${row.rowId} share the stop-point slug ${key}`) + } + bySlug.set(key, row.rowId) + } + + const taskNames = [...new Set(rows.map((r) => r.taskName))] + const tasks = new Map() + for (const name of taskNames) tasks.set(name, await loadTask(name)) + const oracles = loadTaskOracles() + + // The denominator chain and the selected rows, with no container and no + // model call, so the set a run will spend on can be read before it spends. + if (process.argv.includes('--plan')) { + const perTask = new Map() + for (const row of rows) perTask.set(row.taskName, (perTask.get(row.taskName) ?? 0) + 1) + process.stdout.write( + `${JSON.stringify( + { + chain, + selected: rows.length, + rollouts: ROLLOUTS, + perTask: Object.fromEntries(perTask), + excludedTasks: [...oracles] + .filter(([, v]) => !v.stable) + .map(([name, v]) => ({ name, flipRate: v.flipRate, replicates: v.replicates })), + images: Object.fromEntries([...tasks].map(([n, t]) => [n, t.image])), + rowIds: rows.map((r) => r.rowId), + }, + null, + 2, + )}\n`, + ) + return + } + + const { stdout: uname } = await run('uname', ['-a']) + const systemInformation = uname.trim() + + // Reconstructing the stop points needs containers and no model call, so a + // campaign waiting for the measurement seat can build them first and spend + // the seat's time on model calls alone. + if (process.argv.includes('--stop-points-only')) { + const built: Record = {} + await mapLimit(rows, Number(process.env.TBR_FL_CONCURRENCY ?? '6'), async (row) => { + try { + const stopPoint = await buildStopPoint(row, tasks.get(row.taskName)!, systemInformation) + built[row.rowId] = { + reused: stopPoint.reused, + divergences: stopPoint.replay.divergences, + replayed: stopPoint.replay.replayed, + buildWallMs: stopPoint.buildWallMs, + } + log( + `${row.rowId} stop point ${stopPoint.reused ? 'reused' : 'built'} ` + + `div=${stopPoint.replay.divergences}/${stopPoint.replay.replayed} ` + + `${Math.round(stopPoint.buildWallMs / 1000)}s`, + ) + } catch (error) { + built[row.rowId] = { error: (error as Error).message } + log(`${row.rowId} STOP-POINT ERROR ${(error as Error).message}`) + } + }) + writeFileSync(join(WORK, 'stop-points.json'), JSON.stringify(built, null, 2)) + log(`stop points ready: ${Object.keys(built).length}`) + return + } + + const spendBefore = await routerSpend() + log( + `rows=${rows.length} rollouts=${ROLLOUTS} model=${MODEL} seed=${SEED} ` + + `policyDigest=pinned chain=${JSON.stringify(chain)} routerUsageBefore=${spendBefore.totalUsage}`, + ) + + const concurrency = Number(process.env.TBR_FL_CONCURRENCY ?? '4') + const startedMs = Date.now() + const outcomes: (RolloutOutcome | { rowId: string; rolloutIndex: number; error: string })[] = [] + const stopPoints: Record< + string, + { divergences: number; replayed: number; buildWallMs: number; reused: boolean } + > = {} + // A stop point outlives the pass that built it so later passes inherit the + // same one. `--drop-stop-points` is how a finished campaign reclaims them. + const dropStopPoints = process.argv.includes('--drop-stop-points') + await mapLimit(rows, concurrency, async (row) => { + const task = tasks.get(row.taskName)! + let stopPoint: StopPoint + try { + stopPoint = await buildStopPoint(row, task, systemInformation) + stopPoints[row.rowId] = { + divergences: stopPoint.replay.divergences, + replayed: stopPoint.replay.replayed, + buildWallMs: stopPoint.buildWallMs, + reused: stopPoint.reused, + } + log( + `${row.rowId} stop point ${stopPoint.reused ? 'reused' : 'built'} ` + + `div=${stopPoint.replay.divergences}/${stopPoint.replay.replayed} ` + + `${Math.round(stopPoint.buildWallMs / 1000)}s`, + ) + } catch (error) { + const message = (error as Error).message + log(`${row.rowId} STOP-POINT ERROR ${message}`) + for (let offset = 0; offset < ROLLOUTS; offset += 1) { + outcomes.push({ + rowId: row.rowId, + rolloutIndex: ROLLOUT_BASE + offset, + error: `stop point: ${message}`, + }) + } + return + } + try { + // The rollouts of a row share one stop-point image and touch nothing else + // in common, so they run together rather than one after another. + await Promise.all( + Array.from({ length: ROLLOUTS }, async (_unused, offset) => { + const i = ROLLOUT_BASE + offset + try { + outcomes.push(await runRollout(row, task, stopPoint, i, log)) + } catch (error) { + const message = (error as Error).message + log(`${row.rowId}#${i} ERROR ${message}`) + outcomes.push({ rowId: row.rowId, rolloutIndex: i, error: message }) + } + }), + ) + } finally { + if (dropStopPoints) await run('docker', ['rmi', '-f', stopPoint.image]).catch(() => undefined) + } + }) + const wallMs = Date.now() - startedMs + const spendAfter = await routerSpend() + + const report = { + generatedAt: new Date().toISOString(), + wallMs, + concurrency, + rollouts: ROLLOUTS, + policy: CONTINUATION_POLICY, + set: includeDivergent ? 'prefix-divergent' : 'primary', + denominatorChain: chain, + rows: rows.map((r) => ({ + rowId: r.rowId, + taskName: r.taskName, + recordedModel: r.recordedModel, + recordedCommands: r.recordedCommands, + })), + images: Object.fromEntries([...tasks].map(([n, t]) => [n, t.image])), + suiteDigests: Object.fromEntries([...tasks].map(([n, t]) => [n, t.suiteDigest])), + taskOracles: Object.fromEntries( + [...oracles] + .filter(([name]) => taskNames.includes(name)) + .map(([name, v]: [string, OracleDeterminismVerdict]) => [ + name, + { stable: v.stable, flipRate: v.flipRate, replicates: v.replicates }, + ]), + ), + routerSpend: { + before: spendBefore.totalUsage, + after: spendAfter.totalUsage, + deltaUsd: spendAfter.totalUsage - spendBefore.totalUsage, + }, + stopPoints, + outcomes, + } + const outPath = join(WORK, process.env.TBR_FL_OUT ?? 'freelunch.json') + writeFileSync(outPath, JSON.stringify(report, null, 2)) + log(`done wallMs=${wallMs} routerDelta=$${report.routerSpend.deltaUsd.toFixed(4)} -> ${outPath}`) +} + +main().catch((error) => { + process.stderr.write(`${(error as Error).stack}\n`) + process.exit(1) +}) diff --git a/src/trace-repair/continuation-policy.ts b/src/trace-repair/continuation-policy.ts index 9eb5d5f5..efdbf29d 100644 --- a/src/trace-repair/continuation-policy.ts +++ b/src/trace-repair/continuation-policy.ts @@ -214,6 +214,13 @@ export interface RunContinuationOptions { prefix: readonly ContinuationMessage[] /** Rollouts to run for this arm. */ rollouts: number + /** + * Global index of the first rollout this invocation produces. The per-rollout + * seed derives from the global index, so a campaign that adds rollouts in + * later passes must shift this base — an index that repeats is a rollout + * that repeats. Defaults to 0. + */ + rolloutBase?: number model: ContinuationModel environments: ContinuationEnvironmentFactory /** Epoch milliseconds. Injected so tests can assert on records without wall-clock noise. */ @@ -232,19 +239,25 @@ export async function runContinuation( ): Promise { const { policy, arm, rowId, prefix, rollouts, model, environments } = options const clock = options.clock ?? Date.now + const rolloutBase = options.rolloutBase ?? 0 requirePositiveInteger(rollouts, 'rollouts') + if (!Number.isInteger(rolloutBase) || rolloutBase < 0) { + throw new ValidationError( + `continuation policy rolloutBase must be a non-negative integer, got ${rolloutBase}`, + ) + } assertPrefix(prefix) const policyDigest = continuationPolicyDigest(policy) const records: ContinuationRollout[] = [] - for (let index = 0; index < rollouts; index += 1) { + for (let offset = 0; offset < rollouts; offset += 1) { records.push( await runOneRollout({ policy, policyDigest, arm, rowId, - index, + index: rolloutBase + offset, prefix, model, environments, diff --git a/tests/trace-repair/continuation-rollout-base.test.ts b/tests/trace-repair/continuation-rollout-base.test.ts new file mode 100644 index 00000000..e3e3ced7 --- /dev/null +++ b/tests/trace-repair/continuation-rollout-base.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest' +import { ValidationError } from '../../src/errors' +import { + type ContinuationEnvironment, + type ContinuationModelRequest, + continuationSeed, + definePinnedContinuationPolicy, + runContinuation, +} from '../../src/trace-repair/continuation-policy' + +const POLICY = definePinnedContinuationPolicy({ model: 'test-model', seed: 20260810 }) + +const PREFIX = [ + { role: 'system', content: 'system' }, + { role: 'user', content: 'task' }, +] as const + +function fakeEnvironments() { + const environment: ContinuationEnvironment = { + containerRef: 'fake', + describe: async () => ({ networkMode: 'none' as const }), + exec: async () => ({ output: '', returncode: 0, timedOut: false }), + dispose: async () => undefined, + } + return { id: 'fake', create: async () => environment } +} + +/** A model that fails on call one, so a rollout records exactly one seed. */ +function seedCapturingModel(seen: number[]) { + return async (request: ContinuationModelRequest): Promise => { + seen.push(request.seed) + throw new Error('stop after capture') + } +} + +describe('runContinuation rolloutBase', () => { + it('derives the seed and the record index from the global rollout index', async () => { + const seen: number[] = [] + const rollouts = await runContinuation({ + policy: POLICY, + arm: 'no-fix-control', + rowId: 'row-a', + prefix: [...PREFIX], + rollouts: 1, + rolloutBase: 5, + model: seedCapturingModel(seen), + environments: fakeEnvironments(), + }) + expect(rollouts).toHaveLength(1) + expect(rollouts[0]?.index).toBe(5) + expect(rollouts[0]?.seed).toBe(continuationSeed(POLICY.seed, 'row-a', 5)) + expect(seen).toEqual([continuationSeed(POLICY.seed, 'row-a', 5)]) + expect(rollouts[0]?.seed).not.toBe(continuationSeed(POLICY.seed, 'row-a', 0)) + }) + + it('defaults the base to 0 and advances it across rollouts in one call', async () => { + const seen: number[] = [] + const rollouts = await runContinuation({ + policy: POLICY, + arm: 'no-fix-control', + rowId: 'row-a', + prefix: [...PREFIX], + rollouts: 2, + model: seedCapturingModel(seen), + environments: fakeEnvironments(), + }) + expect(rollouts.map((r) => r.index)).toEqual([0, 1]) + expect(seen).toEqual([ + continuationSeed(POLICY.seed, 'row-a', 0), + continuationSeed(POLICY.seed, 'row-a', 1), + ]) + }) + + it('refuses a negative or fractional base', async () => { + for (const rolloutBase of [-1, 0.5]) { + await expect( + runContinuation({ + policy: POLICY, + arm: 'no-fix-control', + rowId: 'row-a', + prefix: [...PREFIX], + rollouts: 1, + rolloutBase, + model: seedCapturingModel([]), + environments: fakeEnvironments(), + }), + ).rejects.toThrow(ValidationError) + } + }) +}) diff --git a/tests/trace-repair/delta-repair.test.ts b/tests/trace-repair/delta-repair.test.ts index 1d8309c5..e45b6462 100644 --- a/tests/trace-repair/delta-repair.test.ts +++ b/tests/trace-repair/delta-repair.test.ts @@ -112,6 +112,22 @@ describe('threats travel with the number', () => { ) }) + it('says when the control that produced that zero could not have rescued anything', () => { + const inert = (rowId: string, rate: number) => ({ + ...measuredRowResult(rowId, rate), + controlScreening: 'declared-inert' as const, + }) + const report = deltaRepair([inert('a', 1), inert('b', 0)]) + const ids = report.threats.map((t) => t.id) + expect(ids).toContain('control-cannot-rescue') + // The stronger claim is not made alongside the weaker one: a control that + // makes no model call did not screen the rows it left at zero. + expect(ids).not.toContain('admission-conditions-on-control-failure') + expect(report.threats.find((t) => t.id === 'control-cannot-rescue')?.statement).toMatch( + /2\/2 rows/, + ) + }) + it('names a denominator carried by rows where nothing ran', () => { const report = deltaRepair([ measuredRowResult('a', 1), diff --git a/tsconfig.script.json b/tsconfig.script.json index 06b8218d..98c1c5e5 100644 --- a/tsconfig.script.json +++ b/tsconfig.script.json @@ -5,6 +5,8 @@ "scripts/tb-corpus-rows.ts", "scripts/tb-gated-stop-ab.ts", "scripts/tb-oracle-determinism.ts", + "scripts/tb-repair-freelunch-model.ts", + "scripts/tb-repair-freelunch.ts", "scripts/tb-repair-milestone1.ts", "src" ],