diff --git a/CHANGELOG.md b/CHANGELOG.md index ca76ae0..c311e6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,31 @@ The project follows [Semantic Versioning](https://semver.org/). ## [Unreleased] -## [0.3.0] - 2026-07-13 +## [0.3.1] - 2026-07-14 + +### Fixed + +- Miscounted unified-diff hunk headers (`@@ -a,b +c,d @@` where the line counts + disagree with the body) are now recomputed from the actual lines instead of + failing the local patch contract. This was the dominant cause of the + expensive full-batch `contract_retry` re-draft: models frequently get the + range counts wrong even when the change itself is correct, and each rejection + re-ran the entire agentic pass, wasting roughly 50–80% of a turn's tokens and + doubling its wall-clock time. +- Patch hunks whose context or removed lines drift on leading/trailing + whitespace or indentation are now located with a whitespace-insensitive match + and canonicalized back to the exact source text, rather than being rejected. +- Both fixes live in the shared `loopbiotic_patch` normalizer/applier, so they + apply to every backend (Codex, Claude, Ollama, generic, and stdio); the + whitespace tolerance is mirrored in the editor's Lua patch applier. Added + lines are never altered, and genuine content mismatches and ambiguous + relocations are still rejected. + +### Changed + +- The card window's token line now splits usage into input (with the cached + portion in parentheses) and output, and shows an estimated cost per turn and + per session using per-model pricing. ### Changed diff --git a/Cargo.lock b/Cargo.lock index 7317290..1ae4156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -342,6 +348,12 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -356,7 +368,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "loopbiotic_backends" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "async-trait", @@ -369,14 +381,14 @@ dependencies = [ [[package]] name = "loopbiotic_context" -version = "0.3.0" +version = "0.3.1" dependencies = [ "loopbiotic_protocol", ] [[package]] name = "loopbiotic_harness" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "async-trait", @@ -392,7 +404,7 @@ dependencies = [ [[package]] name = "loopbiotic_patch" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "loopbiotic_protocol", @@ -401,7 +413,7 @@ dependencies = [ [[package]] name = "loopbiotic_protocol" -version = "0.3.0" +version = "0.3.1" dependencies = [ "serde", "serde_json", @@ -409,14 +421,16 @@ dependencies = [ [[package]] name = "loopbioticd" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "loopbiotic_backends", "loopbiotic_harness", + "loopbiotic_patch", "loopbiotic_protocol", "serde", "serde_json", + "tempfile", "tokio", ] @@ -520,6 +534,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -656,6 +683,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "2.0.18" diff --git a/Cargo.toml b/Cargo.toml index c85aa35..6ead85b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ resolver = "3" [workspace.package] -version = "0.3.0" +version = "0.3.1" edition = "2024" authors = ["Dorian"] license = "MIT" diff --git a/lua/loopbiotic/apply.lua b/lua/loopbiotic/apply.lua index 0a71ddc..05925c3 100644 --- a/lua/loopbiotic/apply.lua +++ b/lua/loopbiotic/apply.lua @@ -1,5 +1,12 @@ local M = {} +-- Two lines match if they are equal ignoring leading/trailing whitespace. +-- Mirrors the daemon's tolerant patch matching so a hunk whose context drifted +-- on indentation still applies against the live buffer. +local function line_matches(a, b) + return vim.trim(a or "") == vim.trim(b or "") +end + function M.patch(file_patch) local buf = M.buffer(file_patch.file) @@ -43,12 +50,13 @@ function M.apply_diff(source, diff) for _, line in ipairs(hunk.lines) do if line.kind == "context" then - if output[offset + 1] ~= line.text then + if not line_matches(output[offset + 1], line.text) then error("Patch context changed while applying") end + -- Keep the buffer's real line, not the model's whitespace-drifted copy. offset = offset + 1 elseif line.kind == "remove" then - if output[offset + 1] ~= line.text then + if not line_matches(output[offset + 1], line.text) then error("Patch context changed while applying") end table.remove(output, offset + 1) @@ -132,7 +140,7 @@ end function M.matches_at(source, start, expected) for index, line in ipairs(expected) do - if source[start + index] ~= line then + if not line_matches(source[start + index], line) then return false end end diff --git a/lua/loopbiotic/card.lua b/lua/loopbiotic/card.lua index 5597584..c1b563f 100644 --- a/lua/loopbiotic/card.lua +++ b/lua/loopbiotic/card.lua @@ -281,54 +281,44 @@ function M.tokens(lines) end table.insert(lines, "") - local turn_raw = tonumber(usage.total_tokens) or 0 - local turn_cached = tonumber(usage.cached_input_tokens) or 0 - local turn_suffix = usage.estimated and " estimated" or "" - if turn_cached > 0 then - table.insert(lines, string.format( - "Turn %s raw · %s cached · %s non-cached%s", - turn_raw, - turn_cached, - math.max(turn_raw - turn_cached, 0), - turn_suffix - )) - else - table.insert(lines, string.format("Turn %s tokens%s", turn_raw, turn_suffix)) + + local pricing = require("loopbiotic.pricing") + local model = state.backend_model + + -- One usage row: input (with cached in parentheses) · output · cost. + local function usage_line(label, u, estimated) + local input = tonumber(u.input_tokens) or 0 + local cached = tonumber(u.cached_input_tokens) or 0 + local output = tonumber(u.output_tokens) or 0 + local text = string.format("%s in %s (%s cached) · out %s", label, input, cached, output) + local cost = pricing.format(pricing.cost(u, model)) + if cost then + text = text .. " · " .. cost + end + if estimated then + text = text .. " · est" + end + table.insert(lines, text) end + usage_line("Turn ", usage, usage.estimated) + local total = state.token_usage + if total and (tonumber(total.total_tokens) or 0) ~= (tonumber(usage.total_tokens) or 0) then + usage_line("Total", total, total.estimated) + end + if total then local budget = tonumber(config.values.backend.token_budget) or 0 - local used = total.total_tokens or 0 - local cached = tonumber(total.cached_input_tokens) or 0 + local used = tonumber(total.total_tokens) or 0 if budget > 0 then - if cached > 0 then - table.insert(lines, string.format( - "Total %s/%s raw · %s cached · %s non-cached", - used, - budget, - cached, - math.max(used - cached, 0) - )) - else - table.insert(lines, string.format("Total %s/%s tokens", used, budget)) - end + table.insert(lines, string.format("Budget %s/%s tokens", used, budget)) if used >= budget then table.insert(lines, "Warning Session token budget exceeded") end - elseif used ~= usage.total_tokens then - if cached > 0 then - table.insert(lines, string.format( - "Total %s raw · %s cached · %s non-cached", - used, - cached, - math.max(used - cached, 0) - )) - else - table.insert(lines, string.format("Total %s tokens", used)) - end end end + local report = state.context_report if report and report.enabled then table.insert(lines, string.format( diff --git a/lua/loopbiotic/pricing.lua b/lua/loopbiotic/pricing.lua new file mode 100644 index 0000000..1623b6d --- /dev/null +++ b/lua/loopbiotic/pricing.lua @@ -0,0 +1,76 @@ +-- Token pricing for the card's billing line. +-- +-- Rates are USD per 1,000,000 tokens. `cached_input` is the prompt-cache READ +-- rate (~10x cheaper than fresh input), which is why a tool-heavy turn that +-- reads tens of thousands of cached tokens still costs very little. Override or +-- extend via `pricing` in setup(), keyed by model id. +local M = {} + +M.rates = { + ["claude-opus-4-8"] = { input = 5.0, cached_input = 0.5, output = 25.0 }, + ["claude-opus-4-7"] = { input = 5.0, cached_input = 0.5, output = 25.0 }, + ["claude-opus-4-6"] = { input = 5.0, cached_input = 0.5, output = 25.0 }, + ["claude-sonnet-5"] = { input = 3.0, cached_input = 0.3, output = 15.0 }, + ["claude-sonnet-4-6"] = { input = 3.0, cached_input = 0.3, output = 15.0 }, + ["claude-haiku-4-5"] = { input = 1.0, cached_input = 0.1, output = 5.0 }, + ["haiku"] = { input = 1.0, cached_input = 0.1, output = 5.0 }, +} + +-- Resolve the rate for a model id: exact match, then user overrides, then a +-- substring match so labels like "codex/gpt-5.1" or dated suffixes still hit. +function M.rate_for(model) + if not model or model == "" then + return nil + end + + local ok, config = pcall(require, "loopbiotic.config") + local overrides = ok and config.values and config.values.pricing or nil + if overrides and overrides[model] then + return overrides[model] + end + if M.rates[model] then + return M.rates[model] + end + if overrides then + for id, rate in pairs(overrides) do + if model:find(id, 1, true) then + return rate + end + end + end + for id, rate in pairs(M.rates) do + if model:find(id, 1, true) then + return rate + end + end + + return nil +end + +-- Estimated USD cost for one usage record. `input_tokens` is the full input +-- (cached + fresh); the cached slice bills at the cheaper cache-read rate. +function M.cost(usage, model) + local rate = M.rate_for(model) + if not rate then + return nil + end + + local input = tonumber(usage.input_tokens) or 0 + local cached = tonumber(usage.cached_input_tokens) or 0 + local output = tonumber(usage.output_tokens) or 0 + local fresh = math.max(input - cached, 0) + + return fresh * rate.input / 1e6 + cached * rate.cached_input / 1e6 + output * rate.output / 1e6 +end + +function M.format(cost) + if not cost then + return nil + end + if cost < 0.01 then + return string.format("$%.4f", cost) + end + return string.format("$%.2f", cost) +end + +return M diff --git a/lua/loopbiotic/version.lua b/lua/loopbiotic/version.lua index eccb8da..62a6f0f 100644 --- a/lua/loopbiotic/version.lua +++ b/lua/loopbiotic/version.lua @@ -1,4 +1,4 @@ return { - plugin = "0.3.0", + plugin = "0.3.1", protocol = 8, } diff --git a/rust/crates/loopbiotic_backends/src/claude_app.rs b/rust/crates/loopbiotic_backends/src/claude_app.rs index d62e6e6..413fd4a 100644 --- a/rust/crates/loopbiotic_backends/src/claude_app.rs +++ b/rust/crates/loopbiotic_backends/src/claude_app.rs @@ -716,10 +716,33 @@ fn parse_stream_event(value: &Value) -> StreamEvent { fn parse_usage(value: Option<&Value>) -> Option { let usage = value?; - let input = usage.get("input_tokens")?.as_u64()? as usize; - let output = usage.get("output_tokens")?.as_u64()? as usize; - - Some(TokenUsage::reported(input, output)) + let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0) as usize; + + // Claude Code splits input across three counters: `input_tokens` is only the + // fresh, uncached slice of the final request, while the (usually dominant) + // rest of the context is billed through `cache_creation_input_tokens` and + // `cache_read_input_tokens`. Reading `input_tokens` alone under-reported a + // tool-heavy turn by an order of magnitude (e.g. 3k of a real 41k input). + // The `result` event's usage is cumulative across the whole tool loop, so + // summing the three input counters yields the real billed input; the cached + // subset is the cache-read portion. + let fresh_input = field("input_tokens"); + let cache_creation = field("cache_creation_input_tokens"); + let cache_read = field("cache_read_input_tokens"); + let output = field("output_tokens"); + + if fresh_input == 0 && cache_creation == 0 && cache_read == 0 && output == 0 { + return None; + } + + let input = fresh_input + cache_creation + cache_read; + Some(TokenUsage { + input_tokens: input, + cached_input_tokens: cache_read, + output_tokens: output, + total_tokens: input + output, + estimated: false, + }) } fn optional_env(name: &str) -> Option { @@ -780,6 +803,32 @@ mod tests { assert!(!usage.estimated); } + #[test] + fn usage_counts_cached_and_cache_creation_input() { + // A tool-heavy turn: the fresh `input_tokens` is a small slice of the + // real billed input, which lives in the two cache counters. + let value = json!({ + "type": "result", + "result": "{\"op\":\"finding\",\"title\":\"T\",\"finding\":\"F\"}", + "usage": { + "input_tokens": 3024, + "cache_creation_input_tokens": 20577, + "cache_read_input_tokens": 17371, + "output_tokens": 152 + } + }); + + let StreamEvent::Result { token_usage, .. } = parse_stream_event(&value) else { + panic!("expected result event"); + }; + let usage = token_usage.unwrap(); + assert_eq!(usage.input_tokens, 3024 + 20577 + 17371); + assert_eq!(usage.cached_input_tokens, 17371); + assert_eq!(usage.output_tokens, 152); + assert_eq!(usage.total_tokens, 3024 + 20577 + 17371 + 152); + assert!(!usage.estimated); + } + #[test] fn extracts_model_from_init_event() { let value = json!({ diff --git a/rust/crates/loopbiotic_harness/src/engine.rs b/rust/crates/loopbiotic_harness/src/engine.rs index 022c397..20e40ad 100644 --- a/rust/crates/loopbiotic_harness/src/engine.rs +++ b/rust/crates/loopbiotic_harness/src/engine.rs @@ -841,6 +841,8 @@ impl Engine { } validate_one_card(candidate)?; + // Correct miscounted hunk headers before the count check rejects them. + PatchNormalizer::normalize_hunk_headers(candidate)?; PatchValidator::validate_card_with_limits( candidate, MAX_GOAL_PATCH_FILES, diff --git a/rust/crates/loopbiotic_patch/src/apply.rs b/rust/crates/loopbiotic_patch/src/apply.rs index c34d203..b883792 100644 --- a/rust/crates/loopbiotic_patch/src/apply.rs +++ b/rust/crates/loopbiotic_patch/src/apply.rs @@ -21,8 +21,10 @@ impl PatchApply { for line in &hunk.lines { match line { DiffLine::Context(expected) => { - require_line(&source, index, expected)?; - output.push(expected.clone()); + // Keep the real source line, not the model's possibly + // whitespace-drifted copy of it. + let actual = require_line(&source, index, expected)?; + output.push(actual); index += 1; } DiffLine::Remove(expected) => { @@ -70,22 +72,25 @@ fn resolve_start(source: &[String], hunk: &crate::Hunk) -> Result { } fn matches_at(source: &[String], start: usize, expected: &[&str]) -> bool { - expected - .iter() - .enumerate() - .all(|(index, line)| source.get(start + index).map(String::as_str) == Some(*line)) + expected.iter().enumerate().all(|(index, line)| { + source + .get(start + index) + .is_some_and(|actual| crate::line_matches(actual, line)) + }) } -fn require_line(source: &[String], index: usize, expected: &str) -> Result<()> { +/// Verifies the source line matches (ignoring whitespace) and returns the +/// actual source text so the applied output preserves the file verbatim. +fn require_line(source: &[String], index: usize, expected: &str) -> Result { let Some(actual) = source.get(index) else { return Err(anyhow!("patch exceeds source")); }; - if actual != expected { + if !crate::line_matches(actual, expected) { return Err(anyhow!("patch context mismatch")); } - Ok(()) + Ok(actual.clone()) } #[cfg(test)] @@ -109,4 +114,17 @@ mod tests { assert_eq!(output, "prefix\nbefore\nnew\nafter\n"); } + + #[test] + fn tolerates_whitespace_drift_and_preserves_source_indentation() { + // The model reproduced the context with the wrong leading indentation + // and a trailing space, but the removed/added intent is unambiguous. + let diff = + UnifiedDiff::parse("@@ -1,3 +1,3 @@\n guard \n- old\n+ new\n tail\n").unwrap(); + let output = PatchApply::apply_to_text("\t\tguard\n\t\told\n\t\ttail\n", &diff).unwrap(); + + // Context lines keep the file's real tabs; only the added line is the + // model's text. + assert_eq!(output, "\t\tguard\n new\n\t\ttail\n"); + } } diff --git a/rust/crates/loopbiotic_patch/src/unified_diff.rs b/rust/crates/loopbiotic_patch/src/unified_diff.rs index 46a1b39..084e7a3 100644 --- a/rust/crates/loopbiotic_patch/src/unified_diff.rs +++ b/rust/crates/loopbiotic_patch/src/unified_diff.rs @@ -21,6 +21,18 @@ pub enum DiffLine { Add(String), } +/// True when a source line and a patch context/remove line are equal ignoring +/// leading and trailing whitespace. Models routinely drift on indentation or +/// trailing spaces when reproducing context, which used to fail the patch +/// contract and force an expensive full re-draft. We locate the hunk with this +/// tolerant comparison and then canonicalize the diff back to the exact source +/// text, so the applied result is byte-for-byte correct — only the *matching* +/// is fuzzy, never the output. Interior whitespace differences are still a real +/// content mismatch and do not match. +pub fn line_matches(source_line: &str, patch_line: &str) -> bool { + source_line.trim() == patch_line.trim() +} + impl UnifiedDiff { pub fn parse(diff: &str) -> Result { let mut hunks = Vec::new(); diff --git a/rust/crates/loopbiotic_patch/src/validate.rs b/rust/crates/loopbiotic_patch/src/validate.rs index 973aa1f..77872d2 100644 --- a/rust/crates/loopbiotic_patch/src/validate.rs +++ b/rust/crates/loopbiotic_patch/src/validate.rs @@ -67,47 +67,81 @@ impl PatchNormalizer { for patch in &mut card.patches { let mut diff = UnifiedDiff::parse(&patch.diff)?; for hunk in &mut diff.hunks { - let expected = hunk - .lines - .iter() - .filter_map(|line| match line { - DiffLine::Context(text) | DiffLine::Remove(text) => Some(text.as_str()), - DiffLine::Add(_) => None, - }) - .collect::>(); - if expected.is_empty() { - if hunk.old_len == 0 && source.is_empty() { - hunk.old_start = context.buffer_start_line; - hunk.new_start = context.buffer_start_line; - continue; - } - return Err(anyhow!( - "patch hunk without source context can only create an empty file" - )); - } - - let declared = hunk.old_start.checked_sub(context.buffer_start_line); - let start = if declared.is_some_and(|start| matches_at(&source, start, &expected)) { - declared.unwrap() - } else { - let matches = (0..=source.len().saturating_sub(expected.len())) - .filter(|start| matches_at(&source, *start, &expected)) + // Models routinely miscount the `@@ -a,b +c,d @@` ranges. The + // counts are pure functions of the line list, so correct them + // instead of failing the contract and forcing a re-draft. + recompute_hunk_lengths(hunk); + + // Locate the hunk in `start`, dropping the borrow of `hunk.lines` + // before we canonicalize them below. `None` means an empty-file + // create that needs no source anchoring. + let start = { + let expected = hunk + .lines + .iter() + .filter_map(|line| match line { + DiffLine::Context(text) | DiffLine::Remove(text) => Some(text.as_str()), + DiffLine::Add(_) => None, + }) .collect::>(); - match matches.as_slice() { - [start] => *start, - [] => { - return Err(anyhow!( - "patch context was not found in the supplied buffer" - )); - } - _ => { + if expected.is_empty() { + if hunk.old_len == 0 && source.is_empty() { + hunk.old_start = context.buffer_start_line; + hunk.new_start = context.buffer_start_line; + None + } else { return Err(anyhow!( - "patch context is ambiguous in the supplied buffer" + "patch hunk without source context can only create an empty file" )); } + } else { + let declared = hunk.old_start.checked_sub(context.buffer_start_line); + let located = if declared + .is_some_and(|start| matches_at(&source, start, &expected)) + { + declared.unwrap() + } else { + let matches = (0..=source.len().saturating_sub(expected.len())) + .filter(|start| matches_at(&source, *start, &expected)) + .collect::>(); + match matches.as_slice() { + [start] => *start, + [] => { + return Err(anyhow!( + "patch context was not found in the supplied buffer" + )); + } + _ => { + return Err(anyhow!( + "patch context is ambiguous in the supplied buffer" + )); + } + } + }; + Some(located) } }; + let Some(start) = start else { + continue; + }; + + // Rewrite context/remove lines to the exact source text so the + // diff applies byte-exact downstream even if the model drifted + // on whitespace. Added lines are never touched. + let mut src_pos = start; + for line in &mut hunk.lines { + match line { + DiffLine::Context(text) | DiffLine::Remove(text) => { + if let Some(actual) = source.get(src_pos) { + *text = (*actual).to_string(); + } + src_pos += 1; + } + DiffLine::Add(_) => {} + } + } + let corrected_old_start = context.buffer_start_line + start; let delta = hunk.new_start as isize - hunk.old_start as isize; let corrected_new_start = corrected_old_start @@ -125,6 +159,26 @@ impl PatchNormalizer { Ok(()) } + + /// Correct each hunk's `@@ -a,b +c,d @@` line counts from its actual lines, + /// without needing source context. Runs before validation on a raw card so + /// a miscounted header (a very common model error) is fixed rather than + /// rejected — the counts are fully derivable from the body. + pub fn normalize_hunk_headers(card: &mut Card) -> Result<()> { + let Card::Patch(card) = card else { + return Ok(()); + }; + + for patch in &mut card.patches { + let mut diff = UnifiedDiff::parse(&patch.diff)?; + for hunk in &mut diff.hunks { + recompute_hunk_lengths(hunk); + } + patch.diff = render_diff(&diff); + } + + Ok(()) + } } impl PatchValidator { @@ -296,16 +350,30 @@ fn validate_hunk_counts(hunk: &crate::Hunk, max_changed_lines: usize) -> Result< } fn matches_at(source: &[&str], start: usize, expected: &[&str]) -> bool { - expected - .iter() - .enumerate() - .all(|(offset, line)| source.get(start + offset).copied() == Some(*line)) + expected.iter().enumerate().all(|(offset, line)| { + source + .get(start + offset) + .is_some_and(|actual| crate::line_matches(actual, line)) + }) } fn render_diff(diff: &UnifiedDiff) -> String { diff.render() } +fn recompute_hunk_lengths(hunk: &mut crate::Hunk) { + hunk.old_len = hunk + .lines + .iter() + .filter(|line| matches!(line, DiffLine::Context(_) | DiffLine::Remove(_))) + .count(); + hunk.new_len = hunk + .lines + .iter() + .filter(|line| matches!(line, DiffLine::Context(_) | DiffLine::Add(_))) + .count(); +} + fn identifiers_for_lines( lines: &[DiffLine], include: impl Fn(&DiffLine) -> bool, @@ -627,6 +695,87 @@ mod tests { assert!(card.patches[0].diff.starts_with("@@ -50,2 +50,2 @@")); } + #[test] + fn recomputes_miscounted_hunk_headers_instead_of_rejecting() { + // Header claims 9,9 but the body is a single-line change. Raw validation + // rejects it; after normalizing headers it passes and the header is fixed. + let raw = FilePatch { + id: "p_1".into(), + file: PathBuf::from("src/work.ts"), + diff: "@@ -1,9 +1,9 @@\n context\n-old\n+new\n".into(), + explanation: "Fix.".into(), + }; + assert!( + PatchValidator::validate_file_patch(&raw) + .unwrap_err() + .to_string() + .contains("header counts") + ); + + let mut card = Card::Patch(loopbiotic_protocol::PatchCard { + id: "c_1".into(), + title: "Fix".into(), + explanation: "Fix.".into(), + warnings: vec![], + goal_complete: false, + patches: vec![raw], + actions: vec![loopbiotic_protocol::Action::Apply], + }); + + PatchNormalizer::normalize_hunk_headers(&mut card).unwrap(); + PatchValidator::validate_card(&card).unwrap(); + + let Card::Patch(card) = card else { + unreachable!() + }; + assert!(card.patches[0].diff.starts_with("@@ -1,2 +1,2 @@")); + } + + #[test] + fn normalizes_a_whitespace_drifted_hunk_to_exact_source() { + // Context/remove lines drift on indentation and trailing space; the + // normalizer must still locate the hunk and canonicalize it so the + // downstream exact-match validator passes. + let mut card = Card::Patch(loopbiotic_protocol::PatchCard { + id: "c_1".into(), + title: "Fix".into(), + explanation: "Fix the guard.".into(), + warnings: vec![], + goal_complete: false, + patches: vec![FilePatch { + id: "p_1".into(), + file: PathBuf::from("src/work.ts"), + diff: "@@ -1,2 +1,2 @@\n guard \n- old\n+ new\n".into(), + explanation: "Fix.".into(), + }], + actions: vec![loopbiotic_protocol::Action::Apply], + }); + let context = ContextBundle { + cwd: PathBuf::from("/tmp/project"), + file: PathBuf::from("src/work.ts"), + cursor: loopbiotic_protocol::Cursor { line: 1, column: 1 }, + selection: None, + buffer_text: "\tguard\n\told\n".into(), + buffer_start_line: 1, + diagnostics: vec![], + hints: vec![], + artifacts: vec![], + report: None, + }; + + PatchNormalizer::normalize_card(&mut card, &context).unwrap(); + // The exact-match validator would fail on the original drifted diff; + // it passes because normalization rewrote context/remove to the source. + PatchValidator::validate_card_against_context(&card, &context).unwrap(); + + let Card::Patch(card) = card else { + unreachable!() + }; + assert!(card.patches[0].diff.contains(" \tguard")); + assert!(card.patches[0].diff.contains("-\told")); + assert!(card.patches[0].diff.contains("+ new")); + } + #[test] fn refuses_to_relocate_ambiguous_hunk_context() { let mut card = Card::Patch(loopbiotic_protocol::PatchCard { diff --git a/rust/crates/loopbioticd/Cargo.toml b/rust/crates/loopbioticd/Cargo.toml index 395d955..66befb4 100644 --- a/rust/crates/loopbioticd/Cargo.toml +++ b/rust/crates/loopbioticd/Cargo.toml @@ -13,7 +13,9 @@ publish = false anyhow.workspace = true loopbiotic_backends = { path = "../loopbiotic_backends" } loopbiotic_harness = { path = "../loopbiotic_harness" } +loopbiotic_patch = { path = "../loopbiotic_patch" } loopbiotic_protocol = { path = "../loopbiotic_protocol" } serde.workspace = true serde_json.workspace = true +tempfile.workspace = true tokio.workspace = true diff --git a/rust/crates/loopbioticd/src/main.rs b/rust/crates/loopbioticd/src/main.rs index 49e4c4b..7501c50 100644 --- a/rust/crates/loopbioticd/src/main.rs +++ b/rust/crates/loopbioticd/src/main.rs @@ -17,6 +17,8 @@ use loopbiotic_protocol::{ use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; +mod token_report; + const OPEN_LOCATION_TIMEOUT: Duration = Duration::from_secs(120); const READ_FILE_TIMEOUT: Duration = Duration::from_secs(10); static NEXT_EDITOR_REQUEST_ID: AtomicU64 = AtomicU64::new(1); @@ -33,6 +35,9 @@ async fn main() -> Result<()> { [cmd, sub] if cmd == "schema" && sub == "card" => print_card_schema(), [cmd, sub] if cmd == "dev" && sub == "mock-session" => print_mock_session().await, [cmd, sub] if cmd == "dev" && sub == "stdio-agent" => run_stdio_agent(), + [cmd, sub, rest @ ..] if cmd == "dev" && sub == "token-report" => { + token_report::run(rest).await + } _ => print_help(), } } @@ -228,7 +233,7 @@ async fn request_editor_context( } } -fn backend_from_env() -> Result> { +pub(crate) fn backend_from_env() -> Result> { match std::env::var("LOOPBIOTIC_BACKEND").as_deref() { Ok("codex_app") | Ok("codex") => Ok(Arc::new(CodexAppBackend::from_env()?)), Ok("claude_app") | Ok("claude") => Ok(Arc::new(ClaudeAppBackend::from_env()?)), @@ -545,6 +550,9 @@ fn print_help() -> Result<()> { eprintln!("loopbioticd schema card"); eprintln!("loopbioticd dev mock-session"); eprintln!("loopbioticd dev stdio-agent"); + eprintln!("loopbioticd dev token-report [--fixtures DIR] [--json FILE] [--max-turns N]"); + eprintln!("loopbioticd dev token-report --render FILE"); + eprintln!("loopbioticd dev token-report --check BASELINE CURRENT"); Ok(()) } diff --git a/rust/crates/loopbioticd/src/token_report.rs b/rust/crates/loopbioticd/src/token_report.rs new file mode 100644 index 0000000..76c6c3b --- /dev/null +++ b/rust/crates/loopbioticd/src/token_report.rs @@ -0,0 +1,583 @@ +//! Live token-consumption harness. +//! +//! Drives the real `Engine` and a real backend (built from the environment, +//! exactly like the daemon) through a full pair-programming session for each +//! buggy TypeScript fixture, and reports how many goal steps, agent turns, and +//! tokens each configured model spent. +//! +//! This is a manual reporting tool, not a CI gate: it needs the real backend +//! CLIs/servers and credentials installed. Correctness is reported as +//! steps-taken + goal completion; there is no compiler or runtime gate, because +//! the fixtures deliberately mask type errors behind wrong `as` assertions that +//! a type checker would wave through. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{Context, Result, anyhow}; +use loopbiotic_harness::Engine; +use loopbiotic_patch::{PatchApply, UnifiedDiff}; +use loopbiotic_protocol::{ + Action, Card, ContextBundle, Cursor, Diagnostic, GoalStatus, Mode, PatchApplyResult, + StartSessionParams, +}; +use serde::{Deserialize, Serialize}; + +/// Hard ceiling on backend round-trips per case so a stuck or looping model +/// cannot run forever. A hard fixture targets 6 steps; the cap leaves headroom +/// for retries, discovery cards, and clarifying replies. +const DEFAULT_MAX_TURNS: usize = 30; + +/// The `case.json` next to every fixture's TypeScript files. +#[derive(Debug, Deserialize)] +struct CaseSpec { + name: String, + prompt: String, + entry: PathBuf, + #[serde(default)] + cursor: Option, + #[serde(default)] + mode: Option, + #[serde(default)] + target_steps: usize, + #[serde(default)] + diagnostics: Vec, +} + +/// One (model x case) result row, also the JSONL record schema. +#[derive(Clone, Debug, Serialize, Deserialize)] +struct CaseReport { + model: String, + case: String, + target_steps: usize, + steps: usize, + turns: usize, + completed: bool, + input: usize, + cached: usize, + output: usize, + total: usize, + estimated: bool, + attempts: usize, + elapsed_ms: u128, + note: String, +} + +/// Entry point for `loopbioticd dev token-report [flags]`. +pub async fn run(args: &[String]) -> Result<()> { + let mut fixtures = PathBuf::from("tests/fixtures/token"); + let mut json_path: Option = std::env::var_os("LOOPBIOTIC_REPORT_JSON").map(Into::into); + let mut max_turns = DEFAULT_MAX_TURNS; + + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--render" => { + let path = args + .get(i + 1) + .ok_or_else(|| anyhow!("--render needs a file"))?; + return render(Path::new(path)); + } + "--check" => { + let base = args + .get(i + 1) + .ok_or_else(|| anyhow!("--check needs baseline"))?; + let cur = args + .get(i + 2) + .ok_or_else(|| anyhow!("--check needs current file"))?; + return check(Path::new(base), Path::new(cur)); + } + "--fixtures" => { + fixtures = args + .get(i + 1) + .ok_or_else(|| anyhow!("--fixtures needs a path"))? + .into(); + i += 1; + } + "--json" => { + json_path = Some( + args.get(i + 1) + .ok_or_else(|| anyhow!("--json needs a path"))? + .into(), + ); + i += 1; + } + "--max-turns" => { + max_turns = args + .get(i + 1) + .ok_or_else(|| anyhow!("--max-turns needs a number"))? + .parse()?; + i += 1; + } + other => return Err(anyhow!("unknown token-report flag {other}")), + } + i += 1; + } + + let model = model_label(); + let backend = crate::backend_from_env()?; + + let mut reports = Vec::new(); + for case_dir in fixture_dirs(&fixtures)? { + let report = match run_case(&case_dir, &model, backend.clone(), max_turns).await { + Ok(report) => report, + Err(error) => { + // A whole-case failure (unreadable fixture, backend refusing the + // first turn) is still a row, not a crash of the whole run. + let name = case_dir + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + CaseReport { + model: model.clone(), + case: name, + note: format!("error: {error}"), + ..empty_report(&model) + } + } + }; + reports.push(report); + } + + if let Some(path) = &json_path { + append_jsonl(path, &reports)?; + } + print_table(&reports); + Ok(()) +} + +/// Runs a single fixture end-to-end against `backend` and returns its row. +async fn run_case( + case_dir: &Path, + model: &str, + backend: Arc, + max_turns: usize, +) -> Result { + let spec_raw = fs::read_to_string(case_dir.join("case.json")) + .with_context(|| format!("reading {}/case.json", case_dir.display()))?; + let spec: CaseSpec = serde_json::from_str(&spec_raw) + .with_context(|| format!("parsing {}/case.json", case_dir.display()))?; + + // Work on a throwaway copy so a run never mutates the committed fixtures, + // and so the agent cannot peek at case.json (which carries the difficulty + // target and the intended prompt). + let workdir = tempfile::Builder::new() + .prefix(&format!("loopbiotic-token-{}-", spec.name)) + .tempdir()?; + copy_fixture(case_dir, workdir.path())?; + let cwd = workdir.path().to_path_buf(); + + let entry_text = fs::read_to_string(cwd.join(&spec.entry)) + .with_context(|| format!("reading entry file {}", spec.entry.display()))?; + + let mut engine = Engine::new(backend); + engine.set_source_context_provider(fs_context_provider(cwd.clone())); + engine.set_location_granter(fs_location_granter(cwd.clone())); + + let params = StartSessionParams { + cwd: cwd.clone(), + file: spec.entry.clone(), + cursor: spec.cursor.clone().unwrap_or(Cursor { line: 1, column: 1 }), + selection: None, + prompt: spec.prompt.clone(), + mode: spec.mode.clone().unwrap_or(Mode::Auto), + buffer_text: entry_text, + buffer_start_line: 1, + diagnostics: spec.diagnostics.clone(), + hints: vec![], + context_policy: Default::default(), + }; + + let started = Instant::now(); + let start = engine.start(params).await?; + let session_id = start.session_id.clone(); + + let mut card = start.card; + let mut goal = start.goal; + let mut total_usage = start.token_usage; + let mut attempts = start.attempts.len(); + let mut turns = 1usize; + let mut replies_used = 0usize; + let mut note = String::new(); + + while turns < max_turns { + match &card { + Card::Summary(_) => break, + Card::Error(err) => { + note = format!("error card: {}", first_line(&err.message)); + break; + } + Card::Patch(patch) => { + let card_id = patch.id.clone(); + let (patch_ids, changed, context) = match apply_patch(&cwd, patch, &spec.entry) { + Ok(applied) => applied, + Err(error) => { + note = format!("patch apply failed: {error}"); + break; + } + }; + let result = engine + .apply_result(PatchApplyResult { + session_id: session_id.clone(), + card_id, + accepted: true, + patch_ids, + changed_files: changed, + error: None, + context, + }) + .await?; + turns += 1; + attempts += result.attempts.len(); + total_usage = result.token_usage; + goal = result.goal; + card = result.card; + } + Card::Choice(_) => { + if replies_used >= 2 { + note = "stopped: repeated clarifying questions".into(); + break; + } + replies_used += 1; + let result = engine + .reply( + &session_id, + "Use your best judgment and make the fix.".into(), + ) + .await?; + turns += 1; + attempts += result.attempts.len(); + total_usage = result.token_usage; + goal = result.goal; + card = result.card; + } + other => { + let Some(action) = drive_action(other.actions()) else { + note = format!("stopped: no drivable action on {:?} card", other.kind()); + break; + }; + let result = engine.action(&session_id, action).await?; + turns += 1; + attempts += result.attempts.len(); + total_usage = result.token_usage; + goal = result.goal; + card = result.card; + } + } + } + + if turns >= max_turns && note.is_empty() { + note = format!("hit turn cap ({max_turns})"); + } + + let completed = goal.status == GoalStatus::Complete; + Ok(CaseReport { + model: model.to_string(), + case: spec.name, + target_steps: spec.target_steps, + steps: goal.completed_steps.len(), + turns, + completed, + input: total_usage.input_tokens, + cached: total_usage.cached_input_tokens, + output: total_usage.output_tokens, + total: total_usage.total_tokens, + estimated: total_usage.estimated, + attempts, + elapsed_ms: started.elapsed().as_millis(), + note, + }) +} + +/// Applies every file patch in a card to the working copy and returns the ids, +/// changed files, and the updated context the editor would send back. +fn apply_patch( + cwd: &Path, + patch: &loopbiotic_protocol::PatchCard, + entry: &Path, +) -> Result<(Vec, Vec, ContextBundle)> { + let mut patch_ids = Vec::new(); + let mut changed = Vec::new(); + let mut primary: Option = None; + + for file_patch in &patch.patches { + let abs = cwd.join(&file_patch.file); + let current = fs::read_to_string(&abs).unwrap_or_default(); + let diff = UnifiedDiff::parse(&file_patch.diff) + .with_context(|| format!("parsing diff for {}", file_patch.file.display()))?; + let updated = PatchApply::apply_to_text(¤t, &diff) + .with_context(|| format!("applying diff to {}", file_patch.file.display()))?; + fs::write(&abs, &updated)?; + patch_ids.push(file_patch.id.clone()); + changed.push(file_patch.file.clone()); + primary.get_or_insert_with(|| file_patch.file.clone()); + } + + let focus = primary.unwrap_or_else(|| entry.to_path_buf()); + let context = file_context(cwd, &focus) + .ok_or_else(|| anyhow!("could not read patched file {}", focus.display()))?; + Ok((patch_ids, changed, context)) +} + +/// Picks the action that best advances a goal from a discovery/deny card. +fn drive_action(actions: &[Action]) -> Option { + const PRIORITY: &[Action] = &[ + Action::Fix, + Action::Next, + Action::Follow, + Action::Open, + Action::ResumeDraft, + Action::RunCheck, + ]; + for wanted in PRIORITY { + if actions.iter().any(|a| a == wanted) { + return Some(wanted.clone()); + } + } + None +} + +/// Builds a context bundle for a file inside the working copy, mirroring what +/// the editor sends when it snapshots a buffer. +fn file_context(cwd: &Path, file: &Path) -> Option { + let abs = if file.is_absolute() { + file.to_path_buf() + } else { + cwd.join(file) + }; + let text = fs::read_to_string(&abs).ok()?; + let relative = abs + .strip_prefix(cwd) + .map(Path::to_path_buf) + .unwrap_or_else(|_| file.to_path_buf()); + Some(ContextBundle { + cwd: cwd.to_path_buf(), + file: relative, + cursor: Cursor { line: 1, column: 1 }, + selection: None, + buffer_text: text, + buffer_start_line: 1, + diagnostics: vec![], + hints: vec![], + artifacts: vec![], + report: None, + }) +} + +fn fs_context_provider(cwd: PathBuf) -> loopbiotic_harness::SourceContextProvider { + Arc::new(move |file, _session_id| { + let cwd = cwd.clone(); + Box::pin(async move { file_context(&cwd, &file) }) + }) +} + +fn fs_location_granter(cwd: PathBuf) -> loopbiotic_harness::LocationGranter { + Arc::new(move |card, _session_id| { + let cwd = cwd.clone(); + Box::pin(async move { file_context(&cwd, &card.location.file) }) + }) +} + +/// Copies every fixture file except `case.json` into the working directory, +/// preserving subdirectories. +fn copy_fixture(src: &Path, dst: &Path) -> Result<()> { + for entry in fs::read_dir(src)? { + let entry = entry?; + let path = entry.path(); + let name = entry.file_name(); + if name == "case.json" { + continue; + } + let target = dst.join(&name); + if path.is_dir() { + fs::create_dir_all(&target)?; + copy_fixture(&path, &target)?; + } else { + fs::copy(&path, &target)?; + } + } + Ok(()) +} + +/// Every immediate subdirectory of `fixtures` that has a `case.json`, sorted. +fn fixture_dirs(fixtures: &Path) -> Result> { + let mut dirs = Vec::new(); + for entry in fs::read_dir(fixtures) + .with_context(|| format!("reading fixtures dir {}", fixtures.display()))? + { + let path = entry?.path(); + if path.is_dir() && path.join("case.json").exists() { + dirs.push(path); + } + } + dirs.sort(); + if dirs.is_empty() { + return Err(anyhow!( + "no case.json fixtures under {}", + fixtures.display() + )); + } + Ok(dirs) +} + +/// A display label for the model under test: `LOOPBIOTIC_REPORT_MODEL` if the +/// caller set one, else the per-backend model env var, else the backend name. +fn model_label() -> String { + if let Ok(label) = std::env::var("LOOPBIOTIC_REPORT_MODEL") { + return label; + } + let backend = std::env::var("LOOPBIOTIC_BACKEND").unwrap_or_else(|_| "mock".into()); + let model = match backend.as_str() { + "codex" | "codex_app" => std::env::var("LOOPBIOTIC_CODEX_MODEL").ok(), + "claude" | "claude_app" => std::env::var("LOOPBIOTIC_CLAUDE_MODEL").ok(), + "ollama" => std::env::var("LOOPBIOTIC_OLLAMA_MODEL").ok(), + _ => None, + }; + match model { + Some(model) => format!("{backend}/{model}"), + None => backend, + } +} + +fn empty_report(model: &str) -> CaseReport { + CaseReport { + model: model.to_string(), + case: String::new(), + target_steps: 0, + steps: 0, + turns: 0, + completed: false, + input: 0, + cached: 0, + output: 0, + total: 0, + estimated: false, + attempts: 0, + elapsed_ms: 0, + note: String::new(), + } +} + +fn append_jsonl(path: &Path, reports: &[CaseReport]) -> Result<()> { + use std::io::Write; + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent)?; + } + } + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + for report in reports { + writeln!(file, "{}", serde_json::to_string(report)?)?; + } + Ok(()) +} + +/// `--render `: print a unified table across every model in the file. +fn render(path: &Path) -> Result<()> { + let reports = read_jsonl(path)?; + print_table(&reports); + Ok(()) +} + +fn read_jsonl(path: &Path) -> Result> { + let raw = fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + let mut reports = Vec::new(); + for line in raw.lines() { + if line.trim().is_empty() { + continue; + } + reports.push(serde_json::from_str(line)?); + } + Ok(reports) +} + +fn print_table(reports: &[CaseReport]) { + println!( + "{:<22} {:<16} {:>7} {:>5} {:>8} {:>7} {:>7} {:>8} {:>4} {:>3} NOTE", + "MODEL", "CASE", "STEPS", "TURNS", "IN", "OUT", "CACHED", "TOTAL", "ATT", "OK" + ); + for r in reports { + let steps = format!("{}/{}", r.steps, r.target_steps); + let ok = if r.completed { "✓" } else { "·" }; + let total = if r.estimated { + format!("~{}", r.total) + } else { + r.total.to_string() + }; + println!( + "{:<22} {:<16} {:>7} {:>5} {:>8} {:>7} {:>7} {:>8} {:>4} {:>3} {}", + truncate(&r.model, 22), + truncate(&r.case, 16), + steps, + r.turns, + r.input, + r.output, + r.cached, + total, + r.attempts, + ok, + r.note, + ); + } +} + +/// `--check `: flag completion regressions and token drift +/// beyond 15%. Exits non-zero if any regression is found. +fn check(baseline: &Path, current: &Path) -> Result<()> { + const DRIFT: f64 = 0.15; + let base = read_jsonl(baseline)?; + let cur = read_jsonl(current)?; + + let mut regressions = 0usize; + for c in &cur { + let Some(b) = base.iter().find(|b| b.model == c.model && b.case == c.case) else { + println!("NEW {} {} (no baseline)", c.model, c.case); + continue; + }; + if b.completed && !c.completed { + println!("REGRESS {} {}: completion lost", c.model, c.case); + regressions += 1; + } + if b.total > 0 { + let delta = (c.total as f64 - b.total as f64) / b.total as f64; + if delta > DRIFT { + println!( + "REGRESS {} {}: tokens {} -> {} (+{:.0}%)", + c.model, + c.case, + b.total, + c.total, + delta * 100.0 + ); + regressions += 1; + } + } + } + + if regressions == 0 { + println!("ok: no completion or token regressions vs baseline"); + Ok(()) + } else { + std::process::exit(1); + } +} + +fn truncate(text: &str, width: usize) -> String { + if text.chars().count() <= width { + text.to_string() + } else { + text.chars() + .take(width.saturating_sub(1)) + .collect::() + + "…" + } +} + +fn first_line(text: &str) -> &str { + text.lines().next().unwrap_or(text) +} diff --git a/scripts/headless-smoke.lua b/scripts/headless-smoke.lua index f4e801b..dae281a 100644 --- a/scripts/headless-smoke.lua +++ b/scripts/headless-smoke.lua @@ -52,15 +52,29 @@ local new_lines = apply.apply_diff({ "" }, "@@ -1,0 +1,2 @@\n+&2; exit 2 ;; + esac +done + +if [[ "$INCLUDE_MOCK" == "1" ]]; then + MODELS+=("LOOPBIOTIC_REPORT_MODEL=mock;LOOPBIOTIC_BACKEND=mock") +fi + +# Locate (or build) the daemon binary. +BIN="${LOOPBIOTIC_LOOPBIOTICD:-}" +if [[ -z "$BIN" ]]; then + echo "building loopbioticd (release)..." >&2 + cargo build --release -p loopbioticd >&2 + BIN="$ROOT/target/release/loopbioticd" +fi + +OUT="$(mktemp)" +: > "$OUT" + +RUN_ARGS=(dev token-report --fixtures "$FIXTURES" --json "$OUT") +[[ -n "$MAX_TURNS" ]] && RUN_ARGS+=(--max-turns "$MAX_TURNS") + +for entry in "${MODELS[@]}"; do + label="${entry#LOOPBIOTIC_REPORT_MODEL=}"; label="${label%%;*}" + echo "running $label ..." >&2 + # Each model runs in a subshell with only its env applied. Per-model stdout + # (its own table) is discarded; the JSONL is what we aggregate. A failure to + # construct the backend skips the model instead of aborting the report. + if ! ( + IFS=';' + for kv in $entry; do export "${kv?}"; done + "$BIN" "${RUN_ARGS[@]}" + ) >/dev/null; then + echo " ! skipped $label (backend unavailable or misconfigured)" >&2 + fi +done + +echo +"$BIN" dev token-report --render "$OUT" + +if [[ -n "$CHECK_BASELINE" ]]; then + echo + "$BIN" dev token-report --check "$CHECK_BASELINE" "$OUT" +fi + +if [[ -n "$KEEP_OUT" ]]; then + cp "$OUT" "$KEEP_OUT" + echo "wrote $KEEP_OUT" >&2 +fi +rm -f "$OUT" diff --git a/tests/fixtures/token/easy/case.json b/tests/fixtures/token/easy/case.json new file mode 100644 index 0000000..4f3d34b --- /dev/null +++ b/tests/fixtures/token/easy/case.json @@ -0,0 +1,17 @@ +{ + "name": "cart-checkout", + "prompt": "Checkout totals are consistently wrong. The very first item in a cart never seems to be counted in the total, and any order placed without a discount comes back as NaN instead of a dollar amount. The project also fails to compile.", + "entry": "pricing.ts", + "cursor": { "line": 16, "column": 1 }, + "mode": "auto", + "target_steps": 1, + "diagnostics": [ + { + "file": "pricing.ts", + "line": 20, + "column": 39, + "severity": "error", + "message": "Property 'quantiy' does not exist on type 'OrderLine'. Did you mean 'quantity'?" + } + ] +} diff --git a/tests/fixtures/token/easy/pricing.ts b/tests/fixtures/token/easy/pricing.ts new file mode 100644 index 0000000..6d60ea7 --- /dev/null +++ b/tests/fixtures/token/easy/pricing.ts @@ -0,0 +1,25 @@ +interface OrderLine { + sku: string; + quantity: number; + unitPrice: number; +} + +interface CheckoutConfig { + /** Discount as a fraction, e.g. 0.1 for 10% off. Omitted means no discount. */ + discount?: number; +} + +/** + * Sums each line (unitPrice * quantity) and applies the optional discount, + * returning the final amount owed. + */ +export function checkoutTotal(lines: OrderLine[], config: CheckoutConfig): number { + let subtotal = 0; + for (let i = 1; i < lines.length; i++) { + const line = lines[i]; + subtotal += line.unitPrice * line.quantiy; + } + + const rate = config.discount as number; + return subtotal * (1 - rate); +} diff --git a/tests/fixtures/token/hard/case.json b/tests/fixtures/token/hard/case.json new file mode 100644 index 0000000..541cfb3 --- /dev/null +++ b/tests/fixtures/token/hard/case.json @@ -0,0 +1,17 @@ +{ + "name": "room-scheduler", + "prompt": "The meeting-room scheduler misbehaves in several ways. Rescheduling an existing booking leaves its end time blank, so the slot no longer has a valid finish, and rescheduling an id that was never booked throws an error instead of doing nothing. Meetings that sit back-to-back (one ending exactly when the next begins) are rejected as if they conflict. Generating the room-usage report throws once every room has been tallied. The project also no longer compiles.", + "entry": "reducer.ts", + "cursor": { "line": 8, "column": 1 }, + "mode": "auto", + "target_steps": 6, + "diagnostics": [ + { + "file": "main.ts", + "line": 8, + "column": 3, + "severity": "error", + "message": "Type '{ kind: \"reschedule\"; id: string; start: number; end: number; }' is not assignable to type 'SchedulerEvent'." + } + ] +} diff --git a/tests/fixtures/token/hard/main.ts b/tests/fixtures/token/hard/main.ts new file mode 100644 index 0000000..bc6cc05 --- /dev/null +++ b/tests/fixtures/token/hard/main.ts @@ -0,0 +1,17 @@ +import { SchedulerEvent } from "./types"; +import { applyAll, emptyState } from "./store"; +import { roomUsage, busiestRoom } from "./report"; + +const events: SchedulerEvent[] = [ + { kind: "book", id: "a", room: "R1", start: 540, end: 600 }, + { kind: "book", id: "b", room: "R1", start: 600, end: 660 }, + { kind: "reschedule", id: "a", start: 555, end: 615 }, + { kind: "cancel", id: "b" }, +]; + +export function run(): void { + const state = applyAll(emptyState(), events); + const usage = roomUsage(state); + const winner = busiestRoom(usage); + console.log(winner, usage); +} diff --git a/tests/fixtures/token/hard/reducer.ts b/tests/fixtures/token/hard/reducer.ts new file mode 100644 index 0000000..e08a04e --- /dev/null +++ b/tests/fixtures/token/hard/reducer.ts @@ -0,0 +1,42 @@ +import { SchedulerEvent, SchedulerState, Booking } from "./types"; +import { findConflict } from "./schedule"; + +export function reduce( + state: SchedulerState, + event: SchedulerEvent +): SchedulerState { + switch (event.kind) { + case "book": { + const candidate: Booking = { + id: event.id, + room: event.room, + start: event.start, + end: event.end, + }; + if (findConflict(state.bookings, candidate)) { + return state; + } + return { bookings: [...state.bookings, candidate] }; + } + case "cancel": { + return { + bookings: state.bookings.filter((b) => b.id !== event.id), + }; + } + default: { + const evt = event as any; + const index = state.bookings.findIndex((b) => b.id === evt.id); + const current = state.bookings[index]!; + const updated: Booking = { + ...current, + start: evt.start, + end: evt.ned, + }; + const others = state.bookings.filter((b) => b.id !== evt.id); + if (findConflict(others, updated)) { + return state; + } + return { bookings: [...others, updated] }; + } + } +} diff --git a/tests/fixtures/token/hard/report.ts b/tests/fixtures/token/hard/report.ts new file mode 100644 index 0000000..fc57a30 --- /dev/null +++ b/tests/fixtures/token/hard/report.ts @@ -0,0 +1,40 @@ +import { SchedulerState, Booking } from "./types"; +import { totalMinutes } from "./schedule"; + +export interface RoomUsage { + room: string; + bookings: number; + minutes: number; +} + +// Groups the current bookings by room and reports usage per room. +export function roomUsage(state: SchedulerState): RoomUsage[] { + const byRoom = new Map(); + for (const booking of state.bookings) { + const list = byRoom.get(booking.room) ?? []; + list.push(booking); + byRoom.set(booking.room, list); + } + + const usage: RoomUsage[] = []; + for (const [room, list] of byRoom) { + usage.push({ + room, + bookings: list.length, + minutes: totalMinutes(list), + }); + } + return usage; +} + +// The busiest room is the one with the most total booked minutes. +export function busiestRoom(usages: RoomUsage[]): string | undefined { + if (usages.length === 0) return undefined; + let busiest = usages[0]; + for (let i = 1; i <= usages.length; i++) { + if (usages[i].minutes > busiest.minutes) { + busiest = usages[i]; + } + } + return busiest.room; +} diff --git a/tests/fixtures/token/hard/schedule.ts b/tests/fixtures/token/hard/schedule.ts new file mode 100644 index 0000000..ad96fd7 --- /dev/null +++ b/tests/fixtures/token/hard/schedule.ts @@ -0,0 +1,25 @@ +import { Booking } from "./types"; + +// Same-room bookings overlap only when each starts strictly before the other ends. +export function overlaps(a: Booking, b: Booking): boolean { + if (a.room !== b.room) return false; + return a.start < b.end && b.start <= a.end; +} + +// Total number of booked minutes across the given bookings. +export function totalMinutes(bookings: Booking[]): number { + return bookings.reduce((sum, bk) => sum + (bk.end - bk.start), 0); +} + +// Returns the first existing booking that conflicts with the candidate. +export function findConflict( + bookings: Booking[], + candidate: Booking +): Booking | undefined { + for (let i = 0; i < bookings.length; i++) { + const existing = bookings[i]; + if (existing.id === candidate.id) continue; + if (overlaps(existing, candidate)) return existing; + } + return undefined; +} diff --git a/tests/fixtures/token/hard/store.ts b/tests/fixtures/token/hard/store.ts new file mode 100644 index 0000000..05db25f --- /dev/null +++ b/tests/fixtures/token/hard/store.ts @@ -0,0 +1,17 @@ +import { SchedulerEvent, SchedulerState } from "./types"; +import { reduce } from "./reducer"; + +export function applyAll( + initial: SchedulerState, + events: SchedulerEvent[] +): SchedulerState { + let state = initial; + for (let i = 0; i < events.length; i++) { + state = reduce(state, events[i]); + } + return state; +} + +export function emptyState(): SchedulerState { + return { bookings: [] }; +} diff --git a/tests/fixtures/token/hard/types.ts b/tests/fixtures/token/hard/types.ts new file mode 100644 index 0000000..15b42d0 --- /dev/null +++ b/tests/fixtures/token/hard/types.ts @@ -0,0 +1,32 @@ +export interface Booking { + id: string; + room: string; + start: number; + end: number; +} + +export interface BookEvent { + kind: "book"; + id: string; + room: string; + start: number; + end: number; +} + +export interface CancelEvent { + kind: "cancel"; + id: string; +} + +export interface RescheduleEvent { + kind: "reschedule"; + id: string; + start: number; + end: number; +} + +export type SchedulerEvent = BookEvent | CancelEvent; + +export interface SchedulerState { + bookings: Booking[]; +} diff --git a/tests/fixtures/token/medium/case.json b/tests/fixtures/token/medium/case.json new file mode 100644 index 0000000..ab0b48e --- /dev/null +++ b/tests/fixtures/token/medium/case.json @@ -0,0 +1,17 @@ +{ + "name": "fulfillment-summary", + "prompt": "The fulfillment summary is unreliable and the code currently fails to compile. Once it builds, skus that are genuinely out of stock are never flagged, while skus that are fully in stock get reported as short. On top of that, generating a summary crashes outright whenever an order references a sku that is not in the product catalog.", + "entry": "summary.ts", + "cursor": { "line": 13, "column": 1 }, + "mode": "auto", + "target_steps": 3, + "diagnostics": [ + { + "file": "inventory.ts", + "line": 24, + "column": 5, + "severity": "error", + "message": "Object literal may only specify known properties, but 'shortfall' does not exist in type 'StockReport'. Did you mean to write 'shorfall'?" + } + ] +} diff --git a/tests/fixtures/token/medium/inventory.ts b/tests/fixtures/token/medium/inventory.ts new file mode 100644 index 0000000..47a0e56 --- /dev/null +++ b/tests/fixtures/token/medium/inventory.ts @@ -0,0 +1,26 @@ +import { Product, StockReport } from "./types"; + +export function indexBySku(products: Product[]): Map { + const index = new Map(); + for (const product of products) { + index.set(product.sku, product); + } + return index; +} + +/** + * Builds a stock report for one sku: how many units short we are, given the + * requested quantity and the quantity currently available. + */ +export function buildReport( + sku: string, + requested: number, + available: number +): StockReport { + return { + sku, + requested, + available, + shortfall: available - requested, + }; +} diff --git a/tests/fixtures/token/medium/summary.ts b/tests/fixtures/token/medium/summary.ts new file mode 100644 index 0000000..239c379 --- /dev/null +++ b/tests/fixtures/token/medium/summary.ts @@ -0,0 +1,39 @@ +import { Order, Product, StockReport } from "./types"; +import { indexBySku, buildReport } from "./inventory"; + +export interface FulfillmentSummary { + totalItems: number; + shortfalls: StockReport[]; +} + +/** + * Aggregates every order line, then reports each sku that cannot be fully + * fulfilled from current stock. + */ +export function summarize( + orders: Order[], + products: Product[] +): FulfillmentSummary { + const index = indexBySku(products); + + const requestedBySku = new Map(); + for (const order of orders) { + for (const item of order.items) { + const prev = requestedBySku.get(item.sku) ?? 0; + requestedBySku.set(item.sku, prev + item.quantity); + } + } + + const shortfalls: StockReport[] = []; + let totalItems = 0; + for (const [sku, requested] of requestedBySku) { + totalItems += requested; + const product = index.get(sku)!; + const report = buildReport(sku, requested, product.stock); + if (report.shortfall > 0) { + shortfalls.push(report); + } + } + + return { totalItems, shortfalls }; +} diff --git a/tests/fixtures/token/medium/types.ts b/tests/fixtures/token/medium/types.ts new file mode 100644 index 0000000..1d06f17 --- /dev/null +++ b/tests/fixtures/token/medium/types.ts @@ -0,0 +1,22 @@ +export interface Product { + sku: string; + name: string; + stock: number; +} + +export interface OrderItem { + sku: string; + quantity: number; +} + +export interface Order { + id: string; + items: OrderItem[]; +} + +export interface StockReport { + sku: string; + requested: number; + available: number; + shorfall: number; +}