feat: implement act apply-model and report baseline tripwire (#607)

src/act/cli.ts

@@ -65,6 +65,44 @@ export function registerActCommands(program: Command): void {
       }
     })
 
+  act
+    .command('apply-model <project>')
+    .description('Apply the model default recommendation for a project')
+    .action(async (project: string) => {
+      try {
+        const { parseAllSessions, filterProjectsByName } = await import('../parser.js')
+        const { recommendModelDefault, buildApplyModelDefaultPlan } = await import('./model-defaults.js')
+        const { runAction } = await import('./apply.js')
+        const chalk = (await import('chalk')).default
+
+        const projects = filterProjectsByName(await parseAllSessions(), [project])
+        const p = projects[0]
+        if (!p) {
+          console.error(`Project "${project}" not found in session history.`)
+          process.exitCode = 1
+          return
+        }
+
+        const recommendation = recommendModelDefault(p)
+        if (!recommendation) {
+          console.error(`No default model recommendation available for ${project} at this time.`)
+          process.exitCode = 1
+          return
+        }
+
+        const plan = await buildApplyModelDefaultPlan(recommendation)
+        const record = await runAction(plan)
+
+        console.log(`Applied default model ${chalk.green(recommendation.candidateModel)} for ${project}`)
+        console.log(chalk.dim(`  Evidence: ${recommendation.candidateEditTurns} turns, ${(recommendation.candidateOneShotRate * 100).toFixed(1)}% one-shot, $${recommendation.candidateCostPerEdit.toFixed(3)}/edit`))
+        console.log(chalk.dim(`  Undo anytime: codeburn act undo ${shortId(record.id)}`))
+        console.log(chalk.dim(`  Per-session override: --model <name>`))
+      } catch (err) {
+        console.error(err instanceof Error ? err.message : String(err))
+        process.exitCode = 1
+      }
+    })
+
   act
     .command('report')
     .description('Realized vs estimated savings for applied actions older than 3 days')

src/act/model-defaults.ts

@@ -0,0 +1,171 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+import { aggregateModelStats, type ModelStats } from '../compare-stats.js'
+import type { ProjectSummary } from '../types.js'
+import { sha256File } from './backup.js'
+import type { ActionPlan } from './types.js'
+
+const MIN_EDIT_TURNS = 30
+const MAX_COST_RATIO = 0.6
+const ONE_SHOT_TOLERANCE = 0.03
+const DEBUGGING_HEAVY_THRESHOLD = 0.4
+const RECENCY_DAYS = 14
+const MS_PER_DAY = 24 * 60 * 60 * 1000
+
+export type ModelDefaultRecommendation = {
+  project: string
+  projectPath: string
+  currentModel: string
+  candidateModel: string
+  provider: string
+  currentEditTurns: number
+  candidateEditTurns: number
+  currentOneShotRate: number
+  candidateOneShotRate: number
+  currentCostPerEdit: number
+  candidateCostPerEdit: number
+  savingsPct: number
+  debuggingHeavy: boolean
+}
+
+function oneShotRate(s: ModelStats): number {
+  return s.editTurns > 0 ? s.oneShotTurns / s.editTurns : 0
+}
+
+function costPerEdit(s: ModelStats): number {
+  return s.editTurns > 0 ? s.editCost / s.editTurns : Number.POSITIVE_INFINITY
+}
+
+function isRecent(lastSeen: string, now: Date): boolean {
+  if (!lastSeen) return false
+  const seen = new Date(lastSeen)
+  if (Number.isNaN(seen.getTime())) return false
+  return now.getTime() - seen.getTime() <= RECENCY_DAYS * MS_PER_DAY
+}
+
+function providerByModel(project: ProjectSummary): Map<string, string> {
+  const providers = new Map<string, string>()
+  for (const session of project.sessions) {
+    for (const turn of session.turns) {
+      const primary = turn.assistantCalls[0]
+      if (!primary || primary.model === '<synthetic>') continue
+      if (!providers.has(primary.model)) providers.set(primary.model, primary.provider)
+      for (const call of turn.assistantCalls) {
+        if (call.model === '<synthetic>') continue
+        if (!providers.has(call.model)) providers.set(call.model, call.provider)
+      }
+    }
+  }
+  return providers
+}
+
+function isDebuggingHeavy(project: ProjectSummary): boolean {
+  let debuggingEditTurns = 0
+  let totalEditTurns = 0
+  for (const session of project.sessions) {
+    for (const breakdown of Object.values(session.categoryBreakdown)) {
+      totalEditTurns += breakdown.editTurns
+    }
+    debuggingEditTurns += session.categoryBreakdown.debugging?.editTurns ?? 0
+  }
+  return totalEditTurns > 0 && debuggingEditTurns / totalEditTurns > DEBUGGING_HEAVY_THRESHOLD
+}
+
+export function recommendModelDefault(project: ProjectSummary, opts: { now?: Date } = {}): ModelDefaultRecommendation | null {
+  const now = opts.now ?? new Date()
+  const stats = aggregateModelStats([project])
+    .filter(s => s.model !== '<synthetic>' && s.editTurns >= MIN_EDIT_TURNS)
+    .sort((a, b) => b.editTurns - a.editTurns || b.editCost - a.editCost)
+
+  const current = stats[0]
+  if (!current) return null
+
+  const providers = providerByModel(project)
+  const provider = providers.get(current.model)
+  if (!provider || !isRecent(current.lastSeen, now)) return null
+
+  const currentRate = oneShotRate(current)
+  const currentCost = costPerEdit(current)
+  if (!Number.isFinite(currentCost) || currentCost <= 0) return null
+
+  const debuggingHeavy = isDebuggingHeavy(project)
+  const tolerance = debuggingHeavy ? 0 : ONE_SHOT_TOLERANCE
+
+  const candidates = stats
+    .slice(1)
+    .filter(candidate => providers.get(candidate.model) === provider)
+    .filter(candidate => isRecent(candidate.lastSeen, now))
+    .map(candidate => ({
+      candidate,
+      candidateRate: oneShotRate(candidate),
+      candidateCost: costPerEdit(candidate),
+    }))
+    .filter(({ candidateRate }) => candidateRate >= currentRate - tolerance)
+    .filter(({ candidateCost }) => candidateCost <= currentCost * MAX_COST_RATIO)
+    .sort((a, b) => {
+      const savingsA = 1 - a.candidateCost / currentCost
+      const savingsB = 1 - b.candidateCost / currentCost
+      return savingsB - savingsA || b.candidateRate - a.candidateRate
+    })
+
+  const best = candidates[0]
+  if (!best) return null
+
+  return {
+    project: project.project,
+    projectPath: project.projectPath,
+    currentModel: current.model,
+    candidateModel: best.candidate.model,
+    provider,
+    currentEditTurns: current.editTurns,
+    candidateEditTurns: best.candidate.editTurns,
+    currentOneShotRate: currentRate,
+    candidateOneShotRate: best.candidateRate,
+    currentCostPerEdit: currentCost,
+    candidateCostPerEdit: best.candidateCost,
+    savingsPct: (1 - best.candidateCost / currentCost) * 100,
+    debuggingHeavy,
+  }
+}
+
+export async function buildApplyModelDefaultPlan(recommendation: ModelDefaultRecommendation): Promise<ActionPlan> {
+  const settingsPath = join(recommendation.projectPath, '.claude', 'settings.json')
+  let settings: Record<string, unknown> = {}
+  let expectedHash: string | null = null
+
+  try {
+    const raw = await readFile(settingsPath, 'utf-8')
+    expectedHash = await sha256File(settingsPath)
+    settings = JSON.parse(raw) as Record<string, unknown>
+    if (!settings || Array.isArray(settings) || typeof settings !== 'object') settings = {}
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code
+    if (code !== 'ENOENT') throw err
+  }
+
+  settings.model = recommendation.candidateModel
+
+  return {
+    kind: 'model-default',
+    findingId: `model-default:${recommendation.project}`,
+    description: `Set Claude Code default model to ${recommendation.candidateModel} for ${recommendation.project}`,
+    changes: [{
+      op: 'edit',
+      path: settingsPath,
+      content: JSON.stringify(settings, null, 2) + '\n',
+      expectedHash,
+    }],
+    baseline: {
+      windowDays: 30,
+      capturedAt: new Date().toISOString(),
+      estimatedTokens: 0,
+      sessions: recommendation.currentEditTurns + recommendation.candidateEditTurns,
+      candidateModel: recommendation.candidateModel,
+      metrics: {
+        [recommendation.candidateModel]: recommendation.candidateOneShotRate,
+        [recommendation.currentModel]: recommendation.currentOneShotRate,
+      },
+    },
+  }
+}

src/act/report.ts

@@ -1,4 +1,5 @@
 import { existsSync } from 'fs'
+import { dirname } from 'node:path'
 import type { DateRange, ProjectSummary, SessionSummary } from '../types.js'
 import type { ActionBaseline, ActionKind, ActionRecord } from './types.js'
 import type { FindingPlan } from './plans.js'
@@ -124,6 +125,26 @@ function sessionsInWindow(projects: ProjectSummary[], start: Date, end: Date): S
   return out
 }
 
+function projectPathKey(path: string): string {
+  const normalized = path.trim().replace(/\\/g, '/').replace(/\/+$/, '')
+  return normalized.toLowerCase()
+}
+
+function modelDefaultSessionsInWindow(
+  rec: ActionRecord, projects: ProjectSummary[], start: Date, end: Date,
+): { projectFound: boolean; sessions: SessionSummary[] } {
+  const settingsPath = rec.changes[0]?.path
+  if (!settingsPath) return { projectFound: false, sessions: [] }
+  const normalizedSettingsPath = settingsPath.replace(/\\/g, '/')
+  const targetProjectPath = dirname(dirname(normalizedSettingsPath))
+  const targetKey = projectPathKey(targetProjectPath)
+  const targetProjects = projects.filter(project => projectPathKey(project.projectPath) === targetKey)
+  return {
+    projectFound: targetProjects.length > 0,
+    sessions: sessionsInWindow(targetProjects, start, end),
+  }
+}
+
 function countToolCalls(sessions: SessionSummary[], names: ReadonlySet<string>): number {
   let n = 0
   for (const s of sessions) {
@@ -284,7 +305,61 @@ async function guardRow(
   }
 }
 
-async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, opts: ActReportOptions): Promise<ActReportRow> {
+async function modelDefaultRow(
+  base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[],
+  baseline: ActionBaseline, afterStart: Date, now: Date, projectFound: boolean,
+): Promise<ActReportRow> {
+  if (!projectFound) {
+    return { ...base, note: 'not measurable: project not found in current data (path may have changed)' }
+  }
+  const models = Object.keys(baseline.metrics)
+  if (models.length < 2) return { ...base, note: 'not measurable: invalid baseline' }
+  const candidateModel = baseline.candidateModel ?? models[0]!
+  const preApplyRate = baseline.metrics[candidateModel]
+  if (preApplyRate === undefined) return { ...base, note: 'not measurable: invalid baseline' }
+
+  const mockProject: ProjectSummary = {
+    project: 'mock',
+    projectPath: 'mock',
+    totalCostUSD: 0,
+    totalSavingsUSD: 0,
+    totalApiCalls: 0,
+    totalProxiedCostUSD: 0,
+    sessions,
+  }
+
+  const { aggregateModelStats } = await import('../compare-stats.js')
+  const stats = aggregateModelStats([mockProject]).find(s => s.model === candidateModel)
+
+  if (!stats || stats.editTurns < 20) {
+    return { ...base, note: `not measurable: < 20 edit turns for ${candidateModel} since apply` }
+  }
+
+  const postApplyRate = stats.oneShotTurns / stats.editTurns
+
+  if (postApplyRate < preApplyRate - 0.05) {
+    return {
+      ...base,
+      status: 'measured',
+      realizedTokens: 0,
+      confidence: 'low',
+      note: `quality regression, consider undo: one-shot rate ${(preApplyRate * 100).toFixed(1)}% -> ${(postApplyRate * 100).toFixed(1)}%`
+    }
+  }
+
+  return {
+    ...base,
+    status: 'measured',
+    realizedTokens: 0,
+    confidence: confidenceFor(sessions.length, baseline, afterStart, now),
+    note: `correlation, not attribution: one-shot rate ${(preApplyRate * 100).toFixed(1)}% -> ${(postApplyRate * 100).toFixed(1)}%`
+  }
+}
+
+async function computeRow(
+  rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date,
+  opts: ActReportOptions, modelDefaultProjectFound = true,
+): Promise<ActReportRow> {
   const estimatedAtApply = rec.baseline?.estimatedTokens ?? 0
   const base: ActReportRow = {
     id: rec.id,
@@ -307,6 +382,7 @@ async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterSt
   if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now)
   if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' }
   if (rec.kind === 'guard-install') return guardRow(base, afterStart, now, baseline, opts)
+  if (rec.kind === 'model-default') return modelDefaultRow(base, rec, sessions, baseline, afterStart, now, modelDefaultProjectFound)
   return { ...base, note: 'not measurable: kind is not tracked by act report' }
 }
 
@@ -361,7 +437,11 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise<Act
   const rows: ActReportRow[] = []
   for (const rec of eligible) {
     const afterStart = new Date(Math.max(new Date(rec.at).getTime(), windowStart.getTime()))
-    rows.push(await computeRow(rec, sessionsInWindow(projects, afterStart, now), afterStart, now, opts))
+    const modelDefaultWindow = rec.kind === 'model-default'
+      ? modelDefaultSessionsInWindow(rec, projects, afterStart, now)
+      : undefined
+    const sessions = modelDefaultWindow?.sessions ?? sessionsInWindow(projects, afterStart, now)
+    rows.push(await computeRow(rec, sessions, afterStart, now, opts, modelDefaultWindow?.projectFound))
   }
 
   const measuredRows = rows.filter(r => r.status === 'measured' && isTokenKind(r.kind))
@@ -405,6 +485,7 @@ function realizedCell(r: ActReportRow): string {
   if (r.status === 'reverted') return 'reverted'
   if (r.status === 'not-measurable') return 'not measurable'
   if (r.correlation) return `abandoned ${r.correlation.abandonedPctThen}% -> ${r.correlation.abandonedPctNow}% (corr.)`
+  if (r.kind === 'model-default') return 'correlation'
   return formatTokens(r.realizedTokens)
 }
 

src/act/types.ts

@@ -33,6 +33,8 @@ export type ActionBaseline = {
   estimatedTokens: number
   sessions: number
   metrics: Record<string, number>
+  // model-default only: identifies the candidate independently of metrics key order.
+  candidateModel?: string
 }
 
 export type ActionRecord = {