Translation bench

ts/packages/benchmarks/.gitignore

@@ -2,4 +2,5 @@ node_modules/
 dist/
 data/
 results/
+local/
 *.tsbuildinfo

ts/packages/benchmarks/AGENTS.md

@@ -0,0 +1,72 @@
+# @typeagent/benchmarks — agent notes
+
+## Layout
+
+- `src/core/` — domain-agnostic infrastructure.
+  - `rateLimiter.ts` — cross-process tokens-per-minute limiter (shared SQLite).
+  - `tokenEstimate.ts` — model-agnostic prompt token estimate for reservations.
+- `src/translationBench/`
+  - `runConfig.ts` + `config.schema.json` — pure JSON run-config loader/resolver.
+  - `synthesizer/` — dataset generation, quality gates, negative fairness.
+  - `runner/` — suite execution, scoring, checkpoints, reports, explainer.
+  - `policy/` — eligible-gold allowlist + action quality picker.
+  - `scripts/tbEval.ts`, `scripts/tbGenerate.ts` — thin production CLIs.
+- Assets (`config.schema.json`, prompt packs) are copied to `dist/` by
+  `scripts/copyAssets.mjs` during build.
+
+## Config: JSON + commander, no `TB_*` env
+
+Run configuration is a JSON file validated by `config.schema.json`. Runtime
+overrides are **commander flags**, prop-drilled into the library — do not read
+`process.env.TB_*`.
+
+```bash
+# eval (requires a pre-approved artifact; never auto-approves)
+node dist/translationBench/scripts/tbEval.js \
+  --draft ./artifacts/benchmark-draft-1000.jsonl \
+  --approved ./artifacts/benchmark-approved-1000.jsonl \
+  --config ./run-config.json \
+  --batch eval
+
+# generate
+node dist/translationBench/scripts/tbGenerate.js \
+  --source ./source/anchors.jsonl \
+  --manifest ./source/source-manifest.json \
+  --config ./run-config.json \
+  --batch synthesizer
+```
+
+`tb-eval` refuses to mint `approval.status: "approved"` and fails when draft
+content drifts from the approved file. See
+`src/translationBench/config/run-config.example.json`.
+
+## Credential env boundary
+
+`OPENAI_*` / `AZURE_*` env is the `@typeagent/aiclient` contract
+(`initRuntimeConfigFromProcessEnv()`) and is intentionally kept.
+
+## TPM rate limiter
+
+`createRateLimiter(tpmLimits, { dbPath, estTokensPerCall, maxWaitMs?, onWait? })`
+requires `dbPath`. Concurrent `run()` calls reserve tokens against the shared
+SQLite ledger over a rolling 60s window and settle to actual usage.
+
+## Runner library
+
+Import via package subpath (not star-exported from the main barrel — names
+overlap synthesizer checkpoint helpers):
+
+```ts
+import {
+  runTranslationBench,
+  scoreTranslationBench,
+} from "@typeagent/benchmarks/translationBench/runner";
+```
+
+Callers own dispatcher bootstrap (`initializeCommandHandlerContext`). The runner
+only crosses into agent-dispatcher at `translateRequest`.
+
+## local/ is gitignored
+
+Scratch run artifacts stay under `local/` (gitignored). Committed code lives
+under `src/`.

ts/packages/benchmarks/README.AUTOGEN.md

@@ -3,7 +3,7 @@
 
 <!-- AUTOGEN:DOCS:START -->
 
-<!-- AUTOGEN:DOCS:HASH:sha256=04ffc6fa34cfbe0d06383b4e35423fc5a203f9c3bd8b8f07c91c20b2cef9903c -->
+<!-- AUTOGEN:DOCS:HASH:sha256=d90f01503aa167c441eaf2d3c2a5086314545f404a3eea19a573fdcc3c423963 -->
 <!-- AUTOGEN:DOCS:SOURCE: ./README.md (hand-written documentation; this file is the AI-generated companion) -->
 
 # @typeagent/benchmarks — AI-generated documentation
@@ -34,7 +34,7 @@ Workspace:
 - [agent-dispatcher](../../packages/dispatcher/dispatcher/README.md)
 - [default-agent-provider](../../packages/defaultAgentProvider/README.md)
 
-External: `commander`, `js-yaml`, `zod`
+External: `commander`, `gpt-tokenizer`, `js-yaml`, `zod`
 
 ### Used by
 
@@ -44,18 +44,18 @@ _None._
 
 - [./src/index.ts](./src/index.ts)
 - [./src/translationBench/index.ts](./src/translationBench/index.ts)
-- [./src/translationBench/synthesizer/catalogGenerator/index.ts](./src/translationBench/synthesizer/catalogGenerator/index.ts)
+- [./src/translationBench/policy/index.ts](./src/translationBench/policy/index.ts)
 - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts)
 - [./src/core/model-prices.generated.json](./src/core/model-prices.generated.json)
 - [./src/core/paths.ts](./src/core/paths.ts)
 - [./src/core/prices.ts](./src/core/prices.ts)
+- [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts)
+- [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts)
 - [./src/core/types.ts](./src/core/types.ts)
-- [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json)
-- [./src/translationBench/catalog.generated.json](./src/translationBench/catalog.generated.json)
-- _…and 29 more under `./src/`._
+- _…and 44 more under `./src/`._
 
 ---
 
-_Auto-generated against commit `54efea2e226011740764eddb4beee99edc562313` on `2026-08-08T00:27:51.771Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
+_Auto-generated against commit `cb113126e39f7f4c5ebb1ec3603a78ecf5241572` on `2026-08-11T01:06:02.321Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
 
 <!-- AUTOGEN:DOCS:END -->

ts/packages/benchmarks/README.md

@@ -4,7 +4,14 @@ Action-translation eval for TypeAgent: pinned catalogs, model prices, and scorin
 
 ## Catalog + action-parameters grader
 
-Pinned `catalog.generated.json` and `action-parameters-grader.generated.json`. Code/script parameters use verify mode `llmAsAJudge` (not exact); synthesizer exclusions are derived from those fields. Regenerate with `pnpm run gen-catalog` (`--force` full rebuild). Tests: `pnpm run test:local`.
+Pinned `catalog.generated.json` and `action-parameters-grader.generated.json`.
+
+Human policy lives in `src/translationBench/policy/action-eligibility.json` (+ `.schema.json`):
+
+- **`removedActions`** — actions that must not be gold targets (`type: "action"` exact ids, or `type: "prefix"` `onboarding.*` only). They stay in the catalog for routing.
+- **`parameterOverrides`** — pin per-field **`verify`** only (`type: "field"`). `create` is never set in policy; type/regex derive minting. Override paths are skipped by the LLM classifier when regenerating the grader.
+
+Regenerate grader: `pnpm run gen-policy` (alias `gen-action-parameters-grader`). Full catalog+grader: `pnpm run gen-catalog`. Tests: `pnpm run test:local`.
 
 ## Dataset synthesizer (part 3)
 

ts/packages/benchmarks/package.json

@@ -14,7 +14,8 @@
   "exports": {
     ".": "./dist/index.js",
     "./translationBench": "./dist/translationBench/index.js",
-    "./internal": "./dist/index.js"
+    "./internal": "./dist/index.js",
+    "./translationBench/runner": "./dist/translationBench/runner/index.js"
   },
   "files": [
     "dist",
@@ -23,22 +24,28 @@
   "scripts": {
     "build": "tsc -b && node ./scripts/copyAssets.mjs",
     "clean": "node ./scripts/clean.mjs",
-    "gen-action-parameters-grader": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genActionParametersGrader.js",
-    "gen-catalog": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genCatalog.js && node --max-old-space-size=4096 dist/translationBench/scripts/genActionParametersGrader.js",
+    "gen-action-parameters-grader": "pnpm run gen-policy",
+    "gen-catalog": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genCatalog.js && node --max-old-space-size=4096 dist/translationBench/scripts/genPolicy.js && node --max-old-space-size=4096 dist/translationBench/scripts/pickEligibleActions.js --model ${TB_PICKER_MODEL:-azure/gpt-5.6-sol} && node ./scripts/copyAssets.mjs",
+    "gen-policy": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genPolicy.js && node ./scripts/copyAssets.mjs",
     "jest-esm": "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js",
+    "pick-eligible-actions": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/pickEligibleActions.js --model ${TB_PICKER_MODEL:-azure/gpt-5.6-sol} && node ./scripts/copyAssets.mjs",
     "prettier": "prettier --check package.json tsconfig.json src scripts test --ignore-path ../../.prettierignore",
     "prettier:fix": "prettier --write package.json tsconfig.json src scripts test --ignore-path ../../.prettierignore",
     "test": "npm run test:local",
     "test:local": "pnpm run build && pnpm run jest-esm --testPathPattern=\".*[.]spec[.]js\"",
-    "tsc": "tsc -b"
+    "tsc": "tsc -b",
+    "tb-eval": "node ./dist/translationBench/scripts/tbEval.js",
+    "tb-generate": "node ./dist/translationBench/scripts/tbGenerate.js"
   },
   "dependencies": {
     "@typeagent/action-schema": "workspace:*",
+    "@typeagent/agent-cache": "workspace:*",
     "@typeagent/agent-sdk": "workspace:*",
     "@typeagent/aiclient": "workspace:*",
     "agent-dispatcher": "workspace:*",
     "commander": "^12.1.0",
     "default-agent-provider": "workspace:*",
+    "gpt-tokenizer": "^2.9.0",
     "js-yaml": "^4.3.0",
     "zod": "^4.1.13"
   },

ts/packages/benchmarks/scripts/copyAssets.mjs

@@ -45,14 +45,36 @@ const files = [
         "src/translationBench/action-parameters-grader.generated.json",
         "dist/translationBench/action-parameters-grader.generated.json",
     ],
+    [
+        "src/translationBench/eligible-gold-actions.generated.json",
+        "dist/translationBench/eligible-gold-actions.generated.json",
+    ],
+    [
+        "src/translationBench/config.schema.json",
+        "dist/translationBench/config.schema.json",
+    ],
+    [
+        "src/translationBench/config/run-config.example.json",
+        "dist/translationBench/config/run-config.example.json",
+    ],
     [
         "src/core/model-prices.generated.json",
         "dist/core/model-prices.generated.json",
     ],
 ];
 
+const requiredGenerated = new Set([
+    "src/translationBench/catalog.generated.json",
+    "src/translationBench/action-parameters-grader.generated.json",
+    "src/translationBench/eligible-gold-actions.generated.json",
+    "src/translationBench/policy/action-eligibility.json",
+]);
 for (const [fromRel, toRel] of files) {
-    copyFileFast(path.join(root, fromRel), path.join(root, toRel));
+    const from = path.join(root, fromRel);
+    if (requiredGenerated.has(fromRel) && !existsSync(from)) {
+        throw new Error(`copyAssets: missing required asset ${fromRel}`);
+    }
+    copyFileFast(from, path.join(root, toRel));
 }
 
 const yamlSrc = path.join(root, "src/translationBench/synthesizer");
@@ -70,6 +92,39 @@ if (existsSync(yamlSrc)) {
     }
 }
 
+const policyFiles = [
+    [
+        "src/translationBench/policy/action-eligibility.json",
+        "dist/translationBench/policy/action-eligibility.json",
+    ],
+    [
+        "src/translationBench/policy/action-eligibility.schema.json",
+        "dist/translationBench/policy/action-eligibility.schema.json",
+    ],
+];
+for (const [fromRel, toRel] of policyFiles) {
+    const from = path.join(root, fromRel);
+    if (!existsSync(from)) {
+        throw new Error(`copyAssets: missing required asset ${fromRel}`);
+    }
+    copyFileFast(from, path.join(root, toRel));
+}
+
+const policyYamlSrc = path.join(root, "src/translationBench/policy");
+const policyYamlDst = path.join(root, "dist/translationBench/policy");
+if (existsSync(policyYamlSrc)) {
+    for (const name of readdirSync(policyYamlSrc, { withFileTypes: true })) {
+        if (!name.isFile()) continue;
+        if (!name.name.endsWith(".yaml") && !name.name.endsWith(".yml")) {
+            continue;
+        }
+        copyFileFast(
+            path.join(policyYamlSrc, name.name),
+            path.join(policyYamlDst, name.name),
+        );
+    }
+}
+
 const seedSrc = path.join(root, "src/translationBench/synthesizer/seed");
 const seedDst = path.join(root, "dist/translationBench/synthesizer/seed");
 if (existsSync(seedSrc)) {