Deepen compare output and benchmark metadata

wimi321 · wimi321 · commit 795045f0be7f · 2026-03-18T21:04:07.000+08:00
diff --git a/README.md b/README.md
@@ -144,6 +144,7 @@ npm run dev -- pack --config ./starter/taskbundle.config.json
 `pack` also supports:
 - automatic git metadata detection
 - artifact hashes and sizes in `bundle.json`
+- benchmark-style outcome fields such as `status`, `score`, and `judgeNotes`
 - optional `.tar.gz` archive creation with `--archive`
 
 ### `taskbundle inspect`
@@ -220,6 +221,7 @@ They represent the same task captured from different tool/model combinations so
 - `workspace/files/`: captured task-related files
 - `git`: optional git root / branch / remote / commit metadata
 - `runner`: optional pack-time runtime metadata
+- `outcome`: optional benchmark or judge result fields
 
 ## Local Development
 
@@ -280,7 +282,7 @@ docs/
 ## Known Limitations
 
 - Archives currently use `.tar.gz`, not `.zip`.
-- The project compares bundle-level metadata and counts, not semantic code quality.
+- The project can compare metadata, scores, and artifact hashes, but it still does not judge semantic code quality on its own.
 - Workspace capture still uses explicit copied file sets instead of repository-wide snapshot strategies.
 - There is no viewer UI yet.
 
diff --git a/README.zh-CN.md b/README.zh-CN.md
@@ -144,6 +144,7 @@ npm run dev -- pack --config ./starter/taskbundle.config.json
 `pack` 现在还支持：
 - 自动采集 git 元数据
 - 在 `bundle.json` 中记录 artifact 哈希和大小
+- 写入 benchmark / judge 结果字段，例如 `status`、`score`、`judgeNotes`
 - 用 `--archive` 直接生成 `.tar.gz`
 
 ### `taskbundle inspect`
@@ -220,6 +221,7 @@ npm run dev -- scan ./examples
 - `workspace/files/`：捕获到的任务相关文件
 - `git`：可选的 git root / branch / remote / commit 信息
 - `runner`：可选的打包运行时信息
+- `outcome`：可选的 benchmark / judge 结果字段
 
 ## 本地开发
 
@@ -280,7 +282,7 @@ docs/
 ## 当前限制
 
 - 归档格式目前是 `.tar.gz`，不是 `.zip`
-- compare 目前主要比较 bundle 元数据和统计信息，不做语义级代码评判
+- compare 现在可以比较 metadata、score 和 artifact hash，但它仍然不会自动做语义级代码评判
 - workspace 捕获仍然基于显式文件集合，而不是整仓库镜像策略
 - 还没有 viewer UI
 
diff --git a/ROADMAP.md b/ROADMAP.md
@@ -9,14 +9,15 @@ Task Bundle started as a small CLI MVP. This roadmap turns it into a practical f
 - Done: schema validation for bundle metadata, workspace manifests, and event logs
 - Done: automatic git metadata detection during packing
 - Done: `compare` command
+- Done: richer `compare` output with artifact hash differences and score deltas
 - Done: `archive` and `extract` commands for `.tar.gz` bundles
 - Done: `validate` and `scan` commands for replay checks and bundle collections
 - Done: artifact hashes and sizes in `bundle.json`
+- Done: benchmark-style outcome fields in bundle metadata
 - Done: CLI smoke tests and GitHub Actions CI
 - Done: Chinese and English documentation
 
 ### v0.3
-- Planned: richer `compare` output with file-level change summaries
 - Planned: machine-readable benchmark result fields and scoring conventions
 - Planned: bundle collections and directory scans for multi-run comparisons
 - Planned: more curated example bundles for benchmark-style demos
diff --git a/ROADMAP.zh-CN.md b/ROADMAP.zh-CN.md
@@ -9,14 +9,15 @@ Task Bundle 目前已经从一个小型 CLI MVP，走到了“可实际使用的
 - 已完成：bundle metadata、workspace manifest、events 的 schema 校验
 - 已完成：打包时自动采集 git 元数据
 - 已完成：`compare` 命令
+- 已完成：更丰富的 `compare` 输出，包括 artifact hash 差异和 score delta
 - 已完成：`.tar.gz` 归档与解压命令
 - 已完成：`validate` 与 `scan` 命令，用于 replay 校验和 bundle 集合扫描
 - 已完成：artifact 哈希和大小写入 `bundle.json`
+- 已完成：bundle metadata 中的 benchmark / judge 结果字段
 - 已完成：CLI smoke tests 和 GitHub Actions CI
 - 已完成：中英文文档
 
 ### v0.3
-- 规划中：更丰富的 `compare` 输出，包括文件级差异摘要
 - 规划中：面向 benchmark 的结果字段和评分约定
 - 规划中：多 bundle 目录扫描与批量比较
 - 规划中：更多用于演示和 benchmark 的标准 example bundles
diff --git a/examples/hello-world-bundle-claude/bundle.json b/examples/hello-world-bundle-claude/bundle.json
@@ -17,5 +17,43 @@
     "events": "events.jsonl",
     "workspaceManifest": "workspace/manifest.json",
     "workspaceFilesDir": "workspace/files"
+  },
+  "artifactInfo": {
+    "task": {
+      "path": "task.md",
+      "sha256": "18f5af61553b0b4e4de1a631c8984b9c260b2d8f04e709919d3e1da361c3ccf8",
+      "size": 327
+    },
+    "summary": {
+      "path": "summary.md",
+      "sha256": "9197bf9acdc1ae2c0b816c768ff8d965beaa0dbbc5c0a912af1d566dac77c53a",
+      "size": 243
+    },
+    "diff": {
+      "path": "result.diff",
+      "sha256": "019b7bc586b6b47ea312f3a2477c6a921ef26863b2d794686735efaa6f9d4f03",
+      "size": 216
+    },
+    "events": {
+      "path": "events.jsonl",
+      "sha256": "99552c51ac717180ccf82cba204b145feac016aedd34357ece403e29862ff91c",
+      "size": 414
+    },
+    "workspaceManifest": {
+      "path": "workspace/manifest.json",
+      "sha256": "597e67b477306fa3cb74838e3273c2aba220940b5ad81530e6ea820ea5174d1a",
+      "size": 255
+    }
+  },
+  "runner": {
+    "os": "darwin",
+    "nodeVersion": "v25.8.0",
+    "cliVersion": "0.3.0",
+    "promptSource": "cli"
+  },
+  "outcome": {
+    "status": "success",
+    "score": 0.89,
+    "judgeNotes": "Correct fix, slightly more event noise, still replay-ready."
   }
 }
diff --git a/examples/hello-world-bundle/bundle.json b/examples/hello-world-bundle/bundle.json
@@ -44,5 +44,16 @@
       "sha256": "597e67b477306fa3cb74838e3273c2aba220940b5ad81530e6ea820ea5174d1a",
       "size": 255
     }
+  },
+  "runner": {
+    "os": "darwin",
+    "nodeVersion": "v25.8.0",
+    "cliVersion": "0.3.0",
+    "promptSource": "cli"
+  },
+  "outcome": {
+    "status": "success",
+    "score": 0.93,
+    "judgeNotes": "Small patch, correct output, complete workspace capture."
   }
 }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "task-bundle",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Portable task bundles for AI coding work.",
   "license": "MIT",
   "type": "commonjs",
diff --git a/src/cli/commands/compare.ts b/src/cli/commands/compare.ts
@@ -28,11 +28,34 @@ export function registerCompareCommand(program: Command): void {
       printKeyValue("Right model", comparison.modelChange.right);
       printKeyValue("Left tool", comparison.toolChange.left);
       printKeyValue("Right tool", comparison.toolChange.right);
+      printKeyValue("Left status", comparison.outcomeChange.leftStatus);
+      printKeyValue("Right status", comparison.outcomeChange.rightStatus);
+      printKeyValue("Left score", comparison.outcomeChange.leftScore?.toString());
+      printKeyValue("Right score", comparison.outcomeChange.rightScore?.toString());
+      printKeyValue("Score delta", formatNumber(comparison.outcomeChange.scoreDelta));
       console.log("");
       printList("Only in left", comparison.artifactDelta.onlyInLeft.length > 0 ? comparison.artifactDelta.onlyInLeft : ["None"]);
       printList("Only in right", comparison.artifactDelta.onlyInRight.length > 0 ? comparison.artifactDelta.onlyInRight : ["None"]);
       console.log("");
+      printList(
+        "Artifact hash changes",
+        comparison.artifactChanges
+          .filter((artifact) => !artifact.sameHash)
+          .map((artifact) => `${artifact.artifact}: ${artifact.left?.sha256 ?? "missing"} -> ${artifact.right?.sha256 ?? "missing"}`)
+          .concat(
+            comparison.artifactChanges.filter((artifact) => !artifact.sameHash).length === 0 ? ["None"] : []
+          )
+      );
+      console.log("");
       printKeyValue("Workspace file delta", String(comparison.counts.workspaceFilesDelta));
       printKeyValue("Event count delta", String(comparison.counts.eventCountDelta));
     });
 }
+
+function formatNumber(value: number | undefined): string | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  return Number(value.toFixed(4)).toString();
+}
diff --git a/src/cli/commands/inspect.ts b/src/cli/commands/inspect.ts
@@ -29,6 +29,9 @@ export function registerInspectCommand(program: Command): void {
       printKeyValue("Repo", inspection.repo);
       printKeyValue("Commit", inspection.commit);
       printKeyValue("Branch", inspection.branch);
+      printKeyValue("Status", inspection.outcome?.status);
+      printKeyValue("Score", inspection.outcome?.score?.toString());
+      printKeyValue("Prompt source", inspection.promptSource);
       printKeyValue("Tags", inspection.tags.join(", "));
       console.log("");
       printList("Artifacts", inspection.artifacts);
diff --git a/src/cli/commands/pack.ts b/src/cli/commands/pack.ts
@@ -26,6 +26,10 @@ interface PackCliOptions {
   archive?: string;
   cwd?: string;
   gitAuto?: boolean;
+  status?: "success" | "failure" | "partial";
+  score?: string;
+  judgeNotes?: string;
+  promptSource?: string;
 }
 
 export function registerPackCommand(program: Command): void {
@@ -48,6 +52,10 @@ export function registerPackCommand(program: Command): void {
     .option("--branch <name>", "Git branch name")
     .option("--archive <file>", "Write a .tar.gz archive after packing")
     .option("--cwd <dir>", "Base working directory for config resolution and git detection")
+    .option("--status <status>", "Outcome status: success, failure, or partial")
+    .option("--score <number>", "Numeric benchmark or judge score")
+    .option("--judge-notes <text>", "Short benchmark or judge notes")
+    .option("--prompt-source <source>", "Prompt source label, such as cli or config")
     .option("--tag <tag>", "Tag to add to bundle metadata", collectTags, [])
     .option("--no-git-auto", "Disable automatic git metadata detection")
     .action(async (options: PackCliOptions, command: Command) => {
@@ -87,8 +95,8 @@ export function registerPackCommand(program: Command): void {
       const runner: RunnerMetadata = {
         os: process.platform,
         nodeVersion: process.version,
-        cliVersion: "0.2.0",
-        promptSource: options.config ? "config" : "cli"
+        cliVersion: "0.3.0",
+        promptSource: merged.promptSource ?? (options.config ? "config" : "cli")
       };
 
       const metadata = await createBundle({
@@ -107,7 +115,15 @@ export function registerPackCommand(program: Command): void {
         branch: merged.branch ?? gitMetadata?.branch,
         tags: merged.tags ?? [],
         git: gitMetadata,
-        runner
+        runner,
+        outcome:
+          merged.status || merged.score !== undefined || merged.judgeNotes
+            ? {
+                status: merged.status,
+                score: merged.score,
+                judgeNotes: merged.judgeNotes
+              }
+            : undefined
       });
 
       console.log(`Created bundle at ${outputDir}`);
@@ -153,6 +169,10 @@ function mergePackOptions(
       out: cli.out,
       archive: cli.archive,
       cwd: cli.cwd,
+      status: cli.status,
+      score: parseNumericOption(cli.score, "score"),
+      judgeNotes: cli.judgeNotes,
+      promptSource: cli.promptSource,
       tags: cliTags,
       gitAuto
     })
@@ -176,3 +196,16 @@ function stripUndefined<T extends Record<string, unknown>>(value: T): Partial<T>
     Object.entries(value).filter(([, entry]) => entry !== undefined)
   ) as Partial<T>;
 }
+
+function parseNumericOption(value: string | undefined, label: string): number | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`${label} must be a valid number.`);
+  }
+
+  return parsed;
+}
diff --git a/src/cli/commands/scan.ts b/src/cli/commands/scan.ts
@@ -26,6 +26,7 @@ export function registerScanCommand(program: Command): void {
         console.log(inspection.title);
         console.log(`- schema: ${inspection.schemaVersion}`);
         console.log(`- tool/model: ${inspection.tool ?? "unknown"} / ${inspection.model ?? "unknown"}`);
+        console.log(`- status/score: ${inspection.outcome?.status ?? "unknown"} / ${inspection.outcome?.score ?? "n/a"}`);
         console.log(`- workspace files: ${inspection.workspaceFileCount}`);
         console.log(`- events: ${inspection.eventCount}`);
       }
diff --git a/src/cli/index.ts b/src/cli/index.ts
@@ -14,7 +14,7 @@ async function main(): Promise<void> {
   program
     .name("taskbundle")
     .description("Package AI coding work into portable task bundles.")
-    .version("0.2.0");
+    .version("0.3.0");
 
   registerInitCommand(program);
   registerPackCommand(program);
diff --git a/src/core/bundle.ts b/src/core/bundle.ts
@@ -213,6 +213,10 @@ export async function createInitTemplate(destinationDir: string): Promise<void>
     model: "gpt-5",
     runtime: "node",
     gitAuto: true,
+    status: "success",
+    score: 1,
+    judgeNotes: "Starter example bundle",
+    promptSource: "config",
     out: "./bundle-output",
     archive: "./bundle-output.tar.gz",
     tags: ["starter"]
@@ -255,7 +259,9 @@ export async function inspectBundle(bundleDir: string): Promise<BundleInspection
     artifacts,
     workspaceFileCount: bundle.workspaceManifest?.fileCount ?? 0,
     eventCount: bundle.parsedEvents?.length ?? countJsonlRecords(bundle.events),
-    artifactInfo: bundle.metadata.artifactInfo ?? {}
+    artifactInfo: bundle.metadata.artifactInfo ?? {},
+    outcome: bundle.metadata.outcome,
+    promptSource: bundle.metadata.runner?.promptSource
   };
 }
 
diff --git a/src/core/compare.ts b/src/core/compare.ts
@@ -1,10 +1,13 @@
 import path from "node:path";
-import { BundleComparison } from "./schema";
 import { inspectBundle } from "./bundle";
+import { ArtifactInfo, BundleComparison } from "./schema";
 
 export async function compareBundles(leftDir: string, rightDir: string): Promise<BundleComparison> {
   const left = await inspectBundle(path.resolve(leftDir));
   const right = await inspectBundle(path.resolve(rightDir));
+  const leftArtifactKeys = Object.keys(left.artifactInfo);
+  const rightArtifactKeys = Object.keys(right.artifactInfo);
+  const sharedArtifactKeys = [...new Set([...leftArtifactKeys, ...rightArtifactKeys])].sort();
 
   return {
     left,
@@ -16,6 +19,18 @@ export async function compareBundles(leftDir: string, rightDir: string): Promise
       onlyInLeft: left.artifacts.filter((artifact) => !right.artifacts.includes(artifact)),
       onlyInRight: right.artifacts.filter((artifact) => !left.artifacts.includes(artifact))
     },
+    artifactChanges: sharedArtifactKeys.map((artifact) => {
+      const leftArtifact = left.artifactInfo[artifact];
+      const rightArtifact = right.artifactInfo[artifact];
+
+      return {
+        artifact,
+        left: leftArtifact,
+        right: rightArtifact,
+        sameHash: compareArtifactField(leftArtifact, rightArtifact, "sha256"),
+        sameSize: compareArtifactField(leftArtifact, rightArtifact, "size")
+      };
+    }),
     counts: {
       workspaceFilesDelta: left.workspaceFileCount - right.workspaceFileCount,
       eventCountDelta: left.eventCount - right.eventCount
@@ -27,6 +42,28 @@ export async function compareBundles(leftDir: string, rightDir: string): Promise
     toolChange: {
       left: left.tool,
       right: right.tool
+    },
+    outcomeChange: {
+      leftStatus: left.outcome?.status,
+      rightStatus: right.outcome?.status,
+      leftScore: left.outcome?.score,
+      rightScore: right.outcome?.score,
+      scoreDelta:
+        left.outcome?.score !== undefined && right.outcome?.score !== undefined
+          ? left.outcome.score - right.outcome.score
+          : undefined
     }
   };
 }
+
+function compareArtifactField(
+  left: ArtifactInfo | undefined,
+  right: ArtifactInfo | undefined,
+  field: "sha256" | "size"
+): boolean {
+  if (!left || !right) {
+    return false;
+  }
+
+  return left[field] === right[field];
+}
diff --git a/src/core/schema.ts b/src/core/schema.ts
diff --git a/src/core/validation.ts b/src/core/validation.ts
diff --git a/test/bundle.test.ts b/test/bundle.test.ts
diff --git a/test/cli.test.ts b/test/cli.test.ts

Original file line number	Diff line number	Diff line change
`@@ -44,5 +44,16 @@`
`44`	`44`	`"sha256": "597e67b477306fa3cb74838e3273c2aba220940b5ad81530e6ea820ea5174d1a",`
`45`	`45`	`"size": 255`
`46`	`46`	`}`
	`47`	`+ },`
	`48`	`+ "runner": {`
	`49`	`+ "os": "darwin",`
	`50`	`+ "nodeVersion": "v25.8.0",`
	`51`	`+ "cliVersion": "0.3.0",`
	`52`	`+ "promptSource": "cli"`
	`53`	`+ },`
	`54`	`+ "outcome": {`
	`55`	`+ "status": "success",`
	`56`	`+ "score": 0.93,`
	`57`	`+ "judgeNotes": "Small patch, correct output, complete workspace capture."`
`47`	`58`	`}`
`48`	`59`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "task-bundle",`
`3`		`- "version": "0.2.0",`
	`3`	`+ "version": "0.3.0",`
`4`	`4`	`"description": "Portable task bundles for AI coding work.",`
`5`	`5`	`"license": "MIT",`
`6`	`6`	`"type": "commonjs",`