feat(vitest-evals): support a direct meta.eval.input, mirroring meta.eval.output

.changeset/vitest-evals-direct-input.md

@@ -0,0 +1,13 @@
+---
+"braintrust": minor
+---
+
+feat: `BraintrustVitestEvalsReporter` accepts a direct `input` on eval task meta
+
+Previously the only way to populate an eval row's `input` field was via
+`harness.run.session.messages` (the first message with `role: "user"`), which
+forces non-conversational harnesses to fabricate a synthetic message just to
+surface their real input. `meta.eval.input` now works exactly like the
+existing `meta.eval.output`: a direct passthrough that takes precedence over
+`session.messages` when present, falling back to the previous behavior when
+it's not set.

js/src/wrappers/vitest-evals/reporter.test.ts

@@ -1,11 +1,11 @@
 import { beforeAll, beforeEach, describe, expect, test, vi } from "vitest";
 import BraintrustVitestEvalsReporter from "./reporter";
-import { configureNode } from "../../node/config";
 import {
   _exportsForTestingOnly,
   type TestBackgroundLogger,
 } from "../../logger";
 import * as logger from "../../logger";
+import { configureNode } from "../../node/config";
 
 configureNode();
 
@@ -220,6 +220,65 @@ describe("Braintrust vitest-evals reporter", () => {
     );
   });
 
+  test("meta.eval.input is a direct passthrough that takes precedence over session.messages", async () => {
+    const reporter = new BraintrustVitestEvalsReporter({
+      displaySummary: false,
+      projectName: "vitest-evals-tests",
+    });
+
+    await reporter.onTestRunEnd([
+      fakeModule([
+        fakeTest({
+          meta: {
+            eval: {
+              avgScore: 1,
+              input: { invoiceId: "inv_123", amount: 42 },
+              output: { status: "approved" },
+            },
+            harness: {
+              run: {
+                session: {
+                  messages: [
+                    {
+                      role: "user",
+                      content: "ignored in favor of meta.eval.input",
+                    },
+                  ],
+                },
+                usage: {},
+              },
+            },
+          },
+          name: "structured input",
+        }),
+        fakeTest({
+          meta: {
+            eval: { avgScore: 1 },
+            harness: { run: { session: { messages: [] }, usage: {} } },
+          },
+          name: "falls back to session when meta.eval.input is absent",
+        }),
+      ]),
+    ] as any);
+
+    await backgroundLogger.flush();
+    const rows = (await backgroundLogger.drain()) as any[];
+
+    const structured = rows.find(
+      (row: any) => row.metadata?.fullName === "structured input",
+    );
+    expect(structured?.input).toMatchObject({
+      input: { invoiceId: "inv_123", amount: 42 },
+    });
+
+    const fallback = rows.find(
+      (row: any) =>
+        row.metadata?.fullName ===
+        "falls back to session when meta.eval.input is absent",
+    );
+    expect(fallback?.input?.input).toBeUndefined();
+  });
+
   test("logs failed eval scores and failure metadata", async () => {
     const reporter = new BraintrustVitestEvalsReporter({
       displaySummary: false,

js/src/wrappers/vitest-evals/reporter.ts

@@ -1,12 +1,12 @@
 import type { Reporter, TestCase, TestModule, Vitest } from "vitest/node";
-import { configureNode } from "../../node/config";
+import { SpanTypeAttribute, isObject } from "../../../util";
 import {
   initExperiment,
   logError,
   type Experiment,
   type Span,
 } from "../../logger";
-import { SpanTypeAttribute, isObject } from "../../../util";
+import { configureNode } from "../../node/config";
 import { summarizeAndFlush } from "../shared/flush";
 
 configureNode();
@@ -31,6 +31,7 @@ type EvalScore = {
 type EvalMeta = {
   scores?: EvalScore[];
   avgScore?: number | null;
+  input?: unknown;
   output?: unknown;
   thresholdFailed?: boolean;
   toolCalls?: ToolCallRecord[];
@@ -231,6 +232,7 @@ function logEvalTest(
   const result = test.result();
   const diagnostic = test.diagnostic();
   const run = meta.harness?.run;
+  const input = meta.eval?.input ?? firstUserMessageContent(run);
   const output = meta.eval?.output ?? run?.output;
   const scores = buildScores(result.state, meta.eval);
   const metrics = buildMetrics(diagnostic?.duration, run);
@@ -247,7 +249,7 @@ function logEvalTest(
     event: {
       input: {
         test: test.fullName || test.name,
-        input: firstUserMessageContent(run),
+        input,
       },
       ...(output !== undefined ? { output } : {}),
       scores,
@@ -519,6 +521,7 @@ function readEvalMeta(input: unknown): EvalMeta | undefined {
   return {
     ...(scores ? { scores } : {}),
     ...(avgScore !== undefined ? { avgScore } : {}),
+    ...(input.input !== undefined ? { input: input.input } : {}),
     ...(input.output !== undefined ? { output: input.output } : {}),
     ...(typeof input.thresholdFailed === "boolean"
       ? { thresholdFailed: input.thresholdFailed }