From a3573283c30915730a50d727aac664ba9c4c898f Mon Sep 17 00:00:00 2001 From: Aislinn Cronin Date: Fri, 31 Jul 2026 17:13:13 -0400 Subject: [PATCH 1/3] feat(vitest-evals): support a direct meta.eval.input, mirroring meta.eval.output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eval-level root span's `input` field could only be populated via harness.run.session.messages (the first message with role: "user"), via firstUserMessageContent(). That forces any non-conversational harness to fabricate a synthetic message just to surface its real input, even though meta.eval.output already supports a direct passthrough for the symmetric case, and logToolCallSpans in this same file already does `input: call.arguments` with no such indirection. Add meta.eval.input as a direct passthrough that takes precedence over session.messages when present, falling back to the existing firstUserMessageContent(run) behavior when it's not set — so existing consumers relying on session.messages are unaffected. Co-Authored-By: Claude Sonnet 5 --- .changeset/vitest-evals-direct-input.md | 13 ++++ js/src/wrappers/vitest-evals/reporter.test.ts | 61 ++++++++++++++++++- js/src/wrappers/vitest-evals/reporter.ts | 13 +++- 3 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 .changeset/vitest-evals-direct-input.md diff --git a/.changeset/vitest-evals-direct-input.md b/.changeset/vitest-evals-direct-input.md new file mode 100644 index 000000000..ce0e20aa6 --- /dev/null +++ b/.changeset/vitest-evals-direct-input.md @@ -0,0 +1,13 @@ +--- +"braintrust": minor +--- + +feat(vitest-evals): support a direct `input` on eval task meta, mirroring `output` + +Previously the only way to populate an eval row's `input` field was via +`harness.run.session.messages` (the first message with `role: "user"`), which +forces non-conversational harnesses to fabricate a synthetic message just to +surface their real input. `meta.eval.input` now works exactly like the +existing `meta.eval.output`: a direct passthrough that takes precedence over +`session.messages` when present, falling back to the previous behavior when +it's not set. diff --git a/js/src/wrappers/vitest-evals/reporter.test.ts b/js/src/wrappers/vitest-evals/reporter.test.ts index 23075f5d1..18bf9f2ee 100644 --- a/js/src/wrappers/vitest-evals/reporter.test.ts +++ b/js/src/wrappers/vitest-evals/reporter.test.ts @@ -1,11 +1,11 @@ import { beforeAll, beforeEach, describe, expect, test, vi } from "vitest"; import BraintrustVitestEvalsReporter from "./reporter"; -import { configureNode } from "../../node/config"; import { _exportsForTestingOnly, type TestBackgroundLogger, } from "../../logger"; import * as logger from "../../logger"; +import { configureNode } from "../../node/config"; configureNode(); @@ -220,6 +220,65 @@ describe("Braintrust vitest-evals reporter", () => { ); }); + test("meta.eval.input is a direct passthrough that takes precedence over session.messages", async () => { + const reporter = new BraintrustVitestEvalsReporter({ + displaySummary: false, + projectName: "vitest-evals-tests", + }); + + await reporter.onTestRunEnd([ + fakeModule([ + fakeTest({ + meta: { + eval: { + avgScore: 1, + input: { memberUuid: "member-1", payPeriodEnd: "2026-06-15" }, + output: { status: "approved" }, + }, + harness: { + run: { + session: { + messages: [ + { + role: "user", + content: "ignored in favor of meta.eval.input", + }, + ], + }, + usage: {}, + }, + }, + }, + name: "structured input", + }), + fakeTest({ + meta: { + eval: { avgScore: 1 }, + harness: { run: { session: { messages: [] }, usage: {} } }, + }, + name: "falls back to session when meta.eval.input is absent", + }), + ]), + ] as any); + + await backgroundLogger.flush(); + const rows = (await backgroundLogger.drain()) as any[]; + + const structured = rows.find( + (row: any) => row.metadata?.fullName === "structured input", + ); + expect(structured?.input).toMatchObject({ + input: { memberUuid: "member-1", payPeriodEnd: "2026-06-15" }, + }); + + const fallback = rows.find( + (row: any) => + row.metadata?.fullName === + "falls back to session when meta.eval.input is absent", + ); + expect(fallback?.input?.input).toBeUndefined(); + }); + test("logs failed eval scores and failure metadata", async () => { const reporter = new BraintrustVitestEvalsReporter({ displaySummary: false, diff --git a/js/src/wrappers/vitest-evals/reporter.ts b/js/src/wrappers/vitest-evals/reporter.ts index 357f51506..d882e27ab 100644 --- a/js/src/wrappers/vitest-evals/reporter.ts +++ b/js/src/wrappers/vitest-evals/reporter.ts @@ -1,12 +1,12 @@ import type { Reporter, TestCase, TestModule, Vitest } from "vitest/node"; -import { configureNode } from "../../node/config"; +import { SpanTypeAttribute, isObject } from "../../../util"; import { initExperiment, logError, type Experiment, type Span, } from "../../logger"; -import { SpanTypeAttribute, isObject } from "../../../util"; +import { configureNode } from "../../node/config"; import { summarizeAndFlush } from "../shared/flush"; configureNode(); @@ -31,6 +31,7 @@ type EvalScore = { type EvalMeta = { scores?: EvalScore[]; avgScore?: number | null; + input?: unknown; output?: unknown; thresholdFailed?: boolean; toolCalls?: ToolCallRecord[]; @@ -231,6 +232,11 @@ function logEvalTest( const result = test.result(); const diagnostic = test.diagnostic(); const run = meta.harness?.run; + // meta.eval.input is a direct passthrough (mirrors meta.eval.output below) for + // harnesses whose input isn't a conversation, so they don't need to fabricate a + // session.messages entry just to populate this field. Falls back to the first + // user message for harnesses that only report a session. + const input = meta.eval?.input ?? firstUserMessageContent(run); const output = meta.eval?.output ?? run?.output; const scores = buildScores(result.state, meta.eval); const metrics = buildMetrics(diagnostic?.duration, run); @@ -247,7 +253,7 @@ function logEvalTest( event: { input: { test: test.fullName || test.name, - input: firstUserMessageContent(run), + input, }, ...(output !== undefined ? { output } : {}), scores, @@ -519,6 +525,7 @@ function readEvalMeta(input: unknown): EvalMeta | undefined { return { ...(scores ? { scores } : {}), ...(avgScore !== undefined ? { avgScore } : {}), + ...(input.input !== undefined ? { input: input.input } : {}), ...(input.output !== undefined ? { output: input.output } : {}), ...(typeof input.thresholdFailed === "boolean" ? { thresholdFailed: input.thresholdFailed } From 1f6725f20c9a9e3b5c530654c3251fa482ead2dd Mon Sep 17 00:00:00 2001 From: Aislinn Cronin Date: Fri, 31 Jul 2026 17:17:52 -0400 Subject: [PATCH 2/3] docs: name BraintrustVitestEvalsReporter explicitly in the changeset The changeset described the internal shape (eval task meta, harness.run.session.messages) without ever naming the reporter it affects, unlike other changesets in this repo that name the concrete integration in the title. Co-Authored-By: Claude Sonnet 5 --- .changeset/vitest-evals-direct-input.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/vitest-evals-direct-input.md b/.changeset/vitest-evals-direct-input.md index ce0e20aa6..ce0872ae5 100644 --- a/.changeset/vitest-evals-direct-input.md +++ b/.changeset/vitest-evals-direct-input.md @@ -2,7 +2,7 @@ "braintrust": minor --- -feat(vitest-evals): support a direct `input` on eval task meta, mirroring `output` +feat: `BraintrustVitestEvalsReporter` accepts a direct `input` on eval task meta Previously the only way to populate an eval row's `input` field was via `harness.run.session.messages` (the first message with `role: "user"`), which From f6f70b74357cc7026fb4b3538812243cb49e4804 Mon Sep 17 00:00:00 2001 From: Aislinn Cronin Date: Fri, 31 Jul 2026 17:19:34 -0400 Subject: [PATCH 3/3] refactor: drop redundant comment, use test's own domain data The comment restated what the code and PR description already say. Test data used our internal domain fields (memberUuid, payPeriodEnd) instead of this file's existing refund/invoice fixtures. Co-Authored-By: Claude Sonnet 5 --- js/src/wrappers/vitest-evals/reporter.test.ts | 4 ++-- js/src/wrappers/vitest-evals/reporter.ts | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/js/src/wrappers/vitest-evals/reporter.test.ts b/js/src/wrappers/vitest-evals/reporter.test.ts index 18bf9f2ee..7b7385306 100644 --- a/js/src/wrappers/vitest-evals/reporter.test.ts +++ b/js/src/wrappers/vitest-evals/reporter.test.ts @@ -232,7 +232,7 @@ describe("Braintrust vitest-evals reporter", () => { meta: { eval: { avgScore: 1, - input: { memberUuid: "member-1", payPeriodEnd: "2026-06-15" }, + input: { invoiceId: "inv_123", amount: 42 }, output: { status: "approved" }, }, harness: { @@ -268,7 +268,7 @@ describe("Braintrust vitest-evals reporter", () => { (row: any) => row.metadata?.fullName === "structured input", ); expect(structured?.input).toMatchObject({ - input: { memberUuid: "member-1", payPeriodEnd: "2026-06-15" }, + input: { invoiceId: "inv_123", amount: 42 }, }); const fallback = rows.find( diff --git a/js/src/wrappers/vitest-evals/reporter.ts b/js/src/wrappers/vitest-evals/reporter.ts index d882e27ab..71ba251e3 100644 --- a/js/src/wrappers/vitest-evals/reporter.ts +++ b/js/src/wrappers/vitest-evals/reporter.ts @@ -232,10 +232,6 @@ function logEvalTest( const result = test.result(); const diagnostic = test.diagnostic(); const run = meta.harness?.run; - // meta.eval.input is a direct passthrough (mirrors meta.eval.output below) for - // harnesses whose input isn't a conversation, so they don't need to fabricate a - // session.messages entry just to populate this field. Falls back to the first - // user message for harnesses that only report a session. const input = meta.eval?.input ?? firstUserMessageContent(run); const output = meta.eval?.output ?? run?.output; const scores = buildScores(result.state, meta.eval);