diff --git a/.changeset/vitest-evals-direct-input.md b/.changeset/vitest-evals-direct-input.md new file mode 100644 index 000000000..ce0872ae5 --- /dev/null +++ b/.changeset/vitest-evals-direct-input.md @@ -0,0 +1,13 @@ +--- +"braintrust": minor +--- + +feat: `BraintrustVitestEvalsReporter` accepts a direct `input` on eval task meta + +Previously the only way to populate an eval row's `input` field was via +`harness.run.session.messages` (the first message with `role: "user"`), which +forces non-conversational harnesses to fabricate a synthetic message just to +surface their real input. `meta.eval.input` now works exactly like the +existing `meta.eval.output`: a direct passthrough that takes precedence over +`session.messages` when present, falling back to the previous behavior when +it's not set. diff --git a/js/src/wrappers/vitest-evals/reporter.test.ts b/js/src/wrappers/vitest-evals/reporter.test.ts index 23075f5d1..7b7385306 100644 --- a/js/src/wrappers/vitest-evals/reporter.test.ts +++ b/js/src/wrappers/vitest-evals/reporter.test.ts @@ -1,11 +1,11 @@ import { beforeAll, beforeEach, describe, expect, test, vi } from "vitest"; import BraintrustVitestEvalsReporter from "./reporter"; -import { configureNode } from "../../node/config"; import { _exportsForTestingOnly, type TestBackgroundLogger, } from "../../logger"; import * as logger from "../../logger"; +import { configureNode } from "../../node/config"; configureNode(); @@ -220,6 +220,65 @@ describe("Braintrust vitest-evals reporter", () => { ); }); + test("meta.eval.input is a direct passthrough that takes precedence over session.messages", async () => { + const reporter = new BraintrustVitestEvalsReporter({ + displaySummary: false, + projectName: "vitest-evals-tests", + }); + + await reporter.onTestRunEnd([ + fakeModule([ + fakeTest({ + meta: { + eval: { + avgScore: 1, + input: { invoiceId: "inv_123", amount: 42 }, + output: { status: "approved" }, + }, + harness: { + run: { + session: { + messages: [ + { + role: "user", + content: "ignored in favor of meta.eval.input", + }, + ], + }, + usage: {}, + }, + }, + }, + name: "structured input", + }), + fakeTest({ + meta: { + eval: { avgScore: 1 }, + harness: { run: { session: { messages: [] }, usage: {} } }, + }, + name: "falls back to session when meta.eval.input is absent", + }), + ]), + ] as any); + + await backgroundLogger.flush(); + const rows = (await backgroundLogger.drain()) as any[]; + + const structured = rows.find( + (row: any) => row.metadata?.fullName === "structured input", + ); + expect(structured?.input).toMatchObject({ + input: { invoiceId: "inv_123", amount: 42 }, + }); + + const fallback = rows.find( + (row: any) => + row.metadata?.fullName === + "falls back to session when meta.eval.input is absent", + ); + expect(fallback?.input?.input).toBeUndefined(); + }); + test("logs failed eval scores and failure metadata", async () => { const reporter = new BraintrustVitestEvalsReporter({ displaySummary: false, diff --git a/js/src/wrappers/vitest-evals/reporter.ts b/js/src/wrappers/vitest-evals/reporter.ts index 357f51506..71ba251e3 100644 --- a/js/src/wrappers/vitest-evals/reporter.ts +++ b/js/src/wrappers/vitest-evals/reporter.ts @@ -1,12 +1,12 @@ import type { Reporter, TestCase, TestModule, Vitest } from "vitest/node"; -import { configureNode } from "../../node/config"; +import { SpanTypeAttribute, isObject } from "../../../util"; import { initExperiment, logError, type Experiment, type Span, } from "../../logger"; -import { SpanTypeAttribute, isObject } from "../../../util"; +import { configureNode } from "../../node/config"; import { summarizeAndFlush } from "../shared/flush"; configureNode(); @@ -31,6 +31,7 @@ type EvalScore = { type EvalMeta = { scores?: EvalScore[]; avgScore?: number | null; + input?: unknown; output?: unknown; thresholdFailed?: boolean; toolCalls?: ToolCallRecord[]; @@ -231,6 +232,7 @@ function logEvalTest( const result = test.result(); const diagnostic = test.diagnostic(); const run = meta.harness?.run; + const input = meta.eval?.input ?? firstUserMessageContent(run); const output = meta.eval?.output ?? run?.output; const scores = buildScores(result.state, meta.eval); const metrics = buildMetrics(diagnostic?.duration, run); @@ -247,7 +249,7 @@ function logEvalTest( event: { input: { test: test.fullName || test.name, - input: firstUserMessageContent(run), + input, }, ...(output !== undefined ? { output } : {}), scores, @@ -519,6 +521,7 @@ function readEvalMeta(input: unknown): EvalMeta | undefined { return { ...(scores ? { scores } : {}), ...(avgScore !== undefined ? { avgScore } : {}), + ...(input.input !== undefined ? { input: input.input } : {}), ...(input.output !== undefined ? { output: input.output } : {}), ...(typeof input.thresholdFailed === "boolean" ? { thresholdFailed: input.thresholdFailed }