feat(eval): add batch-evaluation simulate (dataset simulations)

src/core/eval/dataset/__snapshots__/dataset.test.ts.snap

@@ -0,0 +1,35 @@
+// Bun Snapshot v1, https://bun.sh/docs/test/snapshots
+
+exports[`PredefinedExample ground truth maps to the expected inline shape [golden] 1`] = `
+{
+  "assertions": [
+    {
+      "text": "stays polite",
+    },
+    {
+      "text": "does not promise a date",
+    },
+  ],
+  "expectedTrajectory": {
+    "toolNames": [
+      "refund_lookup",
+      "refund_create",
+    ],
+  },
+  "turns": [
+    {
+      "input": {
+        "prompt": "I want a refund",
+      },
+    },
+    {
+      "expectedResponse": {
+        "text": "Refund started",
+      },
+      "input": {
+        "prompt": "order 123",
+      },
+    },
+  ],
+}
+`;

src/core/eval/dataset/dataset.test.ts

@@ -0,0 +1,191 @@
+import { test, expect, describe } from "bun:test";
+import { DatasetLoader } from "./load";
+import { PredefinedExample } from "./predefined";
+import { SimulatedExample } from "./simulated";
+import type { RunContext, TurnResult } from "./types";
+
+const row = (o: object) => JSON.stringify(o);
+
+// A fake transport that records what was said and returns a canned reply. No AWS.
+function recordingCtx(reply = ""): { ctx: RunContext; calls: string[] } {
+  const calls: string[] = [];
+  const ctx: RunContext = {
+    invokeOnce: async (payload): Promise<TurnResult> => {
+      calls.push(payload);
+      return { text: reply };
+    },
+  };
+  return { ctx, calls };
+}
+
+describe("DatasetLoader.load", () => {
+  test("builds a PredefinedExample from a turns row", () => {
+    const [e] = DatasetLoader.load(row({ example_id: "x", turns: [{ input: "hi" }] }));
+    expect(e).toBeInstanceOf(PredefinedExample);
+    expect(e!.schemaType).toBe("AGENTCORE_EVALUATION_PREDEFINED_V1");
+    expect(e!.exampleId).toBe("x");
+  });
+
+  test("accepts the legacy scenario_id as the example id", () => {
+    const [e] = DatasetLoader.load(row({ scenario_id: "legacy", turns: [{ input: "hi" }] }));
+    expect(e!.exampleId).toBe("legacy");
+  });
+
+  test("refuses a row that is both predefined and simulated", () => {
+    expect(() =>
+      DatasetLoader.load(row({ example_id: "x", turns: [{ input: "a" }], actor_profile: {} })),
+    ).toThrow(/both 'turns' and 'actor_profile'/);
+  });
+
+  test("refuses a row that is neither", () => {
+    expect(() => DatasetLoader.load(row({ example_id: "x" }))).toThrow(
+      /neither 'turns' nor 'actor_profile'/,
+    );
+  });
+
+  test("names a simulated row instead of blaming the data", () => {
+    expect(() =>
+      DatasetLoader.load(row({ example_id: "x", actor_profile: { goal: "g" } })),
+    ).toThrow(/simulated example/);
+  });
+
+  test("rejects duplicate example ids", () => {
+    const two = [
+      row({ example_id: "a", turns: [{ input: "1" }] }),
+      row({ example_id: "a", turns: [{ input: "2" }] }),
+    ].join("\n");
+    expect(() => DatasetLoader.load(two)).toThrow(/duplicate example_id: "a"/);
+  });
+
+  test("rejects a missing example id", () => {
+    expect(() => DatasetLoader.load(row({ turns: [{ input: "hi" }] }))).toThrow(
+      /missing 'example_id'/,
+    );
+  });
+
+  test("rejects an invalid JSON line", () => {
+    expect(() => DatasetLoader.load("{not json")).toThrow(/not valid JSON/);
+  });
+
+  test("rejects an empty dataset", () => {
+    expect(() => DatasetLoader.load("\n  \n")).toThrow(/no examples/);
+  });
+
+  test("ignores blank lines between rows", () => {
+    const examples = DatasetLoader.load(
+      [
+        row({ example_id: "a", turns: [{ input: "1" }] }),
+        "",
+        row({ example_id: "b", turns: [{ input: "2" }] }),
+      ].join("\n"),
+    );
+    expect(examples.map((e) => e.exampleId)).toEqual(["a", "b"]);
+  });
+
+  // `null` is valid JSON but has no fields; dereferencing it once threw a raw TypeError
+  // instead of a clean validation error.
+  test.each([["null"], ["[1,2,3]"], ["42"], ['"hi"'], ["true"]])(
+    "rejects a non-object row (%s) with a clear error",
+    (line) => {
+      expect(() => DatasetLoader.load(line)).toThrow(/not a JSON object/);
+    },
+  );
+
+  test("handles CRLF line endings", () => {
+    const crlf =
+      row({ example_id: "a", turns: [{ input: "1" }] }) +
+      "\r\n" +
+      row({ example_id: "b", turns: [{ input: "2" }] }) +
+      "\r\n";
+    expect(DatasetLoader.load(crlf).map((e) => e.exampleId)).toEqual(["a", "b"]);
+  });
+
+  test("preserves unicode example ids", () => {
+    const [e] = DatasetLoader.load(row({ example_id: "café-日本-🎉", turns: [{ input: "1" }] }));
+    expect(e!.exampleId).toBe("café-日本-🎉");
+  });
+});
+
+describe("SimulatedExample", () => {
+  test("construction throws NotImplementedError (never replayed)", () => {
+    expect(() => new SimulatedExample("x", { actor_profile: {} })).toThrow(/cannot replay yet/);
+  });
+});
+
+describe("PredefinedExample", () => {
+  test("constructor rejects a row with no turns", () => {
+    expect(() => new PredefinedExample("x", { turns: [] })).toThrow(/has no turns/);
+  });
+
+  test("constructor rejects a non-object turn entry", () => {
+    expect(() => new PredefinedExample("x", { turns: [null] })).toThrow(/turn 1 is not an object/);
+  });
+
+  test("omits empty assertions and expected_trajectory arrays", async () => {
+    const { ctx } = recordingCtx();
+    // Only the expectation-bearing turn should survive to ground truth; the empty
+    // assertions/trajectory arrays are dropped (the service rejects zero-length ones).
+    const gt = await new PredefinedExample("x", {
+      turns: [{ input: "t1", expected_response: "r1" }],
+      assertions: [],
+      expected_trajectory: [],
+    }).run(ctx);
+    expect(gt).toBeDefined();
+    expect(gt!.assertions).toBeUndefined();
+    expect(gt!.expectedTrajectory).toBeUndefined();
+    expect(gt!.turns).toHaveLength(1);
+  });
+
+  // A deliberate `expected_response: ""` means "expect an empty reply" — distinct from
+  // omitting the field. The `!== undefined` guard honors it: the turn carries
+  // expectedResponse { text: "" } rather than being treated as expectation-less.
+  test('treats expected_response "" as a real expectation', async () => {
+    const { ctx } = recordingCtx();
+    const gt = await new PredefinedExample("x", {
+      turns: [{ input: "t1", expected_response: "" }],
+    }).run(ctx);
+    expect(gt!.turns).toHaveLength(1);
+    expect(gt!.turns![0]!.expectedResponse).toEqual({ text: "" });
+  });
+
+  test("run replays every turn in order, on one session", async () => {
+    const { ctx, calls } = recordingCtx();
+    await new PredefinedExample("x", { turns: [{ input: "a" }, { input: "b" }] }).run(ctx);
+    expect(calls).toEqual(["a", "b"]);
+  });
+
+  test("sparse expectations keep their turn position", async () => {
+    const { ctx } = recordingCtx();
+    const gt = await new PredefinedExample("x", {
+      turns: [{ input: "t1" }, { input: "t2" }, { input: "t3", expected_response: "42" }],
+    }).run(ctx);
+    // Not 1 — filtering the two expectation-less turns would renumber the rest and score
+    // turn 3's "42" against turn 1.
+    expect(gt!.turns).toHaveLength(3);
+    expect(gt!.turns![2]!.expectedResponse).toEqual({ text: "42" });
+    expect(gt!.turns![0]!.input).toEqual({ prompt: "t1" });
+    expect(gt!.turns![0]!.expectedResponse).toBeUndefined();
+  });
+
+  test("an example with no ground truth returns undefined", async () => {
+    const { ctx } = recordingCtx();
+    const gt = await new PredefinedExample("x", { turns: [{ input: "t1" }] }).run(ctx);
+    expect(gt).toBeUndefined();
+  });
+
+  // Golden: the full inline ground-truth shape for a representative example (assertions +
+  // trajectory + sparse turns). Locks the exact wire shape handed to the grader so a
+  // regression in the mapping is caught, not just its parts.
+  test("ground truth maps to the expected inline shape [golden]", async () => {
+    const { ctx } = recordingCtx();
+    const gt = await new PredefinedExample("orders-1", {
+      turns: [
+        { input: "I want a refund" },
+        { input: "order 123", expected_response: "Refund started" },
+      ],
+      assertions: ["stays polite", "does not promise a date"],
+      expected_trajectory: ["refund_lookup", "refund_create"],
+    }).run(ctx);
+    expect(gt).toMatchSnapshot();
+  });
+});

src/core/eval/dataset/load.ts

@@ -0,0 +1,71 @@
+import type { DatasetSchemaType } from "@aws-sdk/client-bedrock-agentcore-control";
+import { InputValidationError } from "../../../errors";
+import type { Example } from "./types";
+import { PredefinedExample } from "./predefined";
+import { SimulatedExample } from "./simulated";
+
+// Pure parse (no I/O) so it's testable with a plain string; the caller fetches the text.
+export class DatasetLoader {
+  static load(text: string): Example[] {
+    const examples: Example[] = [];
+    const seen = new Set<string>();
+    for (const line of text.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(trimmed);
+      } catch {
+        throw new InputValidationError("dataset contains a line that is not valid JSON");
+      }
+      // Reject non-object rows before dereferencing — `null` is valid JSON and would throw.
+      if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        throw new InputValidationError("dataset contains a line that is not a JSON object");
+      }
+      const row = parsed as Record<string, unknown>;
+
+      // The id joins a session to its ground truth — a missing/duplicate one misassigns it.
+      const exampleId = String(row.example_id ?? row.scenario_id ?? "");
+      if (!exampleId) {
+        throw new InputValidationError("dataset example is missing 'example_id'");
+      }
+      if (seen.has(exampleId)) {
+        throw new InputValidationError(`dataset has a duplicate example_id: "${exampleId}"`);
+      }
+      seen.add(exampleId);
+
+      examples.push(DatasetLoader.build(row, exampleId));
+    }
+    if (examples.length === 0) throw new InputValidationError("dataset has no examples");
+    return examples;
+  }
+
+  // Classify by row shape — a local JSONL carries no schemaType, and AWS's own SDK
+  // dispatches this way. Refuse a both-row rather than silently dropping the actor profile.
+  private static build(row: Record<string, unknown>, exampleId: string): Example {
+    const hasTurns = Array.isArray(row.turns);
+    const hasActor = row.actor_profile != null;
+    if (hasTurns && hasActor) {
+      throw new InputValidationError(
+        `example "${exampleId}" has both 'turns' and 'actor_profile' — one row cannot be both`,
+      );
+    }
+    if (!hasTurns && !hasActor) {
+      throw new InputValidationError(
+        `example "${exampleId}" has neither 'turns' nor 'actor_profile'`,
+      );
+    }
+
+    // Typed as the full SDK enum so a new member makes the switch non-exhaustive and
+    // build fails to compile (TS2366) — the build guard, no map or assertNever needed.
+    const schemaType: DatasetSchemaType = hasTurns
+      ? "AGENTCORE_EVALUATION_PREDEFINED_V1"
+      : "AGENTCORE_EVALUATION_SIMULATED_V1";
+    switch (schemaType) {
+      case "AGENTCORE_EVALUATION_PREDEFINED_V1":
+        return new PredefinedExample(exampleId, row);
+      case "AGENTCORE_EVALUATION_SIMULATED_V1":
+        return new SimulatedExample(exampleId, row);
+    }
+  }
+}

src/core/eval/dataset/predefined.ts

@@ -0,0 +1,66 @@
+import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore";
+import { InputValidationError } from "../../../errors";
+import type { Example, RunContext } from "./types";
+
+type Turn = { input: string; expectedResponse?: string };
+
+export class PredefinedExample implements Example {
+  readonly schemaType = "AGENTCORE_EVALUATION_PREDEFINED_V1" as const;
+  readonly turns: Turn[];
+  readonly assertions?: string[];
+  readonly expectedTrajectory?: string[];
+
+  constructor(
+    readonly exampleId: string,
+    row: Record<string, unknown>,
+  ) {
+    const turns = (Array.isArray(row.turns) ? row.turns : []).map((t: unknown, i: number) => {
+      // Reject a non-object entry here; dereferencing it below would throw a raw TypeError.
+      if (typeof t !== "object" || t === null) {
+        throw new InputValidationError(`example "${exampleId}" turn ${i + 1} is not an object`);
+      }
+      const turn = t as Record<string, unknown>;
+      return {
+        input: String(turn.input ?? ""),
+        expectedResponse: turn.expected_response as string | undefined,
+      };
+    });
+    if (turns.length === 0) {
+      throw new InputValidationError(`example "${exampleId}" has no turns`);
+    }
+    this.turns = turns;
+    this.assertions = row.assertions as string[] | undefined;
+    this.expectedTrajectory = row.expected_trajectory as string[] | undefined;
+  }
+
+  // Sequential and awaited: the turns share one session, so racing them would interleave
+  // the conversation and misalign per-turn traces with the ground truth.
+  async run(ctx: RunContext): Promise<InlineGroundTruth | undefined> {
+    for (const turn of this.turns) await ctx.invokeOnce(turn.input);
+    return this.groundTruth();
+  }
+
+  // Emit every turn (carrying its prompt), not just those with an expectation: filtering
+  // renumbers the rest, scoring turn 3's expectation against turn 1. The service's
+  // alignment rule is undocumented, so the prompt keeps index and content matching both valid.
+  private groundTruth(): InlineGroundTruth | undefined {
+    const turns = this.turns.some((t) => t.expectedResponse !== undefined)
+      ? this.turns.map((t) => ({
+          input: { prompt: t.input },
+          ...(t.expectedResponse !== undefined && {
+            expectedResponse: { text: t.expectedResponse },
+          }),
+        }))
+      : [];
+    const assertions = this.assertions?.map((text) => ({ text }));
+    const inline: InlineGroundTruth = {
+      // Omit empty arrays — the service rejects zero-length assertions/turns (min-1).
+      ...(assertions && assertions.length > 0 && { assertions }),
+      ...(this.expectedTrajectory?.length && {
+        expectedTrajectory: { toolNames: this.expectedTrajectory },
+      }),
+      ...(turns.length > 0 && { turns }),
+    };
+    return Object.keys(inline).length > 0 ? inline : undefined;
+  }
+}