diff --git a/src/core/eval.tsx b/src/core/eval.tsx index aef88d11d..74754d66b 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -101,6 +101,7 @@ import type { CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, + CreateOnlineInsightInput, EvaluateInput, EvaluateResult, GetBatchEvaluationResult, @@ -556,6 +557,62 @@ export class EvalClient implements CoreEvalClient { : retryWhileRolePropagates(() => control.send(command)); } + async createOnlineInsight( + input: CreateOnlineInsightInput, + options: CoreOptions, + ): Promise { + const dataSourceConfig = + input.agent !== undefined + ? await agentDataSource(input.agent, input.endpoint, this.clients, options) + : input.dataSourceConfig; + const control = this.clients.control(toClientConfig(options)); + + const command = new CreateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigName: input.name, + description: input.description, + rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), + dataSourceConfig, + insights: input.insightIds.map((insightId) => ({ insightId })), + clusteringConfig: input.clusteringConfig, + evaluationExecutionRoleArn: input.evaluationExecutionRoleArn, + enableOnCreate: input.enableOnCreate ?? true, + }); + + // The role is caller-supplied, so one that cannot be assumed is a real + // misconfiguration — fail fast rather than retry as we do for a role we just + // provisioned ourselves. + return control.send(command); + } + + // Insight configs are the same resource as eval configs, so reads and lifecycle + // reuse the eval methods; only create/update differ (insights vs evaluators). + getOnlineInsight(id: string, options: CoreOptions): Promise { + return this.getOnlineEvaluationConfig(id, options); + } + + listOnlineInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.listOnlineEvaluationConfigs(nextToken, maxResults, options); + } + + setOnlineInsightExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + return this.setOnlineEvaluationExecutionStatus(id, executionStatus, options); + } + + deleteOnlineInsight( + id: string, + options: CoreOptions, + ): Promise { + return this.deleteOnlineEvaluationConfig(id, options); + } + // updateOnlineEvaluationConfig fetches the current config and merges the // provided fields over it, because UpdateOnlineEvaluationConfig replaces the // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index add16936e..2a9737802 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -5,6 +5,7 @@ import type { AppIO } from "../../io"; import type { Core } from "../types"; import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; +import { createOnlineInsightHandler } from "./online-insight"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; import { createOnDemandHandler } from "./ondemand"; @@ -16,6 +17,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .default(renderTui(core, io)) .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)) + .handler(createOnlineInsightHandler(core, io)) .handler(createDatasetHandler(core, io)) .handler(createBatchEvaluationHandler(core, io)) .handler(createOnDemandHandler(core, io)) diff --git a/src/handlers/eval/online-insight/create/index.tsx b/src/handlers/eval/online-insight/create/index.tsx new file mode 100644 index 000000000..59b013bb9 --- /dev/null +++ b/src/handlers/eval/online-insight/create/index.tsx @@ -0,0 +1,135 @@ +import z from "zod"; +import type { DataSourceConfig, Filter } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; + +const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight."; +const ARN_PREFIX = "arn:"; + +export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create an online insight config", + flags: [ + flag("name", "the name of the online insight config", z.string().optional()), + flag( + "execution-role-arn", + "IAM role the online insight assumes (required; not auto-provisioned)", + z.string().optional(), + ), + flag("agent", "harness ID or runtime ID whose traffic to sample", z.string().optional()), + flag( + "endpoint", + "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", + z.string().optional(), + ), + flag( + "data-source-config", + "the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin), as an alternative to --agent", + z.string().optional(), + ), + flag( + "insight", + "insight ID(s) to apply: Builtin.Insight.* identifiers or full ARNs", + z.array(z.string()).optional(), + ), + flag( + "clustering-frequency", + "insight clustering cadence(s): DAILY, WEEKLY, MONTHLY", + z.array(z.enum(["DAILY", "WEEKLY", "MONTHLY"])).optional(), + ), + flag( + "sampling-rate", + "percentage of sessions to sample (0.01-100)", + z.number().min(0.01).max(100).optional(), + ), + flag( + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete (1-1440, default 15)", + z.number().int().min(1).max(1440).optional(), + ), + flag( + "filters", + "trace filters (JSON Filter[]; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to enable evaluation immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + flag( + "description", + "a description of the config's monitoring purpose", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["execution-role-arn"]) + throw new InputValidationError( + "required option '--execution-role-arn ' not specified", + ); + if (!flags["sampling-rate"]) + throw new InputValidationError( + "required option '--sampling-rate ' not specified", + ); + if (!flags["insight"] || flags["insight"].length === 0) + throw new InputValidationError("required option '--insight ' not specified"); + + for (const id of flags["insight"]) { + if (!id.startsWith(BUILTIN_INSIGHT_PREFIX) && !id.startsWith(ARN_PREFIX)) + throw new InputValidationError( + `invalid insight "${id}": must be a ${BUILTIN_INSIGHT_PREFIX}* identifier or a full ARN`, + ); + } + + const hasAgent = flags["agent"] !== undefined; + const hasDataSource = flags["data-source-config"] !== undefined; + if (hasAgent === hasDataSource) + throw new InputValidationError( + "specify exactly one of '--agent' or '--data-source-config'", + ); + if (hasDataSource && flags["endpoint"]) + throw new InputValidationError("'--endpoint' can only be used with '--agent'"); + + const source = new SourceResolver({ stdin: io.stdin }); + const frequencies = flags["clustering-frequency"]; + const common = { + name: flags["name"], + description: flags["description"], + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters: parseJsonFlag( + "filters", + await source.resolveText("filters", flags["filters"]), + ), + insightIds: flags["insight"], + clusteringConfig: frequencies ? { frequencies } : undefined, + evaluationExecutionRoleArn: flags["execution-role-arn"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", + }; + + const response = await core.eval.createOnlineInsight( + hasAgent + ? { ...common, agent: flags["agent"]!, endpoint: flags["endpoint"] } + : { + ...common, + dataSourceConfig: parseJsonFlag( + "data-source-config", + await source.resolveText("data-source-config", flags["data-source-config"]), + )!, + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-insight/delete/index.tsx b/src/handlers/eval/online-insight/delete/index.tsx new file mode 100644 index 000000000..1572cd7b4 --- /dev/null +++ b/src/handlers/eval/online-insight/delete/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteOnlineInsightHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete an online insight config by id", + flags: [flag("id", "the ID of the online insight config to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteOnlineInsight(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/online-insight/get/index.tsx b/src/handlers/eval/online-insight/get/index.tsx new file mode 100644 index 000000000..dc458c1be --- /dev/null +++ b/src/handlers/eval/online-insight/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetOnlineInsightHandler = (core: Core) => + createHandler({ + name: "get", + description: "get an online insight config by id", + flags: [flag("id", "the ID of the online insight config", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getOnlineInsight(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx new file mode 100644 index 000000000..3931ac8ae --- /dev/null +++ b/src/handlers/eval/online-insight/index.tsx @@ -0,0 +1,19 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createCreateOnlineInsightHandler } from "./create"; +import { createGetOnlineInsightHandler } from "./get"; +import { createListOnlineInsightHandler } from "./list"; +import { createPauseOnlineInsightHandler } from "./pause"; +import { createResumeOnlineInsightHandler } from "./resume"; +import { createDeleteOnlineInsightHandler } from "./delete"; + +export function createOnlineInsightHandler(core: Core, io: AppIO): Router { + return new Router("online-insight", "manage AgentCore online insight configs") + .handler(createCreateOnlineInsightHandler(core, io)) + .handler(createGetOnlineInsightHandler(core)) + .handler(createListOnlineInsightHandler(core)) + .handler(createPauseOnlineInsightHandler(core)) + .handler(createResumeOnlineInsightHandler(core)) + .handler(createDeleteOnlineInsightHandler(core)); +} diff --git a/src/handlers/eval/online-insight/list/index.tsx b/src/handlers/eval/online-insight/list/index.tsx new file mode 100644 index 000000000..284fd21d8 --- /dev/null +++ b/src/handlers/eval/online-insight/list/index.tsx @@ -0,0 +1,23 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListOnlineInsightHandler = (core: Core) => + createHandler({ + name: "list", + description: "list online insight configs", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listOnlineInsights( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-insight/online-insight.test.tsx b/src/handlers/eval/online-insight/online-insight.test.tsx new file mode 100644 index 000000000..6d9c1141a --- /dev/null +++ b/src/handlers/eval/online-insight/online-insight.test.tsx @@ -0,0 +1,179 @@ +import { describe, expect, test } from "bun:test"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { InputValidationError } from "../../../errors"; +import { createRootHandler } from "../../index"; + +const INSIGHT = "Builtin.Insight.FailureAnalysis"; +const ROLE = "arn:aws:iam::123456789012:role/myEvalRole"; + +async function run(args: string[]) { + const io = testIO(); + const core = new TestCoreClient(); + const root = createRootHandler(core, { + io: io.io, + globalConfigAccessor: new TestGlobalConfigAccessor(), + logger: createSilentLogger(), + }); + await root.route([ + "node", + "agentcore", + "eval", + "online-insight", + ...args, + "--region", + "us-west-2", + ]); + return { io, core }; +} + +function lastCreate(core: TestCoreClient) { + const call = [...core.eval.calls].reverse().find((c) => c.method === "createOnlineInsight"); + return call?.args[0] as Record | undefined; +} + +describe("eval online-insight create", () => { + test("agent source sets insights + required role", async () => { + const { core } = await run([ + "create", + "--name", + "prodInsights", + "--execution-role-arn", + ROLE, + "--agent", + "runtime-123", + "--insight", + INSIGHT, + "--sampling-rate", + "50", + ]); + expect(lastCreate(core)).toMatchObject({ + name: "prodInsights", + agent: "runtime-123", + insightIds: [INSIGHT], + evaluationExecutionRoleArn: ROLE, + samplingRate: 50, + }); + }); + + test("clustering frequencies flow through", async () => { + const { core } = await run([ + "create", + "--name", + "x", + "--execution-role-arn", + ROLE, + "--agent", + "runtime-123", + "--insight", + INSIGHT, + "--sampling-rate", + "10", + "--clustering-frequency", + "DAILY", + "WEEKLY", + ]); + expect(lastCreate(core)).toMatchObject({ + clusteringConfig: { frequencies: ["DAILY", "WEEKLY"] }, + }); + }); + + test("custom data-source source", async () => { + const { core } = await run([ + "create", + "--name", + "x", + "--execution-role-arn", + ROLE, + "--data-source-config", + '{"cloudWatchLogs":{"logGroupNames":["/aws/foo"],"serviceNames":["svc"]}}', + "--insight", + INSIGHT, + "--sampling-rate", + "10", + ]); + expect(lastCreate(core)).toMatchObject({ + dataSourceConfig: { cloudWatchLogs: { logGroupNames: ["/aws/foo"] } }, + insightIds: [INSIGHT], + }); + }); + + test.each<[string, string[]]>([ + [ + "missing --name", + ["--execution-role-arn", ROLE, "--agent", "r", "--insight", INSIGHT, "--sampling-rate", "10"], + ], + [ + "missing --execution-role-arn", + ["--name", "x", "--agent", "r", "--insight", INSIGHT, "--sampling-rate", "10"], + ], + [ + "missing --sampling-rate", + ["--name", "x", "--execution-role-arn", ROLE, "--agent", "r", "--insight", INSIGHT], + ], + [ + "missing --insight", + ["--name", "x", "--execution-role-arn", ROLE, "--agent", "r", "--sampling-rate", "10"], + ], + [ + "invalid insight id", + [ + "--name", + "x", + "--execution-role-arn", + ROLE, + "--agent", + "r", + "--insight", + "nope", + "--sampling-rate", + "10", + ], + ], + [ + "both agent and data-source", + [ + "--name", + "x", + "--execution-role-arn", + ROLE, + "--agent", + "r", + "--data-source-config", + "{}", + "--insight", + INSIGHT, + "--sampling-rate", + "10", + ], + ], + [ + "neither agent nor data-source", + ["--name", "x", "--execution-role-arn", ROLE, "--insight", INSIGHT, "--sampling-rate", "10"], + ], + ])("rejects: %s", async (_label, args) => { + await expect(run(["create", ...args])).rejects.toBeInstanceOf(InputValidationError); + }); +}); + +describe("eval online-insight read/lifecycle", () => { + test("get / delete / pause / resume call the dedicated client methods", async () => { + const g = await run(["get", "--id", "oi-1"]); + expect(g.core.eval.calls.some((c) => c.method === "getOnlineInsight")).toBe(true); + + const d = await run(["delete", "--id", "oi-1"]); + expect(d.core.eval.calls.some((c) => c.method === "deleteOnlineInsight")).toBe(true); + + const p = await run(["pause", "--id", "oi-1"]); + expect(p.core.eval.calls.some((c) => c.method === "setOnlineInsightExecutionStatus")).toBe( + true, + ); + + const l = await run(["list"]); + expect(l.core.eval.calls.some((c) => c.method === "listOnlineInsights")).toBe(true); + }); +}); diff --git a/src/handlers/eval/online-insight/pause/index.tsx b/src/handlers/eval/online-insight/pause/index.tsx new file mode 100644 index 000000000..7c28ca54f --- /dev/null +++ b/src/handlers/eval/online-insight/pause/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createPauseOnlineInsightHandler = (core: Core) => + createHandler({ + name: "pause", + description: "pause an online insight config", + flags: [flag("id", "the ID of the online insight config to pause", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.setOnlineInsightExecutionStatus( + flags["id"], + "DISABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/online-insight/resume/index.tsx b/src/handlers/eval/online-insight/resume/index.tsx new file mode 100644 index 000000000..985738709 --- /dev/null +++ b/src/handlers/eval/online-insight/resume/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createResumeOnlineInsightHandler = (core: Core) => + createHandler({ + name: "resume", + description: "resume a paused online insight config", + flags: [flag("id", "the ID of the online insight config to resume", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.setOnlineInsightExecutionStatus( + flags["id"], + "ENABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 226c17f81..195b2415b 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -146,6 +146,24 @@ export type CreateOnlineEvalInput = { | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } ); +// CreateOnlineInsightInput mirrors CreateOnlineEvalInput but applies insights +// instead of evaluators and requires a caller-supplied execution role — insight +// configs are never given an auto-provisioned role. +export type CreateOnlineInsightInput = { + name: string; + description?: string; + samplingRate: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + insightIds: string[]; + clusteringConfig?: { frequencies: ("DAILY" | "WEEKLY" | "MONTHLY")[] }; + evaluationExecutionRoleArn: string; + enableOnCreate?: boolean; +} & ( + | { agent: string; endpoint?: string; dataSourceConfig?: undefined } + | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } +); + // UpdateOnlineEvalInput carries the fields a caller may change on an online // evaluation config. Undefined fields are left untouched by Core (merged over // the current config, since UpdateOnlineEvaluationConfig replaces the whole @@ -344,6 +362,29 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; + // Online insight configs are the same underlying OnlineEvaluationConfig resource + // with insights instead of evaluators, so read/lifecycle share the eval methods; + // create applies insights + optional clustering and requires a BYO role. + createOnlineInsight( + input: CreateOnlineInsightInput, + options: CoreOptions, + ): Promise; + getOnlineInsight(id: string, options: CoreOptions): Promise; + listOnlineInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + setOnlineInsightExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise; + deleteOnlineInsight( + id: string, + options: CoreOptions, + ): Promise; + createConfigurationBundle( input: CreateConfigurationBundleInput, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 50047db83..a4495da0c 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -129,6 +129,7 @@ import type { CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, + CreateOnlineInsightInput, DatasetUpdateResult, DatasetUpdateProgressEvent, EvaluateInput, @@ -1787,6 +1788,56 @@ export class TestEvalClient implements CoreEvalClient { return this.onlineEvalDeleteResponse; } + async createOnlineInsight( + input: CreateOnlineInsightInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createOnlineInsight", args: [input, options] }); + if (this.error) throw this.error; + return this.onlineEvalCreateResponse; + } + + async getOnlineInsight( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "getOnlineInsight", args: [id, options] }); + if (this.error) throw this.error; + return this.onlineEvalGetResponse; + } + + async listOnlineInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listOnlineInsights", args: [nextToken, maxResults, options] }); + if (this.error) throw this.error; + return this.onlineEvalListResponses.get(nextToken) ?? { onlineEvaluationConfigs: [] }; + } + + async setOnlineInsightExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + this.calls.push({ + method: "setOnlineInsightExecutionStatus", + args: [id, executionStatus, options], + }); + if (this.error) throw this.error; + return this.onlineEvalUpdateResponse; + } + + async deleteOnlineInsight( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "deleteOnlineInsight", args: [id, options] }); + if (this.error) throw this.error; + return this.onlineEvalDeleteResponse; + } + async createConfigurationBundle( input: CreateConfigurationBundleInput, options: CoreOptions,