diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 2fa05af70..b5e40cc81 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -1,13 +1,22 @@ -import { ConflictError, ResourceNotFoundError, findConfigRoot, serializeResult, toError } from '../../lib'; +import { + ConflictError, + ResourceNotFoundError, + createConfigIO, + findConfigRoot, + serializeResult, + toError, +} from '../../lib'; import type { Result } from '../../lib/result'; import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema'; import { + BASE_EVALUATOR_ID_PATTERN, EvaluationLevelSchema, EvaluatorModelIdSchema, EvaluatorModelProviderSchema, EvaluatorSchema, isValidKmsKeyArn, } from '../../schema'; +import { getEvaluator } from '../aws/agentcore-control'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, SchemaChange } from '../operations/remove/types'; import { runCliCommand } from '../telemetry/cli-command-run.js'; @@ -125,7 +134,9 @@ export interface ThirdPartyLibraryOptions { export interface AddEvaluatorOptions { name: string; - level: EvaluationLevel; + // Required. For a derived evaluator the CLI resolves it from the base metric + // before calling add(); other types take it from --level. + level?: EvaluationLevel; description?: string; config: EvaluatorConfig; kmsKeyArn?: string; @@ -341,10 +352,20 @@ export class EvaluatorPrimitive extends BasePrimitive', 'Evaluator name') - .option('--level ', 'Evaluation level: SESSION, TRACE, TOOL_CALL') - .option('--type ', 'Evaluator type: llm-as-a-judge (default) or code-based') - .option('--model ', '[LLM] Bedrock inference profile ID or OpenResponses model ID for LLM-as-a-Judge') + .option( + '--level ', + 'Evaluation level: SESSION, TRACE, TOOL_CALL (auto-resolved from the base for --type derived)' + ) + .option('--type ', 'Evaluator type: llm-as-a-judge (default), code-based, or derived') + .option( + '--model ', + '[LLM] Bedrock inference profile ID or OpenResponses model ID; [derived] Bedrock inference profile ID for the judge model' + ) .option('--model-provider ', '[LLM] Model provider: Bedrock (default) or OpenResponses') + .option( + '--base-evaluator-id ', + '[derived] Managed base metric to derive from: "ThirdParty.." or "Builtin."' + ) .option( '--instructions ', '[LLM] Evaluation prompt instructions (must include level-appropriate placeholders, e.g. {context})' @@ -373,6 +394,7 @@ export class EvaluatorPrimitive extends BasePrimitive." or "Builtin."' + ); + } + // The service requires the derived evaluator's level to match the base + // metric's level. Resolve it via GetEvaluator so the customer never has + // to know or type it; --level stays available as an offline override. + resolvedLevel ??= await this.resolveBaseEvaluatorLevel(cliOptions.baseEvaluatorId!); + configJson = { + derived: { + baseEvaluatorId: cliOptions.baseEvaluatorId!, + model: cliOptions.model!, + }, + }; + } else if (threePLibrary) { const libraryConfig = THIRD_PARTY_EVALUATOR_LIBRARIES[threePLibrary]; configJson = this.buildThirdPartyConfig(cliOptions.name!, libraryConfig, cliOptions.timeout); thirdParty = { @@ -553,7 +624,7 @@ export class EvaluatorPrimitive extends BasePrimitive `{${p}}`).join(', '); fail( `--instructions is required in non-interactive mode (or use --config). ` + @@ -561,7 +632,7 @@ export class EvaluatorPrimitive extends BasePrimitive { + // A fresh project has no saved deploy targets, so resolve the region directly + // from the environment/profile fallback (env vars, then the AWS profile's region). + const region = await createConfigIO().resolveRegionFallback(); + if (!region) { + throw new Error( + `Could not resolve an AWS region to look up "${baseEvaluatorId}". Set AWS_REGION or pass --level explicitly.` + ); + } + try { + const base = await getEvaluator({ region, evaluatorId: baseEvaluatorId }); + return base.level; + } catch (err) { + throw new Error( + `Could not resolve the level for base evaluator "${baseEvaluatorId}": ${getErrorMessage(err)}. ` + + 'Pass --level explicitly to override.' + ); + } + } + private async createEvaluator(options: AddEvaluatorOptions): Promise { + if (!options.level) { + throw new Error('Evaluation level is required (SESSION, TRACE, or TOOL_CALL)'); + } + const evaluator: Evaluator = { name: options.name, level: options.level, diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index 071df8956..b8c4ad3ec 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -38,7 +38,7 @@ export const CredentialType = z.enum(['api-key', 'oauth']); // Mirrors DependencySyncOutcome in src/lib/dependency-management/types.ts. export const DepSyncOutcome = z.enum(['synced', 'check-only', 'opted-out', 'skipped', 'failure-suppressed', 'failed']); export const SkillSourceType = z.enum(['path', 's3', 'git', 'aws_skills']); -export const EvaluatorType = z.enum(['llm-as-a-judge', 'code-based']); +export const EvaluatorType = z.enum(['llm-as-a-judge', 'code-based', 'derived']); export const EvaluatorModelProvider = z.enum(['bedrock', 'openresponses']); export const ExitReason = z.enum(['success', 'failure']); export const FilterState = z.enum(['deployed', 'local-only', 'pending-removal', 'none']); diff --git a/src/lib/schemas/io/config-io.ts b/src/lib/schemas/io/config-io.ts index dea1b35f4..f708fed7d 100644 --- a/src/lib/schemas/io/config-io.ts +++ b/src/lib/schemas/io/config-io.ts @@ -201,8 +201,10 @@ export class ConfigIO { /** * Resolve a fallback region from environment variables or AWS profile config. + * Public so callers that need a region before any deploy target is saved (e.g. + * resolving a derived evaluator's level) can reuse the same precedence. */ - private async resolveRegionFallback(): Promise { + async resolveRegionFallback(): Promise { // Check env vars first const envRegion = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION; if (envRegion && AgentCoreRegionSchema.safeParse(envRegion).success) { diff --git a/src/schema/llm-compacted/agentcore.ts b/src/schema/llm-compacted/agentcore.ts index c2915ea8d..2ff15c6df 100644 --- a/src/schema/llm-compacted/agentcore.ts +++ b/src/schema/llm-compacted/agentcore.ts @@ -280,11 +280,25 @@ interface Evaluator { name: string; // @regex ^[a-zA-Z][a-zA-Z0-9_]{0,47}$ @min 1 @max 48 level: 'SESSION' | 'TRACE' | 'TOOL_CALL'; description?: string; - config: { llmAsAJudge: LlmAsAJudgeConfig; codeBased?: never } | { llmAsAJudge?: never; codeBased: CodeBasedConfig }; + config: EvaluatorConfig; // exactly one of llmAsAJudge | codeBased | derived kmsKeyArn?: string; tags?: Tags; } +// Exactly one arm present. +type EvaluatorConfig = + | { llmAsAJudge: LlmAsAJudgeConfig; codeBased?: never; derived?: never } + | { llmAsAJudge?: never; codeBased: CodeBasedConfig; derived?: never } + | { llmAsAJudge?: never; codeBased?: never; derived: DerivedEvaluatorConfig }; + +// A derived evaluator reuses a managed base evaluator's logic (a Builtin.* or +// ThirdParty.. metric) on the customer's own model. The base +// owns the prompt + scoring; level must match the base's (resolved at add time). +interface DerivedEvaluatorConfig { + baseEvaluatorId: string; // "Builtin." or "ThirdParty.." + model: string; // Bedrock inference profile ID +} + interface LlmAsAJudgeConfig { modelProvider?: 'Bedrock' | 'OpenResponses'; // Defaults to Bedrock when omitted model: string; // Bedrock model ID/ARN or OpenAI model ID diff --git a/src/schema/schemas/agentcore-project.ts b/src/schema/schemas/agentcore-project.ts index c1eaa46fd..f45c84c2a 100644 --- a/src/schema/schemas/agentcore-project.ts +++ b/src/schema/schemas/agentcore-project.ts @@ -67,12 +67,14 @@ export type { RatingScale, } from './primitives/evaluator'; export { + BASE_EVALUATOR_ID_PATTERN, BedrockModelIdSchema, isValidBedrockModelId, EvaluatorNameSchema, KMS_KEY_ARN_PATTERN, isValidKmsKeyArn, } from './primitives/evaluator'; +export type { DerivedEvaluatorConfig } from './primitives/evaluator'; export { ConfigBundleSchema }; export type { ComponentConfiguration, ComponentConfigurationMap, ConfigBundle } from './primitives/config-bundle'; export { ConfigBundleNameSchema, ComponentConfigurationMapSchema } from './primitives/config-bundle'; @@ -360,6 +362,8 @@ export type EvaluatorType = z.infer; export const EvaluatorSchema = z.object({ name: EvaluatorNameSchema, + // Required for every evaluator. For a derived evaluator it must match the base + // metric's level; the CLI resolves it via GetEvaluator at add time. level: EvaluationLevelSchema, description: z.string().optional(), config: EvaluatorConfigSchema, diff --git a/src/schema/schemas/primitives/evaluator.ts b/src/schema/schemas/primitives/evaluator.ts index 1df0c1295..05a75f523 100644 --- a/src/schema/schemas/primitives/evaluator.ts +++ b/src/schema/schemas/primitives/evaluator.ts @@ -85,6 +85,34 @@ export const LlmAsAJudgeConfigSchema = z.object({ export type LlmAsAJudgeConfig = z.infer; +// ============================================================================ +// Derived Evaluator Config +// ============================================================================ + +// A derived evaluator reuses a managed base evaluator's logic (prompt + scoring) +// and runs it on the customer's own model. The base is a managed metric — a +// third-party library metric (ThirdParty..) or a built-in +// (Builtin.). The base owns the prompt and scoring; the customer supplies +// only the model. The evaluator's `level` must match the base's level (resolved at +// add time via GetEvaluator), so it lives on the top-level Evaluator, not here. +// Builtin. or ThirdParty... Each segment must be +// non-empty alphanumeric — rejects malformed ids like "ThirdParty.DeepEval", +// "ThirdParty..ToolUse", or "ThirdParty.DeepEval.". +export const BASE_EVALUATOR_ID_PATTERN = /^(Builtin\.[A-Za-z0-9]+|ThirdParty\.[A-Za-z0-9]+\.[A-Za-z0-9]+)$/; + +export const DerivedEvaluatorConfigSchema = z.object({ + baseEvaluatorId: z + .string() + .min(1) + .regex( + BASE_EVALUATOR_ID_PATTERN, + 'Must be a managed base id: "ThirdParty.." or "Builtin."' + ), + model: BedrockModelIdSchema, +}); + +export type DerivedEvaluatorConfig = z.infer; + // ============================================================================ // Code-Based Evaluator Config // ============================================================================ @@ -125,9 +153,16 @@ export const EvaluatorConfigSchema = z .object({ llmAsAJudge: LlmAsAJudgeConfigSchema.optional(), codeBased: CodeBasedConfigSchema.optional(), + derived: DerivedEvaluatorConfigSchema.optional(), }) - .refine(config => Boolean(config.llmAsAJudge) !== Boolean(config.codeBased), { - message: 'Config must have either llmAsAJudge or codeBased, not both', + .superRefine((config, ctx) => { + const arms = [config.llmAsAJudge, config.codeBased, config.derived].filter(Boolean).length; + if (arms !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Config must have exactly one of llmAsAJudge, codeBased, or derived', + }); + } }); export type EvaluatorConfig = z.infer;