feat(evaluator): add derived evaluators (reuse a base metric on your own model)

src/cli/primitives/EvaluatorPrimitive.ts

@@ -1,13 +1,22 @@
-import { ConflictError, ResourceNotFoundError, findConfigRoot, serializeResult, toError } from '../../lib';
+import {
+  ConflictError,
+  ResourceNotFoundError,
+  createConfigIO,
+  findConfigRoot,
+  serializeResult,
+  toError,
+} from '../../lib';
 import type { Result } from '../../lib/result';
 import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
 import {
+  BASE_EVALUATOR_ID_PATTERN,
   EvaluationLevelSchema,
   EvaluatorModelIdSchema,
   EvaluatorModelProviderSchema,
   EvaluatorSchema,
   isValidKmsKeyArn,
 } from '../../schema';
+import { getEvaluator } from '../aws/agentcore-control';
 import { getErrorMessage } from '../errors';
 import type { RemovalPreview, SchemaChange } from '../operations/remove/types';
 import { runCliCommand } from '../telemetry/cli-command-run.js';
@@ -125,7 +134,9 @@ export interface ThirdPartyLibraryOptions {
 
 export interface AddEvaluatorOptions {
   name: string;
-  level: EvaluationLevel;
+  // Required. For a derived evaluator the CLI resolves it from the base metric
+  // before calling add(); other types take it from --level.
+  level?: EvaluationLevel;
   description?: string;
   config: EvaluatorConfig;
   kmsKeyArn?: string;
@@ -341,10 +352,20 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
       .command(this.kind)
       .description('Add a custom evaluator to the project')
       .option('--name <name>', 'Evaluator name')
-      .option('--level <level>', 'Evaluation level: SESSION, TRACE, TOOL_CALL')
-      .option('--type <type>', 'Evaluator type: llm-as-a-judge (default) or code-based')
-      .option('--model <model>', '[LLM] Bedrock inference profile ID or OpenResponses model ID for LLM-as-a-Judge')
+      .option(
+        '--level <level>',
+        'Evaluation level: SESSION, TRACE, TOOL_CALL (auto-resolved from the base for --type derived)'
+      )
+      .option('--type <type>', 'Evaluator type: llm-as-a-judge (default), code-based, or derived')
+      .option(
+        '--model <model>',
+        '[LLM] Bedrock inference profile ID or OpenResponses model ID; [derived] Bedrock inference profile ID for the judge model'
+      )
       .option('--model-provider <provider>', '[LLM] Model provider: Bedrock (default) or OpenResponses')
+      .option(
+        '--base-evaluator-id <id>',
+        '[derived] Managed base metric to derive from: "ThirdParty.<Provider>.<Metric>" or "Builtin.<Metric>"'
+      )
       .option(
         '--instructions <text>',
         '[LLM] Evaluation prompt instructions (must include level-appropriate placeholders, e.g. {context})'
@@ -373,6 +394,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
           type?: string;
           model?: string;
           modelProvider?: string;
+          baseEvaluatorId?: string;
           instructions?: string;
           ratingScale?: string;
           lambdaArn?: string;
@@ -394,12 +416,19 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
                 throw new Error(error);
               };
 
-              if (!cliOptions.name || !cliOptions.level) {
-                fail('--name and --level are required in non-interactive mode');
+              // A derived evaluator resolves its level from the base metric, so
+              // --level is optional (an offline override); required otherwise.
+              const isDerived = cliOptions.type === 'derived' || cliOptions.baseEvaluatorId !== undefined;
+
+              if (!cliOptions.name) {
+                fail('--name is required in non-interactive mode');
+              }
+              if (!isDerived && !cliOptions.level) {
+                fail('--level is required in non-interactive mode');
               }
 
-              const levelResult = EvaluationLevelSchema.safeParse(cliOptions.level);
-              if (!levelResult.success) {
+              const levelResult = cliOptions.level ? EvaluationLevelSchema.safeParse(cliOptions.level) : undefined;
+              if (levelResult && !levelResult.success) {
                 fail(`Invalid --level "${cliOptions.level}". Must be one of: SESSION, TRACE, TOOL_CALL`);
               }
 
@@ -490,10 +519,12 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
                 }
               }
 
-              // Default --type to code-based when 3P template is provided
-              const evalType = cliOptions.type ?? (threePLibrary ? 'code-based' : 'llm-as-a-judge');
-              if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based') {
-                fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based`);
+              // Default --type: derived when a base id is given, else code-based when a
+              // 3P (code) template is provided, else llm-as-a-judge.
+              const evalType =
+                cliOptions.type ?? (isDerived ? 'derived' : threePLibrary ? 'code-based' : 'llm-as-a-judge');
+              if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based' && evalType !== 'derived') {
+                fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based, derived`);
               }
 
               // Cross-validate flags against evaluator type
@@ -502,12 +533,21 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
                 if (cliOptions.timeout) fail('--timeout requires --type code-based');
                 if (threePLibrary) fail('--3p-template-json requires --type code-based');
               }
+              if (evalType !== 'derived' && cliOptions.baseEvaluatorId) {
+                fail('--base-evaluator-id requires --type derived');
+              }
               if (evalType === 'code-based') {
                 if (cliOptions.model) fail('--model cannot be used with --type code-based');
                 if (cliOptions.modelProvider) fail('--model-provider cannot be used with --type code-based');
                 if (cliOptions.instructions) fail('--instructions cannot be used with --type code-based');
                 if (cliOptions.ratingScale) fail('--rating-scale cannot be used with --type code-based');
               }
+              if (evalType === 'derived') {
+                // The base metric owns the prompt and scale.
+                if (cliOptions.instructions) fail('--instructions cannot be used with --type derived');
+                if (cliOptions.ratingScale) fail('--rating-scale cannot be used with --type derived');
+                if (cliOptions.modelProvider) fail('--model-provider cannot be used with --type derived');
+              }
               if (cliOptions.config && cliOptions.modelProvider) {
                 fail(
                   '--model-provider cannot be used with --config; set config.llmAsAJudge.modelProvider in the config file'
@@ -516,8 +556,39 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 
               let configJson: EvaluatorConfig;
               let thirdParty: ThirdPartyLibraryOptions | undefined;
-
-              if (threePLibrary) {
+              // For derived, the level is resolved from the base metric (or --level override).
+              let resolvedLevel: EvaluationLevel | undefined = levelResult?.data;
+
+              if (evalType === 'derived') {
+                // --config carries a full evaluator config for other types; a derived
+                // evaluator is built from --base-evaluator-id + --model, so reject
+                // --config here rather than silently ignoring it.
+                if (cliOptions.config) {
+                  fail('--config is not supported with --type derived; use --base-evaluator-id and --model');
+                }
+                if (!cliOptions.baseEvaluatorId) {
+                  fail('--base-evaluator-id is required for --type derived');
+                }
+                if (!cliOptions.model) {
+                  fail('--model is required for --type derived (you bring the judge model)');
+                }
+                if (!BASE_EVALUATOR_ID_PATTERN.test(cliOptions.baseEvaluatorId!)) {
+                  fail(
+                    `Invalid --base-evaluator-id "${cliOptions.baseEvaluatorId}". ` +
+                      'Must be "ThirdParty.<Provider>.<Metric>" or "Builtin.<Metric>"'
+                  );
+                }
+                // The service requires the derived evaluator's level to match the base
+                // metric's level. Resolve it via GetEvaluator so the customer never has
+                // to know or type it; --level stays available as an offline override.
+                resolvedLevel ??= await this.resolveBaseEvaluatorLevel(cliOptions.baseEvaluatorId!);
+                configJson = {
+                  derived: {
+                    baseEvaluatorId: cliOptions.baseEvaluatorId!,
+                    model: cliOptions.model!,
+                  },
+                };
+              } else if (threePLibrary) {
                 const libraryConfig = THIRD_PARTY_EVALUATOR_LIBRARIES[threePLibrary];
                 configJson = this.buildThirdPartyConfig(cliOptions.name!, libraryConfig, cliOptions.timeout);
                 thirdParty = {
@@ -553,15 +624,15 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
                 const modelProvider = modelProviderResult.data!;
 
                 if (!cliOptions.instructions) {
-                  const level = levelResult.data!;
+                  const level = levelResult!.data!;
                   const placeholders = LEVEL_PLACEHOLDERS[level].map(p => `{${p}}`).join(', ');
                   fail(
                     `--instructions is required in non-interactive mode (or use --config). ` +
                       `Must include at least one placeholder for ${level}: ${placeholders}`
                   );
                 }
 
-                const placeholderCheck = validateInstructionPlaceholders(cliOptions.instructions!, levelResult.data!);
+                const placeholderCheck = validateInstructionPlaceholders(cliOptions.instructions!, levelResult!.data!);
                 if (placeholderCheck !== true) {
                   fail(placeholderCheck);
                 }
@@ -603,7 +674,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 
               const result = await this.add({
                 name: cliOptions.name!,
-                level: levelResult.data!,
+                level: resolvedLevel,
                 config: configJson,
                 kmsKeyArn: cliOptions.kmsKeyArn,
                 thirdParty,
@@ -623,6 +694,10 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
                   console.log(
                     `\n  Next: Edit lambda_function.py with your evaluation logic, then run \`agentcore deploy\``
                   );
+                } else if (evalType === 'derived') {
+                  console.log(
+                    `Added evaluator '${result.evaluatorName}' (derived from ${cliOptions.baseEvaluatorId} evaluator)`
+                  );
                 } else {
                   console.log(`Added evaluator '${result.evaluatorName}'`);
                 }
@@ -644,7 +719,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
 
               return {
                 evaluator_type: standardize(EvaluatorType, evalType),
-                evaluator_level: standardize(EvaluatorLevel, levelResult.data),
+                evaluator_level: standardize(EvaluatorLevel, resolvedLevel),
                 ...(configJson.llmAsAJudge && {
                   evaluator_model_provider: standardize(
                     EvaluatorModelProvider,
@@ -728,7 +803,36 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
     };
   }
 
+  /**
+   * Resolve a derived evaluator's level from its base metric. The service requires
+   * the derived evaluator's level to match the base's, so we read it via
+   * GetEvaluator instead of asking the customer to know it.
+   */
+  private async resolveBaseEvaluatorLevel(baseEvaluatorId: string): Promise<EvaluationLevel> {
+    // A fresh project has no saved deploy targets, so resolve the region directly
+    // from the environment/profile fallback (env vars, then the AWS profile's region).
+    const region = await createConfigIO().resolveRegionFallback();
+    if (!region) {
+      throw new Error(
+        `Could not resolve an AWS region to look up "${baseEvaluatorId}". Set AWS_REGION or pass --level explicitly.`
+      );
+    }
+    try {
+      const base = await getEvaluator({ region, evaluatorId: baseEvaluatorId });
+      return base.level;
+    } catch (err) {
+      throw new Error(
+        `Could not resolve the level for base evaluator "${baseEvaluatorId}": ${getErrorMessage(err)}. ` +
+          'Pass --level explicitly to override.'
+      );
+    }
+  }
+
   private async createEvaluator(options: AddEvaluatorOptions): Promise<Evaluator> {
+    if (!options.level) {
+      throw new Error('Evaluation level is required (SESSION, TRACE, or TOOL_CALL)');
+    }
+
     const evaluator: Evaluator = {
       name: options.name,
       level: options.level,

src/cli/telemetry/schemas/common-shapes.ts

@@ -38,7 +38,7 @@ export const CredentialType = z.enum(['api-key', 'oauth']);
 // Mirrors DependencySyncOutcome in src/lib/dependency-management/types.ts.
 export const DepSyncOutcome = z.enum(['synced', 'check-only', 'opted-out', 'skipped', 'failure-suppressed', 'failed']);
 export const SkillSourceType = z.enum(['path', 's3', 'git', 'aws_skills']);
-export const EvaluatorType = z.enum(['llm-as-a-judge', 'code-based']);
+export const EvaluatorType = z.enum(['llm-as-a-judge', 'code-based', 'derived']);
 export const EvaluatorModelProvider = z.enum(['bedrock', 'openresponses']);
 export const ExitReason = z.enum(['success', 'failure']);
 export const FilterState = z.enum(['deployed', 'local-only', 'pending-removal', 'none']);

src/lib/schemas/io/config-io.ts

@@ -201,8 +201,10 @@ export class ConfigIO {
 
   /**
    * Resolve a fallback region from environment variables or AWS profile config.
+   * Public so callers that need a region before any deploy target is saved (e.g.
+   * resolving a derived evaluator's level) can reuse the same precedence.
    */
-  private async resolveRegionFallback(): Promise<string | undefined> {
+  async resolveRegionFallback(): Promise<string | undefined> {
     // Check env vars first
     const envRegion = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION;
     if (envRegion && AgentCoreRegionSchema.safeParse(envRegion).success) {

src/schema/llm-compacted/agentcore.ts

@@ -280,11 +280,25 @@ interface Evaluator {
   name: string; // @regex ^[a-zA-Z][a-zA-Z0-9_]{0,47}$ @min 1 @max 48
   level: 'SESSION' | 'TRACE' | 'TOOL_CALL';
   description?: string;
-  config: { llmAsAJudge: LlmAsAJudgeConfig; codeBased?: never } | { llmAsAJudge?: never; codeBased: CodeBasedConfig };
+  config: EvaluatorConfig; // exactly one of llmAsAJudge | codeBased | derived
   kmsKeyArn?: string;
   tags?: Tags;
 }
 
+// Exactly one arm present.
+type EvaluatorConfig =
+  | { llmAsAJudge: LlmAsAJudgeConfig; codeBased?: never; derived?: never }
+  | { llmAsAJudge?: never; codeBased: CodeBasedConfig; derived?: never }
+  | { llmAsAJudge?: never; codeBased?: never; derived: DerivedEvaluatorConfig };
+
+// A derived evaluator reuses a managed base evaluator's logic (a Builtin.* or
+// ThirdParty.<Provider>.<Metric> metric) on the customer's own model. The base
+// owns the prompt + scoring; level must match the base's (resolved at add time).
+interface DerivedEvaluatorConfig {
+  baseEvaluatorId: string; // "Builtin.<Metric>" or "ThirdParty.<Provider>.<Metric>"
+  model: string; // Bedrock inference profile ID
+}
+
 interface LlmAsAJudgeConfig {
   modelProvider?: 'Bedrock' | 'OpenResponses'; // Defaults to Bedrock when omitted
   model: string; // Bedrock model ID/ARN or OpenAI model ID

src/schema/schemas/agentcore-project.ts

@@ -67,12 +67,14 @@ export type {
   RatingScale,
 } from './primitives/evaluator';
 export {
+  BASE_EVALUATOR_ID_PATTERN,
   BedrockModelIdSchema,
   isValidBedrockModelId,
   EvaluatorNameSchema,
   KMS_KEY_ARN_PATTERN,
   isValidKmsKeyArn,
 } from './primitives/evaluator';
+export type { DerivedEvaluatorConfig } from './primitives/evaluator';
 export { ConfigBundleSchema };
 export type { ComponentConfiguration, ComponentConfigurationMap, ConfigBundle } from './primitives/config-bundle';
 export { ConfigBundleNameSchema, ComponentConfigurationMapSchema } from './primitives/config-bundle';
@@ -360,6 +362,8 @@ export type EvaluatorType = z.infer<typeof EvaluatorTypeSchema>;
 
 export const EvaluatorSchema = z.object({
   name: EvaluatorNameSchema,
+  // Required for every evaluator. For a derived evaluator it must match the base
+  // metric's level; the CLI resolves it via GetEvaluator at add time.
   level: EvaluationLevelSchema,
   description: z.string().optional(),
   config: EvaluatorConfigSchema,

src/schema/schemas/primitives/evaluator.ts

@@ -85,6 +85,34 @@ export const LlmAsAJudgeConfigSchema = z.object({
 
 export type LlmAsAJudgeConfig = z.infer<typeof LlmAsAJudgeConfigSchema>;
 
+// ============================================================================
+// Derived Evaluator Config
+// ============================================================================
+
+// A derived evaluator reuses a managed base evaluator's logic (prompt + scoring)
+// and runs it on the customer's own model. The base is a managed metric — a
+// third-party library metric (ThirdParty.<Provider>.<Metric>) or a built-in
+// (Builtin.<Metric>). The base owns the prompt and scoring; the customer supplies
+// only the model. The evaluator's `level` must match the base's level (resolved at
+// add time via GetEvaluator), so it lives on the top-level Evaluator, not here.
+// Builtin.<Metric> or ThirdParty.<Provider>.<Metric>. Each segment must be
+// non-empty alphanumeric — rejects malformed ids like "ThirdParty.DeepEval",
+// "ThirdParty..ToolUse", or "ThirdParty.DeepEval.".
+export const BASE_EVALUATOR_ID_PATTERN = /^(Builtin\.[A-Za-z0-9]+|ThirdParty\.[A-Za-z0-9]+\.[A-Za-z0-9]+)$/;
+
+export const DerivedEvaluatorConfigSchema = z.object({
+  baseEvaluatorId: z
+    .string()
+    .min(1)
+    .regex(
+      BASE_EVALUATOR_ID_PATTERN,
+      'Must be a managed base id: "ThirdParty.<Provider>.<Metric>" or "Builtin.<Metric>"'
+    ),
+  model: BedrockModelIdSchema,
+});
+
+export type DerivedEvaluatorConfig = z.infer<typeof DerivedEvaluatorConfigSchema>;
+
 // ============================================================================
 // Code-Based Evaluator Config
 // ============================================================================
@@ -125,9 +153,16 @@ export const EvaluatorConfigSchema = z
   .object({
     llmAsAJudge: LlmAsAJudgeConfigSchema.optional(),
     codeBased: CodeBasedConfigSchema.optional(),
+    derived: DerivedEvaluatorConfigSchema.optional(),
   })
-  .refine(config => Boolean(config.llmAsAJudge) !== Boolean(config.codeBased), {
-    message: 'Config must have either llmAsAJudge or codeBased, not both',
+  .superRefine((config, ctx) => {
+    const arms = [config.llmAsAJudge, config.codeBased, config.derived].filter(Boolean).length;
+    if (arms !== 1) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: 'Config must have exactly one of llmAsAJudge, codeBased, or derived',
+      });
+    }
   });
 
 export type EvaluatorConfig = z.infer<typeof EvaluatorConfigSchema>;