Agent test UI: routing cases, history, registration, scope planning, …

src/lib/helpers/utils/agent-test.js

@@ -35,15 +35,115 @@ export const ASSERTION_TYPES = [
 	'toolNotCalled',
 	'stateEquals',
 	'routedToAgent',
+	'agentChain',
 	'llmJudge'
 ];
 
 /** Types whose `expected` the backend rejects as empty. */
-const EXPECTED_REQUIRED = ['outputContains', 'outputNotContains', 'outputRegex', 'routedToAgent', 'llmJudge'];
+const EXPECTED_REQUIRED = [
+	'outputContains',
+	'outputNotContains',
+	'outputRegex',
+	'routedToAgent',
+	'agentChain',
+	'llmJudge'
+];
 
 /** Types whose `target` the backend rejects as empty. */
 const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals'];
 
+/**
+ * How an `agentChain` assertion compares its expected list (AgentChainModes in
+ * AssertionEvaluator.cs). On this assertion type `target` carries the mode
+ * rather than a function name or state key.
+ */
+export const AGENT_CHAIN_MODES = ['contains', 'ordered', 'exact'];
+
+/**
+ * What a case is verifying (CaseTypes in AgentTestCase.cs). Routing is checked
+ * more strictly -- see `validateCaseType` -- because it is the only type counted
+ * towards a run's routing accuracy. A multi-agent journey is an Agent case whose
+ * `agentChain` assertion describes the hand-offs.
+ */
+export const CASE_TYPES = ['Routing', 'Agent'];
+
+/**
+ * Roles an authored history message may take (HistoryRoles in AgentTestCase.cs).
+ * Only these two: `system` would compete with the agent's own instruction and
+ * `function` would fake a tool call, letting a case claim a tool ran when
+ * nothing did.
+ */
+export const HISTORY_ROLES = ['user', 'assistant'];
+
+/**
+ * How urgent it is to run a case, which is what decides its batch (CasePriorities
+ * in AgentTestCase.cs). Distinct from severity: priority is scheduling,
+ * severity is consequence.
+ */
+export const CASE_PRIORITIES = ['P0', 'P1', 'P2'];
+
+/**
+ * What a failure means (CaseSeverities in AgentTestCase.cs). S0 is a stop rather
+ * than a statistic; S2 must never be able to mask an S0 or S1.
+ */
+export const CASE_SEVERITIES = ['S0', 'S1', 'S2'];
+
+/** Batches run in order: 1 is the stop-loss batch, 3 does not block a release. */
+export const CASE_BATCHES = [1, 2, 3];
+
+/**
+ * Tone for a priority badge. P0 is the stop-loss batch -- one failure there halts the
+ * whole evaluation -- so it reads as urgent; P2 does not block a release and reads as
+ * quiet. The middle stays neutral rather than warning-coloured, because P1 is the
+ * default every untriaged case carries and a wall of amber would say nothing.
+ * @param {string?} priority
+ */
+export function priorityTone(priority) {
+	switch (priority) {
+		case 'P0': return 'danger';
+		case 'P2': return 'secondary';
+		default: return 'primary';
+	}
+}
+
+/**
+ * Tone for a severity badge. S0 is zero-tolerance -- data leakage, an unauthorised
+ * action, a missed escalation -- and has to be visible at a glance in a list, which is
+ * the whole reason severity is worth a column. S2 is phrasing and must never look as
+ * loud as the other two.
+ * @param {string?} severity
+ */
+export function severityTone(severity) {
+	switch (severity) {
+		case 'S0': return 'danger';
+		case 'S1': return 'warning';
+		default: return 'secondary';
+	}
+}
+
+/**
+ * The batch a case will actually run in, mirroring CaseBatches.Effective. An
+ * explicit batch wins; a cross-cutting case is batch 1 whatever its priority,
+ * because a safety case that only runs after everything else has passed cannot
+ * stop anything.
+ * @param {{ batch?: number | null, crossCutting?: boolean, priority?: string }} testCase
+ * @returns {number}
+ */
+export function effectiveBatch(testCase) {
+	if (CASE_BATCHES.includes(Number(testCase?.batch))) {
+		return Number(testCase.batch);
+	}
+	if (testCase?.crossCutting) {
+		return 1;
+	}
+	if (testCase?.priority === 'P0') return 1;
+	if (testCase?.priority === 'P2') return 3;
+	return 2;
+}
+
+/** Assertion types that establish a routing outcome, so a Routing case needs one. */
+const ROUTING_ASSERTION_TYPES = ['routedToAgent', 'agentChain'];
+
 /**
  * Bootstrap contextual class for a run/case status.
  * @param {string?} status
@@ -132,13 +232,76 @@ export function validateAssertion(assertion) {
 		}
 	}
 
+	// agentChain puts the comparison mode in `target`. An unrecognised value fails
+	// at evaluation time rather than falling back to the loosest mode, so catching
+	// it here just moves that failure to where the author can see it. Blank is
+	// legal and means `contains`.
+	if (type === 'agentChain' && assertion.target?.trim()
+		&& !AGENT_CHAIN_MODES.includes(assertion.target.trim().toLowerCase())) {
+		return t('Assertion "agentChain" mode must be one of {modes}.', { modes: AGENT_CHAIN_MODES.join(', ') });
+	}
+
 	if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) {
 		return t('The args match on an assertion is not valid JSON.');
 	}
 
 	return null;
 }
 
+/**
+ * The extra rules the backend applies to a Routing case, checked here so the
+ * author sees them while editing instead of as a 400 on save. Kept in step with
+ * AgentTestController.ValidateRoutingCase -- if these two ever disagree, the
+ * backend wins and the save fails, which is the safe direction.
+ * @param {string} caseType
+ * @param {import('$agentTestTypes').TestTurn[]} turns
+ * @param {import('$agentTestTypes').TestAssertion[]} caseAssertions
+ * @returns {string[]} one message per problem, empty when the case is acceptable
+ */
+export function validateCaseType(caseType, turns, caseAssertions) {
+	if (caseType !== 'Routing') {
+		return [];
+	}
+
+	const problems = [];
+	const all = [...(turns || []).flatMap(turn => turn.assertions || []), ...(caseAssertions || [])];
+
+	// Routing asks a single-turn question: which agent picks this message up. A
+	// second turn asks something else, and its verdict would still land in the
+	// routing accuracy figure.
+	// Authored history is deliberately not counted: replaying a prior exchange and
+	// then asking one question is still a single routing decision, and it is the most
+	// realistic way to test routing that depends on context.
+	if ((turns || []).length !== 1) {
+		problems.push(t('A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.'));
+	}
+
+	// Without one of these the case asserts nothing about routing yet still counts
+	// towards routing accuracy -- it reports Passed for having said anything at all.
+	if (!all.some(a => ROUTING_ASSERTION_TYPES.includes(a?.type))) {
+		problems.push(t('A Routing case needs at least one routedToAgent or agentChain assertion.'));
+	}
+
+	// Routing is scored purely as expected agent == actual agent. An llmJudge would
+	// also make the figure depend on a vendor call, so a vendor outage would read
+	// as a routing regression.
+	if (all.some(a => a?.type === 'llmJudge')) {
+		problems.push(t('A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.'));
+	}
+
+	return problems;
+}
+
+/**
+ * An agent chain rendered the way the backend reports it in an assertion's
+ * `actual`, so the two read the same on screen.
+ * @param {string[] | null | undefined} chain
+ * @returns {string}
+ */
+export function formatAgentChain(chain) {
+	return chain?.length ? chain.join(' -> ') : '--';
+}
+
 /**
  * @param {string} text
  * @returns {boolean}

src/lib/langs/en.json

@@ -453,8 +453,8 @@
     "No mocks. Every tool call this case makes will be blocked.": "No mocks. Every tool call this case makes will be blocked.",
     "A disabled suite cannot be run at all.": "A disabled suite cannot be run at all.",
     "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.",
-    "llmJudge assertions always fail in P1 regardless of these two fields.": "llmJudge assertions always fail in P1 regardless of these two fields.",
-    "Always fails in P1.": "Always fails in P1.",
+    "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.": "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.",
+    "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.": "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.",
     "One function name per line. These run for real during a test -- only list functions with no side effects.": "One function name per line. These run for real during a test -- only list functions with no side effects.",
     "One per line. Blocked even if allow-listed elsewhere.": "One per line. Blocked even if allow-listed elsewhere.",
     "Assertions checked right after this turn": "Assertions checked right after this turn",
@@ -546,5 +546,124 @@
     "This run could not complete": "This run could not complete",
     "Nothing ran -- see the reason above.": "Nothing ran -- see the reason above.",
     "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "Nothing here can run -- every case you picked is disabled. Enable at least one first.",
-    "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped."
+    "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped.",
+    "Case type": "Case type",
+    "Routing case": "Routing case",
+    "Agent case": "Agent case",
+    "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.",
+    "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.",
+    "Entry agent": "Entry agent",
+    "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.": "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.",
+    "A Routing case needs at least one routedToAgent or agentChain assertion.": "A Routing case needs at least one routedToAgent or agentChain assertion.",
+    "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.",
+    "Mode": "Mode",
+    "Assertion \"agentChain\" mode must be one of {modes}.": "Assertion \"agentChain\" mode must be one of {modes}.",
+    "Copilot, Work Order Creator": "Copilot, Work Order Creator",
+    "These agents appear somewhere in the chain, in any order.": "These agents appear somewhere in the chain, in any order.",
+    "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "These agents appear in this order; other agents may come in between. This is the hand-off assertion.",
+    "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.": "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.",
+    "Agent chain": "Agent chain",
+    "Answered by": "Answered by",
+    "Routing accuracy": "Routing accuracy",
+    "agent default": "agent default",
+    "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.": "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.",
+    "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.",
+    "Use the suite's agent": "Use the suite's agent",
+    "The agent the conversation opens on. Leave it unset to use the suite's.": "The agent the conversation opens on. Leave it unset to use the suite's.",
+    "History": "History",
+    "Add Message": "Add Message",
+    "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.": "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.",
+    "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.": "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.",
+    "Role": "Role",
+    "Content": "Content",
+    "Move up": "Move up",
+    "Move down": "Move down",
+    "Remove message": "Remove message",
+    "No history. The case starts from an empty conversation.": "No history. The case starts from an empty conversation.",
+    "History message {n} has no content.": "History message {n} has no content.",
+    "Assistant": "Assistant",
+    "Copy": "Copy",
+    "Copy case": "Copy case",
+    "Copied to \"{name}\". It is disabled until you enable it.": "Copied to \"{name}\". It is disabled until you enable it.",
+    "Failed to copy the case.": "Failed to copy the case.",
+    "Registration": "Registration",
+    "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.": "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.",
+    "Priority": "Priority",
+    "P0 runs in the stop-loss batch, P2 does not block a release.": "P0 runs in the stop-loss batch, P2 does not block a release.",
+    "Severity": "Severity",
+    "S0 is a stop, not a statistic. S2 is phrasing and experience.": "S0 is a stop, not a statistic. S2 is phrasing and experience.",
+    "Batch": "Batch",
+    "Derive from priority": "Derive from priority",
+    "Runs in batch {n}.": "Runs in batch {n}.",
+    "Cross-cutting": "Cross-cutting",
+    "Runs in every scope, whatever changed, and always in batch 1.": "Runs in every scope, whatever changed, and always in batch 1.",
+    "Involved agents": "Involved agents",
+    "Derived from the entry agent": "Derived from the entry agent",
+    "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.": "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.",
+    "Business domain": "Business domain",
+    "Last reviewed": "Last reviewed",
+    "Today": "Today",
+    "Never set automatically -- editing a case is not reviewing it.": "Never set automatically -- editing a case is not reviewing it.",
+    "Expected outcome": "Expected outcome",
+    "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.": "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.",
+    "Latency, tokens and cost": "Latency, tokens and cost",
+    "Model": "Model",
+    "Cases": "Cases",
+    "Latency P50": "Latency P50",
+    "Latency P95": "Latency P95",
+    "Tokens": "Tokens",
+    "Cost": "Cost",
+    "Unit cost (in / out)": "Unit cost (in / out)",
+    "unknown": "unknown",
+    "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.": "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.",
+    "Wall clock for the whole case": "Wall clock for the whole case",
+    "Agent call time only": "Agent call time only",
+    "Tokens / cost": "Tokens / cost",
+    "Plan a scope": "Plan a scope",
+    "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.": "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.",
+    "Changed agents": "Changed agents",
+    "All batches": "All batches",
+    "Platform-wide change": "Platform-wide change",
+    "Narrowing is switched off: no agent is demonstrably untouched.": "Narrowing is switched off: no agent is demonstrably untouched.",
+    "Plan": "Plan",
+    "Failed to work out the scope.": "Failed to work out the scope.",
+    "included": "included",
+    "excluded": "excluded",
+    "of {n} registered": "of {n} registered",
+    "In scope": "In scope",
+    "Suite": "Suite",
+    "Reason": "Reason",
+    "Yes": "Yes",
+    "No": "No",
+    "No involved agents known, so it was included to be safe.": "No involved agents known, so it was included to be safe.",
+    "Select all runs": "Select all runs",
+    "Select run": "Select run",
+    "Delete selected": "Delete selected",
+    "{n} still running and cannot be deleted": "{n} still running and cannot be deleted",
+    "Delete run": "Delete run",
+    "Delete this run and its case results? You won't be able to revert this!": "Delete this run and its case results? You won't be able to revert this!",
+    "Delete {n} runs and all their case results? You won't be able to revert this!": "Delete {n} runs and all their case results? You won't be able to revert this!",
+    "Deleted {runs} run(s) and {results} result(s).": "Deleted {runs} run(s) and {results} result(s).",
+    "Run {id} was kept: {reason}": "Run {id} was kept: {reason}",
+    "Failed to delete the runs.": "Failed to delete the runs.",
+    "Evaluation scope": "Evaluation scope",
+    "No test cases are registered yet.": "No test cases are registered yet.",
+    "batch {n}": "batch {n}",
+    "Write by Chat": "Write by Chat",
+    "Show": "Show",
+    "Hide": "Hide",
+    "Undo": "Undo",
+    "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.": "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.",
+    "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.": "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.",
+    "Working on it...": "Working on it...",
+    "Your instruction": "Your instruction",
+    "Add a turn, change an assertion, ask what a field means...": "Add a turn, change an assertion, ask what a field means...",
+    "Enter sends. Shift+Enter starts a new line.": "Enter sends. Shift+Enter starts a new line.",
+    "Send": "Send",
+    "Changed in this reply": "Changed in this reply",
+    "The assistant could not produce a valid draft:": "The assistant could not produce a valid draft:",
+    "Worth checking:": "Worth checking:",
+    "changed by chat": "changed by chat",
+    "New Case by Chat": "New Case by Chat",
+    "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above.": "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above."
 }