From f064f735f2e7637e8635ab35e142409f2a66393e Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Thu, 20 Aug 2026 21:40:29 +0800 Subject: [PATCH 1/5] Agent test UI: routing cases, history, registration, scope planning, metrics Follows the backend commit in BotSharp. One commit for the same reason: the four agent-test pages and the shared helper each carry changes from several of the features below, and the llmJudge UI work was already sitting uncommitted here. Case editor ----------- Case type (Routing or Agent) with the rules a routing case has to satisfy checked as you type, sharing one helper with the backend so the two cannot drift -- and if they ever do, the backend wins and the save fails, which is the safe direction. Entry agent is a dropdown, not an id field. The id is a guid nobody types from memory and a typo was only caught on save. The dropdown own "Clear selection" is how you fall back to the suite agent, and the payload sends null rather than an empty string for that. Authored history rows (role plus text, reorderable, since the order IS the conversation) with a note that they are not driven through the model and never appear in the agent chain. A registration panel: priority, severity, batch, cross-cutting, involved agents, business domain, expected outcome and a last-reviewed date. The batch help text shows the batch the case will actually run in, since it is derived rather than chosen. Last reviewed has a Today button -- a date field nobody can be bothered to type is a date field that stays empty, and an empty reviewed date is indistinguishable from a case reviewed long ago. An agentChain assertion turns the Target column into a mode dropdown, because that field carries the comparison mode for this one type. Switching an assertion type now resets a mode left behind by the previous type, which would otherwise be saved as the name of a function to assert was called. Case list --------- Type and history columns, so the type a case runs under is visible without opening every one of them, and a copy button. Copying navigates straight into the copy: it lands disabled and named "(copy)", and neither is something anyone leaves as it is. Fixed a bug this branch had introduced: the enable toggle rebuilt a full-replace PUT payload field by field, so a single click erased caseType, entryAgentId and history. It now spreads the case, which cannot drop a field added later. Run detail ---------- Routing accuracy per model, shown as a percentage next to passed/total -- with a handful of routing cases, how much the figure is worth trusting matters more than the figure. Latency, tokens and cost per model, alongside the unit costs the cost was computed from. Unknown pricing renders as unknown, never as zero, which would read as a claim that the model is free. The agent chain on each result, visible without expanding, since "which agent actually answered" is the first thing looked at when a routing case goes red. Per result the agent-call time is shown next to the wall clock, because the percentiles are built from the former. Scope planning -------------- A panel on the index page: name the changed agents, or declare a platform-wide change, and see which cases that needs to run. Deliberately separate from triggering a run -- folding it into a run button would mean the only way to see the plan is to have already paid for it. Included and excluded cases appear in one table with the rule that decided each. Splitting them into two panels would make it possible to read only the reassuring one, and the exclusions are the half that matters: an excluded case produces no result to notice. Translations ------------ Every new string in both dictionaries, no missing keys either way. Case type labels use their own keys rather than the bare words: "Routing" already exists in zh.json meaning the routing-graph screen, so reusing it would have mistranslated the option. .env is deliberately not included -- it points at a local host. Co-Authored-By: Claude Opus 5 --- src/lib/helpers/types/agentTestTypes.js | 160 ++++++- src/lib/helpers/utils/agent-test.js | 135 +++++- src/lib/langs/en.json | 95 +++- src/lib/langs/zh.json | 95 +++- src/lib/services/agent-test-service.js | 34 ++ src/lib/services/api-endpoints.js | 4 +- src/routes/page/agent-test/+page.svelte | 179 +++++++- .../page/agent-test/[suiteId]/+page.svelte | 60 ++- .../[suiteId]/case/[caseId]/+page.svelte | 430 +++++++++++++++++- .../page/agent-test/run/[runId]/+page.svelte | 139 +++++- 10 files changed, 1289 insertions(+), 42 deletions(-) diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 9c9cf950..06627f4b 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -5,8 +5,9 @@ * @property {string} name * @property {string?} description * @property {boolean} enabled - * @property {string?} judgeProvider - Provider for llmJudge assertions. In P1 llmJudge always fails, - * whether or not this is configured. + * @property {string?} judgeProvider - Provider for llmJudge assertions. Both this and judgeModel are + * required for llmJudge to be scored at all; with either missing, an llmJudge assertion records a + * case-level Error rather than being scored with some default model. * @property {string?} judgeModel * @property {string[]} extraAllowedFunctions - Functions let through on top of the default * control-flow allow list. @@ -79,9 +80,13 @@ * null/empty on every run and can never pass. It fails rather than errors, so nothing points at it. * The form is deliberately stricter and treats `expected` as required there too. * - * `llmJudge` always fails in P1 (the backend returns "llmJudge is not available in P1"), regardless - * of minScore or whether the suite configured judgeProvider/judgeModel. The form may offer it, but - * says so next to it. + * `llmJudge` is scored 1-5 by the suite's judge model, and `minScore` is the pass mark (default 4). + * It is the one assertion type that is not reproducible, and the one that can end a case in Error + * rather than Failed: an unconfigured judge model, an unregistered provider, a vendor failure or a + * reply the backend cannot read as a score all mean "no verdict", which is Error. Those are + * deliberately not failures -- a vendor timeout is not an agent regression. Note also that the judge + * only ever sees the criterion and the agent's reply, never the user's message, so a criterion has + * to be self-contained. * * @typedef {Object} TestAssertion * @property {string} type - outputContains | outputNotContains | outputRegex | toolCalled | toolNotCalled | stateEquals | routedToAgent | llmJudge @@ -99,15 +104,41 @@ * @property {string} name * @property {boolean} enabled - Recorded drafts land as false; a human has to review and enable them * before they join a normal run. + * @property {string} caseType - Routing | Agent. Defaults to Agent, which is also what every case + * stored before the field existed reads back as. Routing is validated more strictly (one turn, + * must assert a routing outcome, no llmJudge) and is the only type counted towards a run's routing + * accuracy. A journey across several agents is an Agent case whose `agentChain` assertion + * describes the hand-offs. + * @property {string?} entryAgentId - Agent the conversation opens on, overriding the suite's; null + * uses the suite's. This is the switch between testing routing and testing one agent alone: + * BotSharp sends a routing-type agent through the router (which can hand off) and any other agent + * straight into itself (the router never runs). * @property {TestTurn[]} turns - Length 1 is a single-turn case. * @property {TestAssertion[]} assertions - Case-level assertions, evaluated after every turn has run. * @property {TestState[]} initialStates - Injected before the conversation starts; maps to BotSharp's * MessageState. + * @property {TestHistoryMessage[]} history - Prior turns written into the conversation before the + * case's own turns run, so a real question-and-answer exchange becomes the fixed starting context. + * Not driven through the model (no token cost, no flakiness) and never counted in `agentChain`. * @property {TestToolMock[]} mocks * @property {string} unmockedToolPolicy - P1 accepts only "Block"; sending "Passthrough" is a 400 * ("Passthrough is not supported in P1"). The form should not offer it. * @property {string?} sourceConversationId - The conversation this was recorded from, for * traceability; null for a hand-written case. + * @property {string} priority - P0 | P1 | P2. Decides the batch, and therefore whether a failure + * stops the evaluation. Defaults to P1, which is also what a case stored before the field existed + * reads back as. + * @property {string} severity - S0 | S1 | S2. What a failure MEANS, as opposed to how urgent the case + * is to run. Defaults to S1. + * @property {number?} batch - Explicit override; null derives it from priority and crossCutting. + * @property {boolean} crossCutting - Runs in every scope whatever changed, and always in batch 1. + * @property {string[]} involvedAgents - Agent ids. Empty falls back to the case's entry agent, which + * is right for an Agent case and only a starting point for a Routing case, where the agents that + * matter are downstream of the router. + * @property {string?} businessDomain + * @property {string?} expectedOutcome - For whoever reviews the result; never evaluated. + * @property {string?} lastReviewedDate - ISO date. Never set by the server: editing a case is not + * reviewing it. * @property {string} createDate - ISO date string. * @property {string} updateDate - ISO date string. */ @@ -125,6 +156,20 @@ * @property {string} suiteId * @property {string} name * @property {boolean} [enabled] - Defaults to true. + * @property {string} [caseType] - Routing | Agent; blank or omitted means Agent. Any other value is + * a 400 rather than a silent fallback. + * @property {string?} [entryAgentId] - Send null rather than "" to mean "use the suite's agent"; + * validated to exist at save time. + * @property {TestHistoryMessage[]} [history] - Authored prior turns; every message needs a + * user/assistant role and non-empty content. + * @property {string} [priority] - P0 | P1 | P2; blank keeps P1. Any other value is a 400. + * @property {string} [severity] - S0 | S1 | S2; blank keeps S1. + * @property {number?} [batch] - 1, 2 or 3; null derives it. Out of range is a 400, not a clamp. + * @property {boolean} [crossCutting] + * @property {string[]} [involvedAgents] + * @property {string?} [businessDomain] + * @property {string?} [expectedOutcome] + * @property {string?} [lastReviewedDate] - ISO date; send it only when a human actually reviewed. * @property {TestTurn[]} turns * @property {TestAssertion[]} assertions * @property {TestState[]} initialStates @@ -133,6 +178,16 @@ * @property {string?} [sourceConversationId] */ +/** + * One authored message in a case's history. Just a role and text: a mocked tool call belongs in + * `mocks`, and a fabricated function-call dialog would let a case claim a tool ran when nothing did. + * @typedef {Object} TestHistoryMessage + * @property {string} role - user | assistant. Nothing else: `system` would compete with the agent's + * own instruction and `function` would fake a tool call. + * @property {string} content - Rejected as empty; BotSharp's dialog storage drops blank elements, so + * an empty message would silently not be in the conversation at run time. + */ + /** * One model to run against. * @typedef {Object} TestModel @@ -163,12 +218,57 @@ * @property {number} passedCount * @property {number} failedCount * @property {number} errorCount + * @property {RoutingAccuracy[]} routingAccuracies - One row per model swept, counting only Routing + * cases; empty when the run contained none. + * @property {PerformanceSummary[]} performanceSummaries - Latency, token and cost figures, one row per + * model. Computed when the run finishes, because a percentile needs every value at once. + * @property {ModelPricingSnapshot[]} modelPricing - The unit costs in force when the run executed. A + * cost figure is not comparable with another run's unless these match. Kept per model because a single run-wide figure would + * average a candidate model together with the baseline and hide the difference the run exists to + * measure. * @property {boolean} cancelRequested * @property {string?} startedAt - ISO date string; null until it starts. * @property {string?} completedAt - ISO date string; null until it finishes. * @property {string} createDate - ISO date string. Note AgentTestRun has no updateDate field. */ +/** + * Latency, tokens and cost for one model within a run. Averages are absent on purpose: an average is + * Total/CaseCount, and a stored copy is one more thing that can disagree with the rows it came from. + * @typedef {Object} PerformanceSummary + * @property {string?} provider + * @property {string?} model + * @property {number} caseCount - Only the cases that reached the model. A case that failed before its + * first turn would drag a latency percentile towards zero and make a broken run look fast. + * @property {number} latencyP50Ms - Nearest-rank median of the agent-call time, so it is always a + * duration some case actually took rather than an interpolated one that none did. + * @property {number} latencyP95Ms + * @property {number} totalTokens - Over EVERY result, unlike latency: a case that errored still spent + * what it spent. + * @property {number} totalCost + */ + +/** + * One model's configured text-token unit costs at the moment a run executed. + * @typedef {Object} ModelPricingSnapshot + * @property {string?} provider + * @property {string?} model + * @property {number?} textInputCost - Null means the settings could not be read. Never render it as + * 0, which would read as "this model is free". + * @property {number?} textOutputCost + */ + +/** + * Routing accuracy for one model within a run. Counts, never a stored percentage: "3/4" says how + * much the figure is worth trusting and "75%" does not. + * @typedef {Object} RoutingAccuracy + * @property {string?} provider - Null for both when the run swept no models. + * @property {string?} model + * @property {number} caseCount - Routing cases executed under this model, whatever their outcome -- + * Error rows included, because "could not tell" is not "routed correctly". + * @property {number} passedCount + */ + /** * @typedef {Object} AssertionResult * @property {string} type @@ -185,6 +285,11 @@ * @property {string} userMessage * @property {string?} output * @property {AssertionResult[]} assertions + * @property {number} modelDurationMs - Time the agent call for this turn took, excluding the + * assertion evaluation and conversation reads that follow it. + * @property {string[]} agentChain - The agents that answered during THIS turn, in order, with + * consecutive repeats collapsed. Not derivable from the case-level chain, which collapses across + * turn boundaries too. */ /** @@ -203,6 +308,8 @@ * @property {string} runId * @property {string} caseId * @property {string} caseName + * @property {string} caseType - Copied off the case so a result is self-describing; routing + * accuracy is aggregated from these rows rather than from cases that may since have been edited. * @property {string} status - Passed | Failed | Error | Cancelled (never Pending/Running). * @property {string?} conversationId - The conversation this execution created; live conversations * are never reused. @@ -210,16 +317,57 @@ * LlmConfig was used. * @property {string?} model - As above. In a multi-model run the same `caseId` yields several * results, told apart by these two fields. - * @property {number} durationMs + * @property {number} durationMs - Wall clock for the whole case, including the canary and the + * conversation reads. Comparable between models, but not a model-latency measurement. + * @property {number} modelDurationMs - The agent calls alone, summed over the turns. This is what the + * run's latency percentiles are built from. + * @property {number} totalTokens - Measured as a delta across this case's own execution. Total only: + * the input/output split is not reachable through ITokenStatistics. + * @property {number} cost * @property {string?} error - Infrastructure-level reason for failure (timeout, a dead mock seam, a * case with no turns), as distinct from an assertion failure. Show this text whenever `status` is * `Error`. * @property {TurnResult[]} turns * @property {AssertionResult[]} assertions - Case-level assertion results. * @property {ObservedToolCall[]} observedToolCalls + * @property {string[]} agentChain - Every agent that answered over the whole case, in order, + * consecutive repeats collapsed. The only record of the hand-offs: route_to_agent is allowed + * through the mock seam untouched, so it never appears in `observedToolCalls`. * @property {string} createDate - ISO date string. */ +/** + * One case's place in a scope, carrying the metadata the decision was made from rather than just the + * verdict -- a scope nobody can explain is a scope nobody can review. + * @typedef {Object} ScopedCase + * @property {string} caseId + * @property {string} caseName + * @property {string} suiteId + * @property {string} suiteName + * @property {string} caseType + * @property {string} priority + * @property {string} severity + * @property {boolean} crossCutting + * @property {boolean} enabled + * @property {number} batch - The effective batch, after the priority and cross-cutting derivation. + * @property {string[]} involvedAgentIds - Authored, or derived from the entry agent. + * @property {string} reason - crossCutting | fullPlatform | targetAgent | unknownAgents | + * notInvolved | disabled | otherBatch. + */ + +/** + * The answer to "what will this change actually test". Both halves are returned on purpose: an + * excluded case produces no result to notice, so the exclusions and their reasons are the half worth + * reading. + * @typedef {Object} ScopeSelection + * @property {string[]} targetAgentIds + * @property {boolean} fullPlatform + * @property {number?} batch + * @property {number} totalCases - Every registered case, whatever its state: the coverage denominator. + * @property {ScopedCase[]} included + * @property {ScopedCase[]} excluded + */ + /** * Body of GET /agent-test/runs/{id} (the camelCase projection of the backend's * AgentTestRunDetailDto). diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js index 41c06c92..26d0df00 100644 --- a/src/lib/helpers/utils/agent-test.js +++ b/src/lib/helpers/utils/agent-test.js @@ -35,15 +35,85 @@ export const ASSERTION_TYPES = [ 'toolNotCalled', 'stateEquals', 'routedToAgent', + 'agentChain', 'llmJudge' ]; /** Types whose `expected` the backend rejects as empty. */ -const EXPECTED_REQUIRED = ['outputContains', 'outputNotContains', 'outputRegex', 'routedToAgent', 'llmJudge']; +const EXPECTED_REQUIRED = [ + 'outputContains', + 'outputNotContains', + 'outputRegex', + 'routedToAgent', + 'agentChain', + 'llmJudge' +]; /** Types whose `target` the backend rejects as empty. */ const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals']; +/** + * How an `agentChain` assertion compares its expected list (AgentChainModes in + * AssertionEvaluator.cs). On this assertion type `target` carries the mode + * rather than a function name or state key. + */ +export const AGENT_CHAIN_MODES = ['contains', 'ordered', 'exact']; + +/** + * What a case is verifying (CaseTypes in AgentTestCase.cs). Routing is checked + * more strictly -- see `validateCaseType` -- because it is the only type counted + * towards a run's routing accuracy. A multi-agent journey is an Agent case whose + * `agentChain` assertion describes the hand-offs. + */ +export const CASE_TYPES = ['Routing', 'Agent']; + +/** + * Roles an authored history message may take (HistoryRoles in AgentTestCase.cs). + * Only these two: `system` would compete with the agent's own instruction and + * `function` would fake a tool call, letting a case claim a tool ran when + * nothing did. + */ +export const HISTORY_ROLES = ['user', 'assistant']; + +/** + * How urgent it is to run a case, which is what decides its batch (CasePriorities + * in AgentTestCase.cs). Distinct from severity: priority is scheduling, + * severity is consequence. + */ +export const CASE_PRIORITIES = ['P0', 'P1', 'P2']; + +/** + * What a failure means (CaseSeverities in AgentTestCase.cs). S0 is a stop rather + * than a statistic; S2 must never be able to mask an S0 or S1. + */ +export const CASE_SEVERITIES = ['S0', 'S1', 'S2']; + +/** Batches run in order: 1 is the stop-loss batch, 3 does not block a release. */ +export const CASE_BATCHES = [1, 2, 3]; + +/** + * The batch a case will actually run in, mirroring CaseBatches.Effective. An + * explicit batch wins; a cross-cutting case is batch 1 whatever its priority, + * because a safety case that only runs after everything else has passed cannot + * stop anything. + * @param {{ batch?: number | null, crossCutting?: boolean, priority?: string }} testCase + * @returns {number} + */ +export function effectiveBatch(testCase) { + if (CASE_BATCHES.includes(Number(testCase?.batch))) { + return Number(testCase.batch); + } + if (testCase?.crossCutting) { + return 1; + } + if (testCase?.priority === 'P0') return 1; + if (testCase?.priority === 'P2') return 3; + return 2; +} + +/** Assertion types that establish a routing outcome, so a Routing case needs one. */ +const ROUTING_ASSERTION_TYPES = ['routedToAgent', 'agentChain']; + /** * Bootstrap contextual class for a run/case status. * @param {string?} status @@ -132,6 +202,15 @@ export function validateAssertion(assertion) { } } + // agentChain puts the comparison mode in `target`. An unrecognised value fails + // at evaluation time rather than falling back to the loosest mode, so catching + // it here just moves that failure to where the author can see it. Blank is + // legal and means `contains`. + if (type === 'agentChain' && assertion.target?.trim() + && !AGENT_CHAIN_MODES.includes(assertion.target.trim().toLowerCase())) { + return t('Assertion "agentChain" mode must be one of {modes}.', { modes: AGENT_CHAIN_MODES.join(', ') }); + } + if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) { return t('The args match on an assertion is not valid JSON.'); } @@ -139,6 +218,60 @@ export function validateAssertion(assertion) { return null; } +/** + * The extra rules the backend applies to a Routing case, checked here so the + * author sees them while editing instead of as a 400 on save. Kept in step with + * AgentTestController.ValidateRoutingCase -- if these two ever disagree, the + * backend wins and the save fails, which is the safe direction. + * @param {string} caseType + * @param {import('$agentTestTypes').TestTurn[]} turns + * @param {import('$agentTestTypes').TestAssertion[]} caseAssertions + * @returns {string[]} one message per problem, empty when the case is acceptable + */ +export function validateCaseType(caseType, turns, caseAssertions) { + if (caseType !== 'Routing') { + return []; + } + + const problems = []; + const all = [...(turns || []).flatMap(turn => turn.assertions || []), ...(caseAssertions || [])]; + + // Routing asks a single-turn question: which agent picks this message up. A + // second turn asks something else, and its verdict would still land in the + // routing accuracy figure. + // Authored history is deliberately not counted: replaying a prior exchange and + // then asking one question is still a single routing decision, and it is the most + // realistic way to test routing that depends on context. + if ((turns || []).length !== 1) { + problems.push(t('A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.')); + } + + // Without one of these the case asserts nothing about routing yet still counts + // towards routing accuracy -- it reports Passed for having said anything at all. + if (!all.some(a => ROUTING_ASSERTION_TYPES.includes(a?.type))) { + problems.push(t('A Routing case needs at least one routedToAgent or agentChain assertion.')); + } + + // Routing is scored purely as expected agent == actual agent. An llmJudge would + // also make the figure depend on a vendor call, so a vendor outage would read + // as a routing regression. + if (all.some(a => a?.type === 'llmJudge')) { + problems.push(t('A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.')); + } + + return problems; +} + +/** + * An agent chain rendered the way the backend reports it in an assertion's + * `actual`, so the two read the same on screen. + * @param {string[] | null | undefined} chain + * @returns {string} + */ +export function formatAgentChain(chain) { + return chain?.length ? chain.join(' -> ') : '--'; +} + /** * @param {string} text * @returns {boolean} diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 1c0f2772..13ca37fc 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -446,8 +446,8 @@ "No mocks. Every tool call this case makes will be blocked.": "No mocks. Every tool call this case makes will be blocked.", "A disabled suite cannot be run at all.": "A disabled suite cannot be run at all.", "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.", - "llmJudge assertions always fail in P1 regardless of these two fields.": "llmJudge assertions always fail in P1 regardless of these two fields.", - "Always fails in P1.": "Always fails in P1.", + "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.": "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.", + "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.": "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.", "One function name per line. These run for real during a test -- only list functions with no side effects.": "One function name per line. These run for real during a test -- only list functions with no side effects.", "One per line. Blocked even if allow-listed elsewhere.": "One per line. Blocked even if allow-listed elsewhere.", "Assertions checked right after this turn": "Assertions checked right after this turn", @@ -539,5 +539,94 @@ "This run could not complete": "This run could not complete", "Nothing ran -- see the reason above.": "Nothing ran -- see the reason above.", "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "Nothing here can run -- every case you picked is disabled. Enable at least one first.", - "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped." + "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped.", + "Case type": "Case type", + "Routing case": "Routing case", + "Agent case": "Agent case", + "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.", + "Entry agent": "Entry agent", + "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.": "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.", + "A Routing case needs at least one routedToAgent or agentChain assertion.": "A Routing case needs at least one routedToAgent or agentChain assertion.", + "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.", + "Mode": "Mode", + "Assertion \"agentChain\" mode must be one of {modes}.": "Assertion \"agentChain\" mode must be one of {modes}.", + "Copilot, Work Order Creator": "Copilot, Work Order Creator", + "These agents appear somewhere in the chain, in any order.": "These agents appear somewhere in the chain, in any order.", + "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "These agents appear in this order; other agents may come in between. This is the hand-off assertion.", + "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.": "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.", + "Agent chain": "Agent chain", + "Answered by": "Answered by", + "Routing accuracy": "Routing accuracy", + "agent default": "agent default", + "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.": "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.", + "Use the suite's agent": "Use the suite's agent", + "The agent the conversation opens on. Leave it unset to use the suite's.": "The agent the conversation opens on. Leave it unset to use the suite's.", + "History": "History", + "Add Message": "Add Message", + "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.": "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.", + "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.": "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.", + "Role": "Role", + "Content": "Content", + "Move up": "Move up", + "Move down": "Move down", + "Remove message": "Remove message", + "No history. The case starts from an empty conversation.": "No history. The case starts from an empty conversation.", + "History message {n} has no content.": "History message {n} has no content.", + "Assistant": "Assistant", + "Copy": "Copy", + "Copy case": "Copy case", + "Copied to \"{name}\". It is disabled until you enable it.": "Copied to \"{name}\". It is disabled until you enable it.", + "Failed to copy the case.": "Failed to copy the case.", + "Registration": "Registration", + "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.": "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.", + "Priority": "Priority", + "P0 runs in the stop-loss batch, P2 does not block a release.": "P0 runs in the stop-loss batch, P2 does not block a release.", + "Severity": "Severity", + "S0 is a stop, not a statistic. S2 is phrasing and experience.": "S0 is a stop, not a statistic. S2 is phrasing and experience.", + "Batch": "Batch", + "Derive from priority": "Derive from priority", + "Runs in batch {n}.": "Runs in batch {n}.", + "Cross-cutting": "Cross-cutting", + "Runs in every scope, whatever changed, and always in batch 1.": "Runs in every scope, whatever changed, and always in batch 1.", + "Involved agents": "Involved agents", + "Derived from the entry agent": "Derived from the entry agent", + "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.": "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.", + "Business domain": "Business domain", + "Last reviewed": "Last reviewed", + "Today": "Today", + "Never set automatically -- editing a case is not reviewing it.": "Never set automatically -- editing a case is not reviewing it.", + "Expected outcome": "Expected outcome", + "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.": "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.", + "Latency, tokens and cost": "Latency, tokens and cost", + "Model": "Model", + "Cases": "Cases", + "Latency P50": "Latency P50", + "Latency P95": "Latency P95", + "Tokens": "Tokens", + "Cost": "Cost", + "Unit cost (in / out)": "Unit cost (in / out)", + "unknown": "unknown", + "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.": "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.", + "Wall clock for the whole case": "Wall clock for the whole case", + "Agent call time only": "Agent call time only", + "Tokens / cost": "Tokens / cost", + "Plan a scope": "Plan a scope", + "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.": "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.", + "Changed agents": "Changed agents", + "All batches": "All batches", + "Platform-wide change": "Platform-wide change", + "Narrowing is switched off: no agent is demonstrably untouched.": "Narrowing is switched off: no agent is demonstrably untouched.", + "Plan": "Plan", + "Failed to work out the scope.": "Failed to work out the scope.", + "included": "included", + "excluded": "excluded", + "of {n} registered": "of {n} registered", + "In scope": "In scope", + "Suite": "Suite", + "Reason": "Reason", + "Yes": "Yes", + "No": "No", + "No involved agents known, so it was included to be safe.": "No involved agents known, so it was included to be safe." } diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 078783bf..9f52628c 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -640,8 +640,8 @@ "No mocks. Every tool call this case makes will be blocked.": "没有 mock。这条用例发起的每一次工具调用都会被阻断。", "A disabled suite cannot be run at all.": "停用的套件完全无法运行。", "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "这个套件已停用。在设置里重新启用之前，服务端会拒绝任何运行请求。", - "llmJudge assertions always fail in P1 regardless of these two fields.": "P1 阶段 llmJudge 断言恒定判失败，与这两个字段是否配置无关。", - "Always fails in P1.": "P1 阶段恒定判失败。", + "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.": "llmJudge 断言需要这两项。两项缺一，llmJudge 断言会记为 Error 而不是给分 —— 不会用默认模型代跑。", + "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.": "由测试集的判定模型按 1-5 打分，及格线默认 4。", "One function name per line. These run for real during a test -- only list functions with no side effects.": "每行一个函数名。这些函数在测试期间会真实执行——只填没有副作用的函数。", "One per line. Blocked even if allow-listed elsewhere.": "每行一个。即使在别处被放行也照样阻断。", "Assertions checked right after this turn": "本轮结束后立即求值的断言", @@ -733,5 +733,94 @@ "This run could not complete": "这次运行没能执行", "Nothing ran -- see the reason above.": "什么都没跑，原因见上方。", "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "这样跑不起来——你勾选的用例全部处于停用状态。请先至少启用一条。", - "{count} of the selected case(s) are disabled and will be skipped.": "勾选的用例里有 {count} 条处于停用状态，将被跳过。" + "{count} of the selected case(s) are disabled and will be skipped.": "勾选的用例里有 {count} 条处于停用状态，将被跳过。", + "Case type": "用例类型", + "Routing case": "路由用例", + "Agent case": "Agent 用例", + "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "单轮，从 Copilot 入口 agent 进入，只断言由哪个 agent 接走了这段对话。只有这一类会计入跑批的路由准确率。", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "单个 agent 自身的行为。直接从该 agent 进入，把 Router 排除在被测范围之外。", + "Entry agent": "入口 agent", + "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.": "路由类型的 agent 会走 Router 并可能交接；其他 agent 是直接进入的，Router 完全不参与。", + "A Routing case needs at least one routedToAgent or agentChain assertion.": "路由用例至少需要一条 routedToAgent 或 agentChain 断言。", + "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "路由用例不能用 llmJudge：路由只按由哪个 agent 处理了对话来判定。", + "Mode": "比较方式", + "Assertion \"agentChain\" mode must be one of {modes}.": "agentChain 断言的比较方式只能是 {modes} 之一。", + "Copilot, Work Order Creator": "Copilot, Work Order Creator", + "These agents appear somewhere in the chain, in any order.": "这些 agent 出现在链路中的任意位置，不论顺序。", + "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "这些 agent 按此顺序出现，中间允许有其他 agent。这条就是交接断言。", + "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.": "链路恰好是这些 agent，不多不少。只写一个名字就等于断言没有路由出去。", + "Agent chain": "Agent 链路", + "Answered by": "由谁回答", + "Routing accuracy": "路由准确率", + "agent default": "agent 默认模型", + "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.": "路由用例必须只有一轮。多轮请用 Agent 用例。", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.": "单个 agent 自身的行为。直接从该 agent 进入，把 Router 排除在被测范围之外。跨多个 agent 的旅程也算 Agent 用例 —— 交接用 agentChain 断言来表达。", + "Use the suite's agent": "使用测试集的 agent", + "The agent the conversation opens on. Leave it unset to use the suite's.": "对话从哪个 agent 开始。不选则使用测试集的。", + "History": "历史消息", + "Add Message": "新增消息", + "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.": "在第一轮开始前写入对话，把已有的一段问答变成这条用例固定的起始上下文。", + "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.": "这些消息不经过模型驱动，不消耗 token，也不会出现在 Agent 链路里。", + "Role": "角色", + "Content": "内容", + "Move up": "上移", + "Move down": "下移", + "Remove message": "删除消息", + "No history. The case starts from an empty conversation.": "没有历史消息，用例从空对话开始。", + "History message {n} has no content.": "第 {n} 条历史消息没有内容。", + "Assistant": "助手", + "Copy": "复制", + "Copy case": "复制用例", + "Copied to \"{name}\". It is disabled until you enable it.": "已复制为“{name}”，处于停用状态，需要你手动启用。", + "Failed to copy the case.": "复制用例失败。", + "Registration": "登记信息", + "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.": "决定这条用例什么时候跑、以及它失败意味着什么。用于推算一次变更实际需要跑哪些用例。", + "Priority": "优先级", + "P0 runs in the stop-loss batch, P2 does not block a release.": "P0 在止损批次里跑，P2 不阻塞发布。", + "Severity": "严重级别", + "S0 is a stop, not a statistic. S2 is phrasing and experience.": "S0 是止损，不是统计项。S2 是措辞和体验问题。", + "Batch": "批次", + "Derive from priority": "按优先级推导", + "Runs in batch {n}.": "将在第 {n} 批运行。", + "Cross-cutting": "跨切面", + "Runs in every scope, whatever changed, and always in batch 1.": "任何变更范围都会跑，且固定在第 1 批。", + "Involved agents": "涉及的 agent", + "Derived from the entry agent": "由入口 agent 推导", + "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.": "留空则使用入口 agent。路由用例值得填 —— 真正要紧的是 Router 下游的那些 agent。", + "Business domain": "业务域", + "Last reviewed": "最近复核", + "Today": "今天", + "Never set automatically -- editing a case is not reviewing it.": "永不自动填写 —— 改用例不等于复核用例。", + "Expected outcome": "期望结果", + "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.": "给看结果的人读的，永不参与判定 —— 机器能检查的期望结果就是断言，应该写在下面那些会被真正执行的地方。", + "Latency, tokens and cost": "延迟、Token 与成本", + "Model": "模型", + "Cases": "用例数", + "Latency P50": "延迟 P50", + "Latency P95": "延迟 P95", + "Tokens": "Token", + "Cost": "成本", + "Unit cost (in / out)": "单价（输入 / 输出）", + "unknown": "未知", + "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.": "百分位取最近秩，基于真正调用到模型的那些用例的 agent 调用耗时，所以每个数字都是某条用例真实花掉的时间。第一轮之前就失败的用例不计入百分位，但仍计入 Token 与成本。", + "Wall clock for the whole case": "整条用例的墙钟时间", + "Agent call time only": "仅 agent 调用耗时", + "Tokens / cost": "Token / 成本", + "Plan a scope": "规划评估范围", + "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.": "一次变更需要跑哪些用例、以及哪些不需要。只读 —— 它只做规划，不会启动跑批。", + "Changed agents": "变更涉及的 agent", + "All batches": "全部批次", + "Platform-wide change": "平台级变更", + "Narrowing is switched off: no agent is demonstrably untouched.": "关闭裁剪：没有哪个 agent 能被证明不受影响。", + "Plan": "规划", + "Failed to work out the scope.": "计算评估范围失败。", + "included": "纳入", + "excluded": "排除", + "of {n} registered": "共登记 {n} 条", + "In scope": "是否纳入", + "Suite": "测试集", + "Reason": "依据", + "Yes": "是", + "No": "否", + "No involved agents known, so it was included to be safe.": "涉及的 agent 未知，出于安全默认纳入。" } diff --git a/src/lib/services/agent-test-service.js b/src/lib/services/agent-test-service.js index cc41d747..4a7d3b8b 100644 --- a/src/lib/services/agent-test-service.js +++ b/src/lib/services/agent-test-service.js @@ -112,6 +112,40 @@ export async function updateCase(id, body) { return response.data; } +/** + * Work out which cases a change needs to run, before running anything. + * + * Read-only: it plans a run, it does not start one. Triggering stays per-suite, + * so a caller takes the included case ids from here and triggers each suite that + * appears among them. + * @param {{ targetAgentIds?: string[], fullPlatform?: boolean, batch?: number | null }} body + * @returns {Promise} + */ +export async function selectScope(body) { + const url = endpoints.agentTestScopeUrl; + const response = await axios.post(url, body); + return response.data; +} + +/** + * Duplicate an agent test case inside its own suite. + * + * Server-side rather than a get-then-create here, so the copy carries every + * field the case has. Rebuilding the payload from this client would drop + * anything it does not know about, and a copy missing its mocks looks identical + * in the list right up to the run where it blocks every tool. + * + * The copy lands disabled -- an exact duplicate joining the next run would + * measure the same thing twice. + * @param {string} id + * @returns {Promise} + */ +export async function copyCase(id) { + const url = endpoints.agentTestCaseCopyUrl.replace("{id}", id); + const response = await axios.post(url); + return response.data; +} + /** * Delete a test case * @param {string} id diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index 8984b2b8..b3ac4c19 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -143,10 +143,12 @@ export const endpoints = { agentTestSuiteRunUrl: `${host}/agent-test/suites/{id}/run`, agentTestCaseListUrl: `${host}/agent-test/cases`, agentTestCaseDetailUrl: `${host}/agent-test/cases/{id}`, + agentTestCaseCopyUrl: `${host}/agent-test/cases/{id}/copy`, agentTestRunListUrl: `${host}/agent-test/runs`, agentTestRunDetailUrl: `${host}/agent-test/runs/{id}`, agentTestRunCancelUrl: `${host}/agent-test/runs/{id}/cancel`, agentTestRecordUrl: `${host}/agent-test/record`, - agentTestMockTargetsUrl: `${host}/agent-test/mock-targets` + agentTestMockTargetsUrl: `${host}/agent-test/mock-targets`, + agentTestScopeUrl: `${host}/agent-test/scope` } diff --git a/src/routes/page/agent-test/+page.svelte b/src/routes/page/agent-test/+page.svelte index 9e3427be..ca2bcc03 100644 --- a/src/routes/page/agent-test/+page.svelte +++ b/src/routes/page/agent-test/+page.svelte @@ -9,8 +9,8 @@ import LoadingToComplete from '$lib/common/spinners/LoadingToComplete.svelte'; import Select from '$lib/common/dropdowns/Select.svelte'; import { getAgentOptions } from '$lib/services/agent-service.js'; - import { getSuites, createSuite, deleteSuite } from '$lib/services/agent-test-service.js'; - import { t } from '$lib/helpers/utils/agent-test.js'; + import { getSuites, createSuite, deleteSuite, selectScope } from '$lib/services/agent-test-service.js'; + import { errorMessage, t } from '$lib/helpers/utils/agent-test.js'; const duration = 3000; const nameMaxLength = 200; @@ -32,6 +32,62 @@ /** @type {import('$commonTypes').LabelValuePair[]} */ let agentOptions = $state([]); + /** + * Scope planning. Deliberately a separate panel and not part of triggering a run: working out + * what a change needs to test is a decision to review before spending anything, and folding it + * into a run button would mean the only way to see the plan is to have already paid for it. + * @type {{ targetAgentIds: string[], fullPlatform: boolean, batch: number | null }} + */ + let scopeQuery = $state({ targetAgentIds: [], fullPlatform: false, batch: null }); + + /** @type {import('$agentTestTypes').ScopeSelection | null} */ + let scope = $state(null); + let isScoping = $state(false); + let scopeError = $state(''); + + let canScope = $derived( + !isScoping && (scopeQuery.fullPlatform || scopeQuery.targetAgentIds.length > 0)); + + /** @param {any} e */ + function changeScopeAgents(e) { + scopeQuery.targetAgentIds = (e?.detail?.selecteds || []).map((/** @type {any} */ s) => s.value); + } + + function planScope() { + if (!canScope) return; + isScoping = true; + scopeError = ''; + selectScope({ + targetAgentIds: scopeQuery.targetAgentIds, + fullPlatform: scopeQuery.fullPlatform, + batch: scopeQuery.batch + }).then(res => { + scope = res; + }).catch(err => { + scope = null; + scopeError = errorMessage(err, t('Failed to work out the scope.')); + }).finally(() => { + isScoping = false; + }); + } + + /** + * Bootstrap class per reason. Included reasons are not all equal: unknownAgents means the harness + * could not show the change misses this case and included it to be safe, which is a nudge to fill + * in the metadata rather than a clean match. + * @param {string} reason + */ + function reasonColor(reason) { + switch (reason) { + case 'crossCutting': return 'info'; + case 'targetAgent': return 'success'; + case 'fullPlatform': return 'success'; + case 'unknownAgents': return 'warning'; + case 'disabled': return 'secondary'; + default: return 'light text-body'; + } + } + /** @type {Record} */ let agentNameById = $state({}); @@ -219,6 +275,125 @@ +
+
+
+
+
{$_('Plan a scope')}
+
+
+

+ {$_('Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.')} +

+
+
+ + + + + + + +
+
+
+ + +
{$_('Narrowing is switched off: no agent is demonstrably untouched.')}
+
+
+
+ +
+
+ + {#if scopeError} + + {/if} + + {#if scope} +
+
+ {scope.included.length} {$_('included')} + + {scope.excluded.length} {$_('excluded')} + + {$_('of {n} registered', { values: { n: scope.totalCases } })} +
+
+ + + + + + + + + + + + + + {#each [...scope.included, ...scope.excluded] as scoped (scoped.caseId)} + {@const included = scope.included.some(c => c.caseId === scoped.caseId)} + + + + + + + + + {/each} + +
{$_('In scope')}{$_('Case')}{$_('Suite')}{$_('Batch')}{$_('Severity')}{$_('Reason')}
+ {#if included} + {$_('Yes')} + {:else} + {$_('No')} + {/if} + {scoped.caseName}{scoped.suiteName}{scoped.batch}{scoped.severity} + {scoped.reason} + {#if scoped.reason === 'unknownAgents'} + + {$_('No involved agents known, so it was included to be safe.')} + + {/if} +
+
+
+ {/if} +
+
+
+
+ { + notifySuccess(t('Copied to "{name}". It is disabled until you enable it.', { name: copy.name })); + // Straight into the copy: it lands disabled and named "(copy)", and neither is + // something anyone leaves as it is -- a copy is made in order to be edited. + goToCase(copy.id); + }).catch(err => { + notifyError(errorMessage(err, t('Failed to copy the case.'))); + isLoading = false; + }); + } + function goToNewCase() { goto(`/page/agent-test/${suiteId}/case/new`); } @@ -353,18 +368,15 @@ /** @param {import('$agentTestTypes').AgentTestCase} testCase */ function toggleCaseEnabled(testCase) { isLoading = true; - // Full replace again: everything the editor does not touch here still has - // to travel, or flipping the toggle wipes turns/mocks/assertions. + // PUT is a full replace, so everything has to travel or flipping this toggle wipes it. + // Spread rather than listed field by field: a hand-written payload silently drops each + // field added to a case afterwards, which is how caseType, entryAgentId and history all + // came to be erased by a single click on this toggle. Server-owned keys (id, createDate, + // updateDate) ride along harmlessly -- the upsert request has no such properties, so model + // binding ignores them. updateCase(testCase.id, { - suiteId: testCase.suiteId, - name: testCase.name, - enabled: !testCase.enabled, - turns: testCase.turns || [], - assertions: testCase.assertions || [], - initialStates: testCase.initialStates || [], - mocks: testCase.mocks || [], - unmockedToolPolicy: testCase.unmockedToolPolicy || 'Block', - sourceConversationId: testCase.sourceConversationId + ...testCase, + enabled: !testCase.enabled }).then(() => { notifySuccess(testCase.enabled ? t('Case disabled.') : t('Case enabled.')); return loadCases(); @@ -675,7 +687,9 @@ /> {$_('Name')} + {$_('Type')} {$_('Turns')} + {$_('History')} {$_('Mocks')} {$_('Assertions')} {$_('Source')} @@ -706,7 +720,19 @@ {testCase.name} + + + {#if testCase.caseType === 'Routing'} + {$_('Routing case')} + {:else} + {$_('Agent case')} + {/if} + {testCase.turns?.length || 0} + {testCase.history?.length || 0} {testCase.mocks?.length || 0} {assertionCount} @@ -750,6 +776,16 @@ +
  • + +
  • + +
    {$_('Never set automatically -- editing a case is not reviewing it.')}
    + +
    + + +
    + {$_('For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.')} +
    +
    + + + + + + +
    +
    +
    +
    +
    +
    {$_('History')} ({form.history.length})
    + +
    +
    +
    +

    + {$_('Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.')} + {$_('These messages are not driven through the model, cost no tokens, and never appear in the agent chain.')} +

    + {#each form.history as message, i} +
    +
    +
    + + +
    +
    + + +
    +
    + + + +
    +
    +
    + {/each} + {#if form.history.length === 0} +

    {$_('No history. The case starts from an empty conversation.')}

    + {/if} +
    +
    +
    +
    +
    diff --git a/src/routes/page/agent-test/run/[runId]/+page.svelte b/src/routes/page/agent-test/run/[runId]/+page.svelte index 26c09148..9a5376d9 100644 --- a/src/routes/page/agent-test/run/[runId]/+page.svelte +++ b/src/routes/page/agent-test/run/[runId]/+page.svelte @@ -7,7 +7,15 @@ import HeadTitle from '$lib/common/shared/HeadTitle.svelte'; import LoadingToComplete from '$lib/common/spinners/LoadingToComplete.svelte'; import { getRun, cancelRun, triggerRun } from '$lib/services/agent-test-service.js'; - import { statusColor, isTerminalStatus, errorMessage, formatDuration, formatDateTime, t } from '$lib/helpers/utils/agent-test.js'; + import { + statusColor, + isTerminalStatus, + errorMessage, + formatDuration, + formatDateTime, + formatAgentChain, + t + } from '$lib/helpers/utils/agent-test.js'; const duration = 3000; const pollIntervalMs = 2000; @@ -52,6 +60,37 @@ /** Column order for the comparison grid; follows the run's own model order. */ let modelColumns = $derived((run?.models || []).map(m => ({ ...m, key: `${m.provider}/${m.model}` }))); + + /** + * Per-model routing accuracy, straight off the run. Shown as "passed / total" next to + * the percentage on purpose: the framework gate is expressed in percentage points, but + * 3/4 says how much the figure is worth trusting and 75% does not -- and with a handful + * of routing cases that distinction is the difference between a gate and a guess. + */ + /** + * Latency, token and cost figures per model, as stored on the run. Rendered as a table + * rather than tiles because there is one row per model and the whole point is comparing + * them side by side. + */ + let performance = $derived((run?.performanceSummaries || []).map(s => ({ + ...s, + label: s.model || $_('agent default') + }))); + + /** + * The unit costs the cost column was computed from, keyed for lookup. Shown next to the + * figures because a cost is not comparable with another run's unless these match -- a + * provider price change would otherwise read as a cost regression with nothing to point at. + */ + let pricingByModel = $derived(Object.fromEntries( + (run?.modelPricing || []).map(p => [p.model || '', p]) + )); + + let routingAccuracies = $derived((run?.routingAccuracies || []).map(a => ({ + ...a, + label: a.model || $_('agent default'), + percent: a.caseCount > 0 ? Math.round((a.passedCount / a.caseCount) * 1000) / 10 : null + }))); let isComparison = $derived(modelColumns.length > 1); /** @@ -298,6 +337,70 @@
    + {#if routingAccuracies.length > 0} + +
    +
    {$_('Routing accuracy')}
    +
    + {#each routingAccuracies as accuracy (accuracy.label)} +
    +
    {accuracy.label}
    + {accuracy.percent}% + {accuracy.passedCount}/{accuracy.caseCount} +
    + {/each} +
    +
    + {/if} + {#if performance.length > 0} +
    +
    {$_('Latency, tokens and cost')}
    +
    + + + + + + + + + + + + + + {#each performance as row (row.label)} + {@const pricing = pricingByModel[row.model || '']} + + + + + + + + + + {/each} + +
    {$_('Model')}{$_('Cases')}{$_('Latency P50')}{$_('Latency P95')}{$_('Tokens')}{$_('Cost')}{$_('Unit cost (in / out)')}
    {row.label}{row.caseCount}{formatDuration(row.latencyP50Ms)}{formatDuration(row.latencyP95Ms)}{row.totalTokens}{row.totalCost.toFixed(4)} + {#if pricing && pricing.textInputCost != null} + {pricing.textInputCost} / {pricing.textOutputCost} + {:else} + + {$_('unknown')} + {/if} +
    +
    +
    + {$_('Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.')} +
    +
    + {/if}
    @@ -436,7 +539,33 @@ which model produced which row. --> {result.model} {/if} - {formatDuration(result.durationMs)} + + {formatDuration(result.durationMs)} + + {#if result.modelDurationMs > 0} + + + ({formatDuration(result.modelDurationMs)}) + + {/if} + {#if result.totalTokens > 0} + + {result.totalTokens} · {result.cost.toFixed(4)} + + {/if} + {#if result.agentChain?.length > 0} + + + {formatAgentChain(result.agentChain)} + + {/if} {#if result.conversationId} {result.conversationId} {/if} @@ -473,6 +602,12 @@
    {$_('Agent output')}
    {turn.output || '--'}
    + {#if turn.agentChain?.length > 0} +
    +
    {$_('Answered by')}
    +
    {formatAgentChain(turn.agentChain)}
    +
    + {/if} {#if turn.assertions?.length > 0} {@render assertionTable(turn.assertions)} {:else} From 0081cb9e55555f414eb603e06c11a57a41c30966 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Fri, 21 Aug 2026 10:11:16 +0800 Subject: [PATCH 2/5] Run history: select-all and delete A checkbox column and a select-all in the run history table, mirroring the case table above it, plus a delete on each finished row and a bulk delete for the selection. The button count excludes runs that are still executing, and says so next to itself, because the server refuses those -- a count that included them would promise a delete that never happens. On a live row the only action offered is Cancel: deleting a run that is still going would not stop it, so the other button would not do what it says. Skipped runs are surfaced one by one rather than folded into the success message. A toast saying "deleted 2" while a third row stays put is how someone concludes the button is broken. Reloading the list drops selections for runs that are gone, so a later delete cannot post ids the server can only answer "already deleted" for. Co-Authored-By: Claude Opus 5 --- src/lib/helpers/types/agentTestTypes.js | 10 ++ src/lib/langs/en.json | 12 +- src/lib/langs/zh.json | 12 +- src/lib/services/agent-test-service.js | 18 ++ src/lib/services/api-endpoints.js | 1 + .../page/agent-test/[suiteId]/+page.svelte | 155 +++++++++++++++++- 6 files changed, 198 insertions(+), 10 deletions(-) diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 06627f4b..738d3883 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -368,6 +368,16 @@ * @property {ScopedCase[]} excluded */ +/** + * What clearing run history actually did. The skipped half matters: a still-running run is left + * alone, and the caller has to be able to say why one survived rather than leaving the user to + * notice a row that quietly stayed. + * @typedef {Object} RunDeleteResult + * @property {string[]} deletedRunIds + * @property {number} deletedResultCount - Case results removed along with those runs. + * @property {{ runId: string, reason: string }[]} skipped + */ + /** * Body of GET /agent-test/runs/{id} (the camelCase projection of the backend's * AgentTestRunDetailDto). diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index eb4877d6..35b23d83 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -635,5 +635,15 @@ "Reason": "Reason", "Yes": "Yes", "No": "No", - "No involved agents known, so it was included to be safe.": "No involved agents known, so it was included to be safe." + "No involved agents known, so it was included to be safe.": "No involved agents known, so it was included to be safe.", + "Select all runs": "Select all runs", + "Select run": "Select run", + "Delete selected": "Delete selected", + "{n} still running and cannot be deleted": "{n} still running and cannot be deleted", + "Delete run": "Delete run", + "Delete this run and its case results? You won't be able to revert this!": "Delete this run and its case results? You won't be able to revert this!", + "Delete {n} runs and all their case results? You won't be able to revert this!": "Delete {n} runs and all their case results? You won't be able to revert this!", + "Deleted {runs} run(s) and {results} result(s).": "Deleted {runs} run(s) and {results} result(s).", + "Run {id} was kept: {reason}": "Run {id} was kept: {reason}", + "Failed to delete the runs.": "Failed to delete the runs." } diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 98d5796e..85b1d90d 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -829,5 +829,15 @@ "Reason": "依据", "Yes": "是", "No": "否", - "No involved agents known, so it was included to be safe.": "涉及的 agent 未知，出于安全默认纳入。" + "No involved agents known, so it was included to be safe.": "涉及的 agent 未知，出于安全默认纳入。", + "Select all runs": "全选跑批记录", + "Select run": "选择这条跑批", + "Delete selected": "删除所选", + "{n} still running and cannot be deleted": "{n} 条仍在运行，无法删除", + "Delete run": "删除跑批记录", + "Delete this run and its case results? You won't be able to revert this!": "删除这条跑批记录及其全部用例结果？此操作不可撤销！", + "Delete {n} runs and all their case results? You won't be able to revert this!": "删除 {n} 条跑批记录及其全部用例结果？此操作不可撤销！", + "Deleted {runs} run(s) and {results} result(s).": "已删除 {runs} 条跑批记录、{results} 条用例结果。", + "Run {id} was kept: {reason}": "跑批 {id} 未删除：{reason}", + "Failed to delete the runs.": "删除跑批记录失败。" } diff --git a/src/lib/services/agent-test-service.js b/src/lib/services/agent-test-service.js index 4a7d3b8b..0329b477 100644 --- a/src/lib/services/agent-test-service.js +++ b/src/lib/services/agent-test-service.js @@ -127,6 +127,24 @@ export async function selectScope(body) { return response.data; } +/** + * Clear run history: delete the named runs and the case results underneath them. + * + * Bulk-only on the server, and the single-row button calls it with one id -- the + * guard that refuses to delete a still-running run is worth having in exactly one + * place. + * + * A run that is still executing is skipped rather than failing the whole call, so + * the response reports what actually went and what did not. + * @param {string[]} runIds + * @returns {Promise} + */ +export async function deleteRuns(runIds) { + const url = endpoints.agentTestRunDeleteUrl; + const response = await axios.post(url, { runIds: runIds }); + return response.data; +} + /** * Duplicate an agent test case inside its own suite. * diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index 8ed217f4..96be6d21 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -144,6 +144,7 @@ export const endpoints = { agentTestRunListUrl: `${host}/agent-test/runs`, agentTestRunDetailUrl: `${host}/agent-test/runs/{id}`, agentTestRunCancelUrl: `${host}/agent-test/runs/{id}/cancel`, + agentTestRunDeleteUrl: `${host}/agent-test/runs/delete`, agentTestRecordUrl: `${host}/agent-test/record`, agentTestMockTargetsUrl: `${host}/agent-test/mock-targets`, agentTestScopeUrl: `${host}/agent-test/scope` diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte index 920fc9a0..e2395258 100644 --- a/src/routes/page/agent-test/[suiteId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -22,6 +22,7 @@ getRuns, triggerRun, cancelRun, + deleteRuns, recordCases } from '$lib/services/agent-test-service.js'; import { statusColor, isTerminalStatus, errorMessage, formatDateTime, formatDuration, t } from '$lib/helpers/utils/agent-test.js'; @@ -68,6 +69,13 @@ /** @type {import('$agentTestTypes').AgentTestCase | null} */ let caseToDelete = $state(null); + /** @type {string[]} */ + let selectedRunIds = $state([]); + + /** Runs the confirm dialog is currently asking about; empty means it is closed. */ + /** @type {import('$agentTestTypes').AgentTestRun[]} */ + let runsToDelete = $state([]); + /** true = the run modal was opened by "Run Selected", false = "Run All Enabled". */ let runPartial = $state(false); @@ -172,6 +180,14 @@ let canSaveSettings = $derived(!!settingsDraft.name?.trim() && settingsDraft.caseTimeoutSeconds > 0); let canRecord = $derived(!!recordConversationId?.trim()); let allCasesSelected = $derived(cases.length > 0 && selectedCaseIds.length === cases.length); + let allRunsSelected = $derived(runs.length > 0 && selectedRunIds.length === runs.length); + + /** + * Selected runs that can actually go. A run still executing is refused by the server, so counting + * it here would promise a delete that will not happen. + */ + let deletableSelectedRuns = $derived( + runs.filter(r => selectedRunIds.includes(r.id) && isTerminalStatus(r.status))); onMount(async () => { isLoading = true; @@ -259,6 +275,10 @@ function loadRuns() { return getRuns(suiteId).then(res => { runs = res || []; + // Drop selections for runs that are gone, or a later delete would post ids the server + // can only answer "already deleted" for. + const ids = new Set(runs.map(r => r.id)); + selectedRunIds = selectedRunIds.filter(id => ids.has(id)); }).catch(() => { runs = []; }); @@ -318,6 +338,55 @@ selectedCaseIds = allCasesSelected ? [] : cases.map(x => x.id); } + /** @param {string} runId */ + function toggleRun(runId) { + selectedRunIds = selectedRunIds.includes(runId) + ? selectedRunIds.filter(id => id !== runId) + : [...selectedRunIds, runId]; + } + + function toggleAllRuns() { + selectedRunIds = allRunsSelected ? [] : runs.map(x => x.id); + } + + /** @param {import('$agentTestTypes').AgentTestRun[]} target */ + function openDeleteRunsModal(target) { + runsToDelete = target; + } + + function closeDeleteRunsModal() { + runsToDelete = []; + } + + function confirmDeleteRuns() { + const target = runsToDelete.map(r => r.id); + runsToDelete = []; + if (target.length === 0) return; + + isLoading = true; + deleteRuns(target).then(res => { + const deleted = res?.deletedRunIds?.length || 0; + if (deleted > 0) { + notifySuccess(t('Deleted {runs} run(s) and {results} result(s).', { + runs: deleted, + results: res?.deletedResultCount || 0 + })); + } + // Reported, not swallowed: a still-running run is left alone on purpose, and the reason + // says what to do about it. Silently showing "deleted 2" while a third row stays put is + // how a user concludes the button is broken. + (res?.skipped || []).forEach((/** @type {any} */ s) => { + notifyError(t('Run {id} was kept: {reason}', { id: s.runId.substring(0, 8), reason: s.reason })); + }); + selectedRunIds = []; + return loadRuns(); + }).catch(err => { + notifyError(errorMessage(err, t('Failed to delete the runs.'))); + }).finally(() => { + isLoading = false; + }); + } + function goBack() { goto('/page/agent-test'); } @@ -596,6 +665,21 @@ errorText={errorText} /> + 0} + icon="warning" + title={t('Are you sure?')} + text={runsToDelete.length === 1 + ? t('Delete this run and its case results? You won\'t be able to revert this!') + : t('Delete {n} runs and all their case results? You won\'t be able to revert this!', { n: runsToDelete.length })} + confirmBtnText={t('Yes, delete it!')} + cancelBtnText={t('Cancel')} + confirmBtnColor="danger" + confirm={confirmDeleteRuns} + cancel={closeDeleteRunsModal} + toggleModal={closeDeleteRunsModal} +/> +
    - +
    + {#if selectedRunIds.length > 0} + + {#if deletableSelectedRuns.length < selectedRunIds.length} + + + {$_('{n} still running and cannot be deleted', { + values: { n: selectedRunIds.length - deletableSelectedRuns.length } + })} + + {/if} + {/if} + +
    @@ -909,6 +1016,15 @@ + @@ -920,6 +1036,15 @@ {#each runs as run (run.id)} + + + @@ -884,6 +896,24 @@ {$_('Agent case')} {/if} + + diff --git a/src/routes/page/agent-test/scope/+page.svelte b/src/routes/page/agent-test/scope/+page.svelte index ddbfde88..1e8a16d7 100644 --- a/src/routes/page/agent-test/scope/+page.svelte +++ b/src/routes/page/agent-test/scope/+page.svelte @@ -8,7 +8,7 @@ import Select from '$lib/common/dropdowns/Select.svelte'; import { getAgentOptions } from '$lib/services/agent-service.js'; import { selectScope } from '$lib/services/agent-test-service.js'; - import { errorMessage, t } from '$lib/helpers/utils/agent-test.js'; + import { errorMessage, severityTone, t } from '$lib/helpers/utils/agent-test.js'; /** * Planning which cases a change needs to run is a different job from browsing suites, and it @@ -227,7 +227,11 @@ - +
    + toggleAllRuns()} + /> + {$_('Status')} {$_('Result')} {$_('Scope')}
    + toggleRun(run.id)} + /> + {$_(run.status)} @@ -970,7 +1095,21 @@ - {#if !isTerminalStatus(run.status)} + {#if isTerminalStatus(run.status)} +
  • + +
    • + {:else} +
  • - - - - {#if scopeError} - - {/if} - - {#if scope} -
    -
    - {scope.included.length} {$_('included')} - - {scope.excluded.length} {$_('excluded')} - - {$_('of {n} registered', { values: { n: scope.totalCases } })} -
    -
    - - - - - - - - - - - - - - {#each [...scope.included, ...scope.excluded] as scoped (scoped.caseId)} - {@const included = scope.included.some(c => c.caseId === scoped.caseId)} - - - - - - - - - {/each} - -
    {$_('In scope')}{$_('Case')}{$_('Suite')}{$_('Batch')}{$_('Severity')}{$_('Reason')}
    - {#if included} - {$_('Yes')} - {:else} - {$_('No')} - {/if} - {scoped.caseName}{scoped.suiteName}{scoped.batch}{scoped.severity} - {scoped.reason} - {#if scoped.reason === 'unknownAgents'} - - {$_('No involved agents known, so it was included to be safe.')} - - {/if} -
    -
    -
    - {/if} - - - - - - +
    + + + + {$_('Plan a scope')} + + +
    diff --git a/src/routes/page/agent-test/scope/+page.svelte b/src/routes/page/agent-test/scope/+page.svelte new file mode 100644 index 00000000..ddbfde88 --- /dev/null +++ b/src/routes/page/agent-test/scope/+page.svelte @@ -0,0 +1,251 @@ + + + + + + + +
    +
    +
    +
    +
    +
    + + + +
    +
    {$_('Plan a scope')}
    +

    + {$_('Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.')} +

    +
    +
    + +
    +
    +
    +
    +
    + + + + + + + +
    +
    +
    + + +
    +

    + {$_('Narrowing is switched off: no agent is demonstrably untouched.')} +

    +
    +
    + +
    +
    + + {#if errorText} + + {/if} +
    +
    +
    +
    + +{#if scope} +
    +
    +
    +
    +
    + + + +
    +
    {$_('Evaluation scope')}
    +

    + {scope.included.length} + {$_('included')} · + {scope.excluded.length} + {$_('excluded')} · + {$_('of {n} registered', { values: { n: scope.totalCases } })} +

    +
    +
    +
    +
    + {#if scope.totalCases === 0} +
    +

    {$_('No test cases are registered yet.')}

    +
    + {:else} +
    + + + + + + + + + + + + + + {#each [...scope.included, ...scope.excluded] as scoped (scoped.caseId)} + {@const included = scope.included.some(c => c.caseId === scoped.caseId)} + + + + + + + + + {/each} + +
    {$_('In scope')}{$_('Case')}{$_('Suite')}{$_('Batch')}{$_('Severity')}{$_('Reason')}
    + {#if included} + {$_('Yes')} + {:else} + {$_('No')} + {/if} + {scoped.caseName}{scoped.suiteName}{scoped.batch}{scoped.severity} + + {scoped.reason} + + {#if scoped.reason === 'unknownAgents'} + + {$_('No involved agents known, so it was included to be safe.')} + + {/if} +
    +
    + {/if} +
    +
    +
    +
    +{/if} diff --git a/svelte.config.js b/svelte.config.js index 130d3715..b1979395 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -78,6 +78,7 @@ const config = { "/page/knowledge-base/dictionary", "/page/knowledge-base/[embed]/[embedType]", "/page/agent-test", + "/page/agent-test/scope", "/page/agent-test/[suiteId]", "/page/agent-test/[suiteId]/case/[caseId]", "/page/agent-test/run/[runId]" From 79c28e1c75b2b1e386794c68b6355a8bdd06a96c Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Fri, 21 Aug 2026 10:41:46 +0800 Subject: [PATCH 4/5] Show priority and severity in the case list Two columns on the case table, so the decision of what needs running can be made from the list instead of by opening every case. The priority cell also shows the batch the case will actually run in, and that is the part that matters. Cross-cutting overrides priority and forces batch 1, so a P2 safety case runs FIRST -- a bare "P2" would read as "runs last", which is the opposite of the truth and precisely the call this column exists to inform. Tones are chosen so the list can be skimmed rather than read. P0 and S0 are loud: P0 is the stop-loss batch where one failure halts the evaluation, and S0 is zero tolerance -- data leakage, an unauthorised action, a missed escalation. P1 stays neutral rather than amber, because it is the default every untriaged case carries and a wall of warning colour would say nothing at all. S2 is quiet and must never look as loud as the other two. The scope page shows severity as a badge now too, from the same helper, so a severity means the same thing wherever it appears. No backend change: the case list already returns these fields. Co-Authored-By: Claude Opus 5 --- src/lib/helpers/utils/agent-test.js | 30 +++++++++++++++++ src/lib/langs/en.json | 3 +- src/lib/langs/zh.json | 3 +- .../page/agent-test/[suiteId]/+page.svelte | 32 ++++++++++++++++++- src/routes/page/agent-test/scope/+page.svelte | 8 +++-- 5 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js index 26d0df00..67a078bd 100644 --- a/src/lib/helpers/utils/agent-test.js +++ b/src/lib/helpers/utils/agent-test.js @@ -91,6 +91,36 @@ export const CASE_SEVERITIES = ['S0', 'S1', 'S2']; /** Batches run in order: 1 is the stop-loss batch, 3 does not block a release. */ export const CASE_BATCHES = [1, 2, 3]; +/** + * Tone for a priority badge. P0 is the stop-loss batch -- one failure there halts the + * whole evaluation -- so it reads as urgent; P2 does not block a release and reads as + * quiet. The middle stays neutral rather than warning-coloured, because P1 is the + * default every untriaged case carries and a wall of amber would say nothing. + * @param {string?} priority + */ +export function priorityTone(priority) { + switch (priority) { + case 'P0': return 'danger'; + case 'P2': return 'secondary'; + default: return 'primary'; + } +} + +/** + * Tone for a severity badge. S0 is zero-tolerance -- data leakage, an unauthorised + * action, a missed escalation -- and has to be visible at a glance in a list, which is + * the whole reason severity is worth a column. S2 is phrasing and must never look as + * loud as the other two. + * @param {string?} severity + */ +export function severityTone(severity) { + switch (severity) { + case 'S0': return 'danger'; + case 'S1': return 'warning'; + default: return 'secondary'; + } +} + /** * The batch a case will actually run in, mirroring CaseBatches.Effective. An * explicit batch wins; a cross-cutting case is batch 1 whatever its priority, diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 6a40a495..0424ea5d 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -647,5 +647,6 @@ "Run {id} was kept: {reason}": "Run {id} was kept: {reason}", "Failed to delete the runs.": "Failed to delete the runs.", "Evaluation scope": "Evaluation scope", - "No test cases are registered yet.": "No test cases are registered yet." + "No test cases are registered yet.": "No test cases are registered yet.", + "batch {n}": "batch {n}" } diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index b0115f3c..05dae298 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -841,5 +841,6 @@ "Run {id} was kept: {reason}": "跑批 {id} 未删除：{reason}", "Failed to delete the runs.": "删除跑批记录失败。", "Evaluation scope": "评估范围", - "No test cases are registered yet.": "还没有登记任何测试用例。" + "No test cases are registered yet.": "还没有登记任何测试用例。", + "batch {n}": "第 {n} 批" } diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte index e2395258..42d1f9d7 100644 --- a/src/routes/page/agent-test/[suiteId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -25,7 +25,17 @@ deleteRuns, recordCases } from '$lib/services/agent-test-service.js'; - import { statusColor, isTerminalStatus, errorMessage, formatDateTime, formatDuration, t } from '$lib/helpers/utils/agent-test.js'; + import { + statusColor, + isTerminalStatus, + priorityTone, + severityTone, + effectiveBatch, + errorMessage, + formatDateTime, + formatDuration, + t + } from '$lib/helpers/utils/agent-test.js'; const duration = 3000; const nameMaxLength = 200; @@ -841,6 +851,8 @@
    		• {$_('Name')} {$_('Type')}{$_('Priority')}{$_('Severity')} {$_('Turns')} {$_('History')} {$_('Mocks')} + + {testCase.priority || 'P1'} + + +
    + {$_('batch {n}', { values: { n: effectiveBatch(testCase) } })} +
    +     		+ + {testCase.severity || 'S1'} + + {testCase.turns?.length || 0} {testCase.history?.length || 0} {testCase.mocks?.length || 0}{scoped.caseName} {scoped.suiteName} {scoped.batch}{scoped.severity} + + {scoped.severity} + + {scoped.reason} From da74abe738c46bebb5f7fc4f8f0382863499b9d2 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Mon, 24 Aug 2026 16:11:12 +0800 Subject: [PATCH 5/5] Agent test UI: chat panel for authoring/editing a case Adds a collapsible "Write by Chat" column to the case editor: a message log, an instruction box (Enter to send), a model picker defaulting to the suite's judge model with an inline warning when neither is set, per-turn "changed by chat" badges on the sections a reply touched, and local-only undo (a draft snapshot stack, no request involved). The chat's target is the same `form` state the hand-editing UI already uses, so validation, payload building and the save button are untouched. "New Case by Chat" on the suite page opens the editor straight into this mode via `?chat=1`. Co-Authored-By: Claude Sonnet 5 --- src/lib/helpers/types/agentTestTypes.js | 40 ++ src/lib/langs/en.json | 19 +- src/lib/langs/zh.json | 19 +- src/lib/services/agent-test-service.js | 17 + src/lib/services/api-endpoints.js | 1 + src/lib/styles/pages/_agent-test.scss | 55 ++ .../page/agent-test/[suiteId]/+page.svelte | 17 +- .../[suiteId]/case/[caseId]/+page.svelte | 479 ++++++++++++++++-- 8 files changed, 588 insertions(+), 59 deletions(-) diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 738d3883..7def88b1 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -386,4 +386,44 @@ * @property {AgentTestCaseResult[]} results */ +/** + * Body of POST /agent-test/author. Carries the whole conversation and the whole draft because the + * server keeps no authoring session -- there is no draft row to orphan when a tab is closed. + * @typedef {Object} AgentTestAuthorRequest + * @property {string} suiteId + * @property {string?} [caseId] - The case being edited; null while creating. Supplied so the backend + * can ground the model in that case's most recent run -- what the agent really replied and what + * arguments it really passed -- instead of letting it invent both. + * @property {AuthorChatMessage[]} messages - Oldest first; the last one is the new instruction. + * @property {AgentTestCaseUpsertRequest?} [draft] - The draft as it stands, or null to start empty. + * @property {TestModel?} [model] - Omit to use the suite's judge model. + */ + +/** + * @typedef {Object} AuthorChatMessage + * @property {string} role - user | assistant. + * @property {string} content + */ + +/** + * One changed field of a draft. + * @typedef {Object} AuthorChange + * @property {string} field - camelCase draft field name, e.g. "turns". + * @property {string} detail - Short human summary, e.g. "3 -> 4 item(s)". + */ + +/** + * Result of one authoring turn. + * @typedef {Object} AgentTestAuthorResult + * @property {string} reply - What to show in the chat. + * @property {AgentTestCaseUpsertRequest} draft - Always populated, so the client can assign it + * unconditionally. + * @property {boolean} draftChanged + * @property {AuthorChange[]} changes - Computed by the backend by diffing the two drafts, never read + * off the model's own account of what it did. + * @property {string[]} validationErrors - Non-empty means the draft cannot be saved as it stands. + * @property {string[]} warnings - Silently wrong things that were corrected (a mock for a function + * this agent cannot call) or are merely suspect and were left alone (an unfamiliar state key). + */ + export default {}; diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 0424ea5d..084812fa 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -648,5 +648,22 @@ "Failed to delete the runs.": "Failed to delete the runs.", "Evaluation scope": "Evaluation scope", "No test cases are registered yet.": "No test cases are registered yet.", - "batch {n}": "batch {n}" + "batch {n}": "batch {n}", + "Write by Chat": "Write by Chat", + "Show": "Show", + "Hide": "Hide", + "Undo": "Undo", + "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.": "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.", + "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.": "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.", + "Working on it...": "Working on it...", + "Your instruction": "Your instruction", + "Add a turn, change an assertion, ask what a field means...": "Add a turn, change an assertion, ask what a field means...", + "Enter sends. Shift+Enter starts a new line.": "Enter sends. Shift+Enter starts a new line.", + "Send": "Send", + "Changed in this reply": "Changed in this reply", + "The assistant could not produce a valid draft:": "The assistant could not produce a valid draft:", + "Worth checking:": "Worth checking:", + "changed by chat": "changed by chat", + "New Case by Chat": "New Case by Chat", + "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above.": "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above." } diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 05dae298..ab7f6f34 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -842,5 +842,22 @@ "Failed to delete the runs.": "删除跑批记录失败。", "Evaluation scope": "评估范围", "No test cases are registered yet.": "还没有登记任何测试用例。", - "batch {n}": "第 {n} 批" + "batch {n}": "第 {n} 批", + "Write by Chat": "用对话写", + "Show": "展开", + "Hide": "收起", + "Undo": "撤销", + "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.": "用自己的话说说这个用例应该覆盖什么。每次回复都会直接改这页上的草稿——按下保存之前什么都不会存进去。", + "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.": "例如：住户说冰箱在漏水，问什么时候有人来；应该转到工单 agent 并查一下工单。", + "Working on it...": "处理中……", + "Your instruction": "你的指令", + "Add a turn, change an assertion, ask what a field means...": "加一轮对话、改一条断言、或者问某个字段是什么意思……", + "Enter sends. Shift+Enter starts a new line.": "Enter 发送，Shift+Enter 换行。", + "Send": "发送", + "Changed in this reply": "本次回复改动了", + "The assistant could not produce a valid draft:": "助手没能给出一份能保存的草稿：", + "Worth checking:": "建议确认一下：", + "changed by chat": "对话改动过", + "New Case by Chat": "用对话新建", + "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above.": "这个套件没有配置判官模型，对话出题也就没有默认可用的模型——请在上面选一个。" } diff --git a/src/lib/services/agent-test-service.js b/src/lib/services/agent-test-service.js index 0329b477..f35d0839 100644 --- a/src/lib/services/agent-test-service.js +++ b/src/lib/services/agent-test-service.js @@ -260,3 +260,20 @@ export async function getMockTargets(agentId) { }); return response.data; } + +/** + * One turn of authoring a case by conversation. + * + * Stateless on the server: the whole chat and the whole current draft go up every time, and what + * comes back replaces the draft. Nothing is saved -- the returned draft still has to go through + * createCase/updateCase, which is the only path that validates the entry agent and the only one the + * user presses deliberately. + * + * @param {import('$agentTestTypes').AgentTestAuthorRequest} body + * @returns {Promise} + */ +export async function authorCase(body) { + const url = endpoints.agentTestCaseAuthorUrl; + const response = await axios.post(url, body); + return response.data; +} diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index 96be6d21..71dac76e 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -146,6 +146,7 @@ export const endpoints = { agentTestRunCancelUrl: `${host}/agent-test/runs/{id}/cancel`, agentTestRunDeleteUrl: `${host}/agent-test/runs/delete`, agentTestRecordUrl: `${host}/agent-test/record`, + agentTestCaseAuthorUrl: `${host}/agent-test/author`, agentTestMockTargetsUrl: `${host}/agent-test/mock-targets`, agentTestScopeUrl: `${host}/agent-test/scope` } diff --git a/src/lib/styles/pages/_agent-test.scss b/src/lib/styles/pages/_agent-test.scss index 9a19511e..1f098c9d 100644 --- a/src/lib/styles/pages/_agent-test.scss +++ b/src/lib/styles/pages/_agent-test.scss @@ -878,3 +878,58 @@ font-size: 0.875rem; color: var(--color-muted); } + +/* ======================================================================== + * Authoring chat (case editor right-hand column) + * ======================================================================== + * A transcript, not a chat product: no avatars, no timestamps, no streaming. + * The two roles are told apart by ground and alignment only, because the + * thing worth looking at is the form next to it -- the chat is the input + * device, not the output. + * + * The log is height-capped and scrolls on its own so that a long + * conversation cannot push the composer off the bottom of a sticky column, + * which is exactly where it stops being usable. + */ +.ats-chat-log { + display: flex; + flex-direction: column; + gap: 0.5rem; + max-height: 22rem; + margin: 0.75rem 0; + overflow-y: auto; + padding-right: 0.25rem; +} + +.ats-chat-msg { + max-width: 92%; + padding: 0.5rem 0.75rem; + border-radius: 0.75rem; + font-size: 0.8125rem; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-word; +} + +.ats-chat-msg-user { + align-self: flex-end; + background-color: color-mix(in srgb, var(--color-primary) 12%, transparent); + color: rgb(31 41 55); + + .dark & { + color: rgb(229 231 235); + } +} + +.ats-chat-msg-agent { + align-self: flex-start; + border: 1px solid rgb(229 231 235); + background-color: rgb(249 250 251); + color: rgb(31 41 55); + + .dark & { + border-color: rgb(55 65 81); + background-color: rgb(17 24 39 / 0.6); + color: rgb(229 231 235); + } +} diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte index 42d1f9d7..b433a8d3 100644 --- a/src/routes/page/agent-test/[suiteId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -424,6 +424,15 @@ goto(`/page/agent-test/${suiteId}/case/new`); } + /** + * Same editor, with the authoring chat already open -- see the `chat` query parameter there. + * A separate entry rather than a mode of its own: whatever the chat produces still has to be + * reviewed and saved in the normal editor, so landing anywhere else would only add a hop. + */ + function goToNewCaseWithChat() { + goto(`/page/agent-test/${suiteId}/case/new?chat=1`); + } + /** @param {string} runId */ function goToRun(runId) { goto(`/page/agent-test/run/${runId}`); @@ -756,6 +765,9 @@ + @@ -827,7 +839,10 @@

    {$_('No test cases in this suite yet.')}

    - +
    {:else} -
    + +
    +
    +
    +
    +
    +
    + + + +
    {$_('Write by Chat')}
    +
    +
    + {#if draftHistory.length > 0} + + {/if} + +
    +
    +
    + + {#if isChatOpen} +
    +

    + {$_('Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.')} +

    + +
    + {$_('Model')} + + +
    + {$_('Enter sends. Shift+Enter starts a new line.')} + +
    + + {#if authorErrorText} + + {/if} + + {#if authorChanges.length > 0} + +
    +

    {$_('Changed in this reply')}

    +
      + {#each authorChanges as change} +
    • {change.field} — {change.detail}
      • + {/each} +
    +
    + {/if} + + {#if authorErrors.length > 0} + + {/if} + + {#if authorWarnings.length > 0} + + {/if} +
    + {/if} +
    +
    + +
    @@ -825,7 +1166,12 @@
    -
    {$_('Turns')}
    +
    + {$_('Turns')} + {#if changedFields.includes('turns')} + {$_('changed by chat')} + {/if} +

    {form.turns.length}

    @@ -914,7 +1260,12 @@
    -
    {$_('Case Assertions')}
    +
    + {$_('Case Assertions')} + {#if changedFields.includes('assertions')} + {$_('changed by chat')} + {/if} +

    {form.assertions.length}

    @@ -1070,7 +1421,12 @@
    -
    {$_('History')}
    +
    + {$_('History')} + {#if changedFields.includes('history')} + {$_('changed by chat')} + {/if} +

    {form.history.length}

    @@ -1157,7 +1513,12 @@
    -
    {$_('Initial States')}
    +
    + {$_('Initial States')} + {#if changedFields.includes('initialStates')} + {$_('changed by chat')} + {/if} +

    {form.initialStates.length}

    @@ -1191,7 +1552,12 @@
    -
    {$_('Tool Mocks')}
    +
    + {$_('Tool Mocks')} + {#if changedFields.includes('mocks')} + {$_('changed by chat')} + {/if} +

    {form.mocks.length}

    @@ -1325,5 +1691,6 @@
    + {/if}