diff --git a/src/observability/dashboard/state/feed.js b/src/observability/dashboard/state/feed.js index d686b0c..110c325 100644 --- a/src/observability/dashboard/state/feed.js +++ b/src/observability/dashboard/state/feed.js @@ -30,9 +30,12 @@ export function buildActivityFeed(runs) { for (const ev of run.events ?? []) { if (skipInFeed.has(ev.type)) continue; - const summary = plainLanguageSummary(ev); + // July harness vocabulary (#215) gets first-class summaries here; + // everything else keeps the shared plain-language path. Unknown event + // types still degrade exactly as before: no summary, no feed line. + const summary = opsEventSummary(ev) ?? plainLanguageSummary(ev); if (!summary) continue; - activityFeed.push({ + const item = { ts: ev.ts, summary, type: ev.type, @@ -40,7 +43,13 @@ export function buildActivityFeed(runs) { projectRoot: run.projectRoot, goal: run.manifest?.goal, level: eventLevel(ev), - }); + }; + // Structured detail for the client panels (retry state, guardrail + // depth, audit rejections) — only real fields from the event, never + // fabricated defaults. + const data = opsEventData(ev); + if (data) item.data = data; + activityFeed.push(item); } } @@ -49,6 +58,86 @@ export function buildActivityFeed(runs) { .slice(0, 200); } +// Plain-English lines for the loop-engineering signals shipped 2026-07 +// (#156 remainder + #215): checkpoints, context pressure, approval audit +// rejections, memory write decisions, metrics drift, retry decisions and +// goal-loop evaluations. Returns null for anything it does not own. +function opsEventSummary(ev) { + switch (ev.type) { + case 'stage_checkpoint_before_saved': + return checkpointSummary('before', ev); + case 'stage_checkpoint_after_saved': + return checkpointSummary('after', ev); + case 'context_pressure_warning': { + const size = fmtCount(ev.size); + const threshold = fmtCount(ev.threshold); + const metric = ev.metric ?? 'chars'; + return `Context pressure — ${ev.source ?? 'context'} at ${size} ${metric} (threshold ${threshold}); warning only, nothing was pruned`; + } + case 'approval_audit_failed': + return `Approval record rejected by audit — ${ev.artifact ?? 'unknown artifact'} treated as absent; the gate stayed closed`; + case 'episode_memory_written': + return `Episode memory written for ${ev.task_id ?? 'run'}${ev.trusted === true ? ' (trusted)' : ev.trusted === false ? ' (stored untrusted — never injected into prompts)' : ''}`; + case 'episode_memory_skipped_untrusted': + return `Episode memory skipped for ${ev.task_id ?? 'task'} — ${ev.reason ?? 'write policy refused the episode'}${ev.write_policy ? ` (policy: ${ev.write_policy})` : ''}`; + case 'metrics_write_failed': + return `Metrics write failed${ev.operation ? ` (${ev.operation})` : ''} — persisted totals are behind the events; run totals fall back to event recompute`; + case 'retry_decision': + // Pinned #123 emitter contract: { task_id, stage_id, attempt, + // max_attempts, retry_recommendation, action, next_status, reason, + // issues[] } with action ∈ complete|retry|exhausted|human_context|block. + return `Retry decision for ${ev.task_id ?? 'task'}: ${ev.action ?? '?'}${ev.next_status ? ` → ${ev.next_status}` : ''}${ev.reason ? ` — ${ev.reason}` : ''}`; + // goal_evaluated keeps its shared plain-language summary (goal_id / + // status / score / reason — the real pipeline-loop.js emitter fields); + // the ops layer only attaches the structured data below. + default: + return null; + } +} + +function checkpointSummary(phase, ev) { + const verified = ev.verified === true + ? 'verified restorable' + : ev.verified === false ? 'NOT verified — restore may fail' : 'verification unknown'; + return `Checkpoint saved ${phase} ${ev.stage_id ?? 'stage'} (${verified})`; +} + +function fmtCount(value) { + const num = Number(value); + return Number.isFinite(num) ? num.toLocaleString('en-US') : '?'; +} + +// Structured per-type detail carried on feed items so the Alerts & +// Guardrails / Approvals panels can render depth (limit vs value, attempt +// counts, audit issues) without a second data path. Only fields that exist +// on the event are copied — absent fields stay absent. +const OPS_EVENT_DATA_FIELDS = { + guardrail_triggered: ['task_id', 'limit_name', 'current_value', 'limit_value', 'reason'], + guardrail_overridden: ['task_id', 'artifact'], + task_retry_scheduled: ['task_id', 'stage_id', 'attempt', 'max_attempts', 'retry_recommendation', 'reason'], + task_retry_exhausted: ['task_id', 'stage_id', 'attempt', 'max_attempts', 'retry_recommendation', 'reason'], + task_human_context_required: ['task_id', 'stage_id', 'attempt', 'max_attempts', 'retry_recommendation', 'reason'], + retry_decision: ['task_id', 'stage_id', 'attempt', 'max_attempts', 'retry_recommendation', 'action', 'next_status', 'reason'], + context_pressure_warning: ['task_id', 'source', 'metric', 'size', 'threshold', 'blocking', 'stage_id'], + approval_audit_failed: ['record_id', 'artifact', 'status', 'issues', 'reason'], + stage_checkpoint_before_saved: ['stage_id', 'task_id', 'verified'], + stage_checkpoint_after_saved: ['stage_id', 'task_id', 'verified'], + episode_memory_written: ['task_id', 'trusted'], + episode_memory_skipped_untrusted: ['task_id', 'reason', 'write_policy'], + metrics_write_failed: ['task_id', 'operation', 'error'], + goal_evaluated: ['iteration', 'max_iterations', 'goal_id', 'status', 'score', 'critical_count', 'failing_stages', 'recommended_rerun_stages'], +}; + +function opsEventData(ev) { + const fields = OPS_EVENT_DATA_FIELDS[ev.type]; + if (!fields) return null; + const data = {}; + for (const field of fields) { + if (ev[field] !== undefined) data[field] = ev[field]; + } + return Object.keys(data).length ? data : null; +} + function eventLevel(ev) { if (ev.type === 'task_validated' && ev.status === 'FAIL') return 'fail'; if (ev.type === 'validation_failed') return 'fail'; @@ -59,7 +148,23 @@ function eventLevel(ev) { if (ev.type === 'task_retry_exhausted') return 'fail'; if (ev.type === 'task_human_context_required') return 'blocked'; if (ev.type === 'task_blocked_by_validator') return 'blocked'; + if (ev.type === 'retry_decision') { + // next_status is the task transition the harness actually made (#123): + // PASS | FAIL (re-claimable) | BLOCKED | NEEDS_CONTEXT. + const nextStatus = String(ev.next_status ?? '').toUpperCase(); + if (nextStatus === 'BLOCKED' || nextStatus === 'NEEDS_CONTEXT') return 'blocked'; + if (nextStatus === 'PASS') return 'pass'; + return 'warn'; + } if (/^retry_/.test(String(ev.type ?? ''))) return 'warn'; + if (ev.type === 'stage_checkpoint_before_saved' || ev.type === 'stage_checkpoint_after_saved') { + return ev.verified === false ? 'warn' : 'info'; + } + if (ev.type === 'context_pressure_warning') return 'warn'; + if (ev.type === 'approval_audit_failed') return 'fail'; + if (ev.type === 'episode_memory_written') return 'info'; + if (ev.type === 'episode_memory_skipped_untrusted') return 'warn'; + if (ev.type === 'metrics_write_failed') return 'warn'; if (ev.type === 'goal_evaluated') return ev.status === 'PASS' ? 'pass' : 'warn'; if (ev.type === 'loop_iteration_retrying_stages') return 'warn'; if (ev.type === 'loop_completed') return 'pass'; diff --git a/src/observability/dashboard/ui/pages/alerts-guardrails.js b/src/observability/dashboard/ui/pages/alerts-guardrails.js index 4dc6901..73ffb43 100644 --- a/src/observability/dashboard/ui/pages/alerts-guardrails.js +++ b/src/observability/dashboard/ui/pages/alerts-guardrails.js @@ -3,9 +3,34 @@ // Alerts & Guardrails page module — renders into #page-alerts-guardrails. Plain client JS // concatenated into the served bundle by ui/client.js; self-registers its // renderer with the page registry (ui/lib.js). +// +// Loop-engineering depth (#156 remainder + #215 ops slice): retry state per +// task, guardrail triggers with limit-vs-value and override status, and +// context-pressure warnings. All three panels are derived from the recent +// server-composed event feed plus task/approval state already in the +// snapshot — nothing is fabricated, and each panel says when it is empty. export const alertsGuardrailsScript = ` // ── page: alerts-guardrails ──────────────────────────────────────────────── +// Panel injection uses opsEnsureSection, declared in pages/live-feed.js — the +// earliest-concatenated wave:ops module in the served bundle. +var OPS_ALERTS_PANELS_HTML = + '
' + + '
Retry State
' + + '
' + + '
Derived from the recent event stream: retry scheduling, budget exhaustion and human-context pauses per task.
' + + '
' + + '
' + + '
' + + '
Guardrail Triggers
' + + '
' + + '
' + + '
' + + '
Context Pressure Warnings
' + + '
' + + '
' + + '
'; + function renderAlertsGuardrails(s) { var alerts = s.alerts || []; var blocked = s.blockedGates || []; @@ -15,15 +40,186 @@ function renderAlertsGuardrails(s) { setHTML('blocked-list', blocked.map(function(gate) { return '
' + esc(gate.title) + '
' + esc(gate.detail) + '
' + esc(gate.runId || '') + '' + esc(fmtTime(gate.ts)) + '
'; }).join('') || emptyHtml('No blocked gates', 'Blocked approval gate history appears here.')); + + opsEnsureSection('alerts-guardrails', 'ops-retry-panel', OPS_ALERTS_PANELS_HTML); + renderOpsRetryPanel(s); + renderOpsGuardrailPanel(s); + renderOpsPressurePanel(s); } function alertHtml(alert) { return '
' + esc(alert.title || alert.type || 'Alert') + '
' + esc(alert.detail || '') + '
' + esc(alert.type || '') + '' + esc(alert.runId || '') + '
' + pill(alert.level || 'info') + '
'; } +// ── Retry state (#156: retry traces) ──────────────────────────────────────── +var OPS_RETRY_EVENT_TYPES = { + task_retry_scheduled: 'scheduled', + task_retry_exhausted: 'exhausted', + task_human_context_required: 'human_required' +}; + +// retry_decision action → panel state (pinned #123 contract: +// action ∈ complete|retry|exhausted|human_context|block). +var OPS_DECISION_STATES = { + retry: 'scheduled', + exhausted: 'exhausted', + human_context: 'human_required', + block: 'validator_blocked', + complete: 'resolved' +}; + +function opsRetryStateLabel(state) { + if (state === 'scheduled') return pill('warn', 'retry scheduled'); + if (state === 'exhausted') return pill('fail', 'budget exhausted'); + if (state === 'human_required') return pill('blocked', 'human input required'); + if (state === 'validator_blocked') return pill('blocked', 'validator blocked'); + if (state === 'resolved') return pill('pass', 'resolved'); + if (state === 'blocked_unknown') return pill('blocked', 'blocked'); + return pill('info', state || 'unknown'); +} + +// Latest retry signal per task for one run, folded from the recent feed. +// Oldest-first; at EQUAL timestamps the action-specific event +// (task_retry_exhausted / _scheduled / _human_context_required) outranks the +// paired retry_decision — the emitter appends both in the same tick, decision +// first, and the specific event is the final word on the state. Without this +// tiebreak an exhausted task could render under the decision's mapping alone. +function opsRetryStateForRun(run, feed) { + var items = []; + feed.forEach(function(item) { + if (item.runId !== run.runId) return; + if (item.type !== 'retry_decision' && !OPS_RETRY_EVENT_TYPES[item.type]) return; + if (!item.data || !item.data.task_id) return; + items.push(item); + }); + items.sort(function(a, b) { + var byTs = String(a.ts || '').localeCompare(String(b.ts || '')); + if (byTs !== 0) return byTs; + return (a.type === 'retry_decision' ? 0 : 1) - (b.type === 'retry_decision' ? 0 : 1); + }); + var byTask = {}; + items.forEach(function(item) { + var d = item.data; + var prev = byTask[d.task_id] || {}; + if (item.type === 'retry_decision') { + byTask[d.task_id] = { + state: OPS_DECISION_STATES[String(d.action || '').toLowerCase()] || prev.state || 'unknown', + attempt: d.attempt != null ? d.attempt : prev.attempt, + maxAttempts: d.max_attempts != null ? d.max_attempts : prev.maxAttempts, + reason: d.reason || prev.reason, + action: d.action || prev.action, + nextStatus: d.next_status || prev.nextStatus, + ts: item.ts + }; + } else { + byTask[d.task_id] = { + state: OPS_RETRY_EVENT_TYPES[item.type], + attempt: d.attempt != null ? d.attempt : prev.attempt, + maxAttempts: d.max_attempts != null ? d.max_attempts : prev.maxAttempts, + reason: d.reason || prev.reason, + action: prev.action, + nextStatus: prev.nextStatus, + ts: item.ts + }; + } + }); + return byTask; +} + +function opsRetryRowHtml(run, task, info) { + var attempts = info && info.attempt != null + ? 'attempt ' + info.attempt + (info.maxAttempts != null ? '/' + info.maxAttempts : '') + : ''; + var decisionMeta = info && (info.action || info.nextStatus) + ? 'decision: ' + (info.action || '?') + (info.nextStatus ? ' → ' + info.nextStatus : '') + : ''; + return '
' + + '
' + esc(task.id) + (task.stageId ? ' (' + esc(task.stageId) + ')' : '') + '
' + + (info && info.reason ? '
' + esc(info.reason) + '
' : '') + + (!info ? '
Task is BLOCKED — cause outside the recent event window (validator, DoR, destructive-gate or attempt budget; no retry evidence in view).
' : '') + + '
' + esc((run.runId || '').slice(-16)) + '' + (attempts ? '' + esc(attempts) + '' : '') + (decisionMeta ? '' + esc(decisionMeta) + '' : '') + 'task status: ' + esc(task.status || 'READY') + '
' + + // "budget exhausted" is earned only by real task_retry_exhausted / + // exhausted-decision evidence — a BLOCKED task without in-window events + // gets the generic blocked pill, never a fabricated cause. + '
' + opsRetryStateLabel(info ? info.state : 'blocked_unknown') + '
'; +} + +function renderOpsRetryPanel(s) { + var feed = s.feed || []; + var rows = []; + (s.runs || []).forEach(function(run) { + var byTask = opsRetryStateForRun(run, feed); + (run.tasks || []).forEach(function(task) { + var info = byTask[task.id] || null; + if (!info && task.status !== 'BLOCKED') return; + rows.push(opsRetryRowHtml(run, task, info)); + }); + }); + setText('ops-retry-count', rows.length + ' task(s) in retry flow'); + setHTML('ops-retry-list', rows.join('') || emptyHtml('No retry activity', 'No task in the recent event window has needed a second attempt or hit its attempt budget.')); +} + +// ── Guardrail triggers + override status (#156: guardrail depth) ──────────── +function opsOverrideStatusHtml(runId, taskId, s) { + if (!taskId) return '
No task id on this trigger — cannot look up an override.
'; + var artifact = 'guardrail-override:' + taskId; + var consumed = (s.feed || []).some(function(item) { + return item.type === 'guardrail_overridden' && item.runId === runId && item.data && item.data.task_id === taskId; + }); + if (consumed) return '
Override consumed — exactly one extra attempt was granted, then the gate re-armed.
'; + var record = null; + (s.runs || []).forEach(function(run) { + if (run.runId !== runId) return; + (run.approvals || []).forEach(function(approval) { + if (approval.artifact === artifact) record = approval; + }); + }); + if (!record) return '
No override on file — the task stays blocked until a ' + esc(artifact) + ' approval is granted.
'; + var status = String(record.status || '').toUpperCase(); + if (status === 'APPROVED') return '
Override approved' + (record.approver ? ' by ' + esc(record.approver) : '') + ' — the next claim gets exactly one attempt.
'; + if (status === 'REJECTED') return '
Override rejected' + (record.approver ? ' by ' + esc(record.approver) : '') + ' — the task stays blocked.
'; + return '
Override request on file (status: ' + esc(record.status || 'pending') + ').
'; +} + +function renderOpsGuardrailPanel(s) { + var items = (s.feed || []).filter(function(item) { return item.type === 'guardrail_triggered'; }).slice(0, 20); + setText('ops-guardrail-count', items.length + ' trigger(s)'); + setHTML('ops-guardrail-list', items.map(function(item) { + var d = item.data || {}; + var limit = d.limit_name || 'guardrail'; + var values = d.current_value != null && d.limit_value != null + ? 'value ' + d.current_value + ' hit limit ' + d.limit_value + : (d.reason || 'limit reached'); + return '
' + + '
' + esc(limit) + '
' + + '
' + esc(values) + (d.task_id ? ' — task ' + esc(d.task_id) : '') + '
' + + '
' + esc((item.runId || '').slice(-16)) + '' + esc(fmtTime(item.ts)) + '
' + + opsOverrideStatusHtml(item.runId, d.task_id, s) + + '
' + pill('blocked', 'blocked') + '
'; + }).join('') || emptyHtml('No guardrail triggers', 'No guardrail has blocked a task in the recent event window.')); +} + +// ── Context pressure warnings (#215 / #211) ───────────────────────────────── +function renderOpsPressurePanel(s) { + var items = (s.feed || []).filter(function(item) { return item.type === 'context_pressure_warning'; }).slice(0, 20); + setText('ops-pressure-count', items.length + ' warning(s)'); + setHTML('ops-pressure-list', items.map(function(item) { + var d = item.data || {}; + var size = d.size != null && d.threshold != null + ? d.size + ' vs threshold ' + d.threshold + (d.metric ? ' ' + d.metric : '') + : 'size unavailable'; + return '
' + + '
' + esc(d.source || 'context') + '
' + + '
' + esc(size) + (d.task_id ? ' — task ' + esc(d.task_id) : '') + (d.stage_id ? ' (' + esc(d.stage_id) + ')' : '') + '
' + + '
' + esc((item.runId || '').slice(-16)) + '' + esc(fmtTime(item.ts)) + '
' + + '
Detect-only warning — nothing was pruned or truncated.
' + + '
' + pill('warn', 'pressure') + '
'; + }).join('') || emptyHtml('No context pressure warnings', 'Builder context stayed under every configured threshold in the recent event window.')); +} + registerPage('alerts-guardrails', { errLabel: 'alerts', - sub: 'Threshold alerts, blocked gates, guardrails, stalled work and spend risks.', + sub: 'Threshold alerts, blocked gates, guardrails, retry state, context pressure, stalled work and spend risks.', unscoped: true, render: renderAlertsGuardrails }); diff --git a/src/observability/dashboard/ui/pages/approvals.js b/src/observability/dashboard/ui/pages/approvals.js index a20b163..440e321 100644 --- a/src/observability/dashboard/ui/pages/approvals.js +++ b/src/observability/dashboard/ui/pages/approvals.js @@ -6,6 +6,19 @@ export const approvalsScript = ` // ── page: approvals ──────────────────────────────────────────────── +// Audit rejections (#215 / #133): approval records are a trust boundary — +// a record that fails the consistency audit is treated as absent and the +// gate stays closed. That rejection must be visible here, never silent: +// this panel is the tampering-visibility surface. +var OPS_AUDIT_PANEL_HTML = + '
' + + '
Audit Rejections
' + + '
' + + '
Every approval record is audited before it is trusted. A record listed here failed the consistency audit and was treated as absent — the gate stayed closed and the gated work did not proceed. A rejection can mean corruption, drift, or a forged record: review it before re-approving.
' + + '
' + + '
' + + '
'; + function renderApprovals(s) { var approvals = s.approvals || []; var pending = approvals.filter(function(item) { return !item.status || item.status === 'pending'; }); @@ -13,6 +26,28 @@ function renderApprovals(s) { setText('approvals-count', pending.length + ' pending'); setHTML('approvals-list', pending.map(function(item) { return approvalHtml(item, true); }).join('') || emptyHtml('No pending approvals', 'Only queue-backed approvals appear here.')); setHTML('approvals-resolved', resolved.slice(0, 20).map(function(item) { return approvalHtml(item, false); }).join('') || emptyHtml('No resolved approvals', 'Approved and rejected queue entries appear here.')); + opsEnsureSection('approvals', 'ops-audit-panel', OPS_AUDIT_PANEL_HTML); + renderOpsAuditRejections(s); +} + +function opsAuditRejectionHtml(item) { + var d = item.data || {}; + // This panel renders records that already failed a trust audit — the + // issues field itself is hostile input. A forged non-array value is shown + // as a single line instead of throwing into the page error banner. + var issues = Array.isArray(d.issues) ? d.issues : (d.issues != null ? [String(d.issues)] : []); + return '
' + + '
' + esc(d.artifact || 'unknown artifact') + '
' + + '
' + esc(d.reason || 'Record failed the consistency audit — treated as absent; the gate stayed closed.') + '
' + + (issues.length ? '
    ' + issues.map(function(issue) { return '
  • ' + esc(issue) + '
    • '; }).join('') + '
' : '') + + '
' + (d.record_id ? 'record ' + esc(d.record_id) + '' : '') + (d.status ? 'claimed status: ' + esc(d.status) + '' : '') + '' + esc((item.runId || '').slice(-16)) + '' + esc(fmtTime(item.ts)) + '
' + + '
' + pill('fail', 'rejected') + '
'; +} + +function renderOpsAuditRejections(s) { + var items = (s.feed || []).filter(function(item) { return item.type === 'approval_audit_failed'; }).slice(0, 20); + setText('ops-audit-count', items.length + ' rejection(s)'); + setHTML('ops-audit-list', items.map(opsAuditRejectionHtml).join('') || emptyHtml('No audit rejections', 'Every approval record read in the recent event window passed the consistency audit.')); } function approvalHtml(item, canAct) { diff --git a/src/observability/dashboard/ui/pages/live-feed.js b/src/observability/dashboard/ui/pages/live-feed.js index 5b0e605..c0b90b3 100644 --- a/src/observability/dashboard/ui/pages/live-feed.js +++ b/src/observability/dashboard/ui/pages/live-feed.js @@ -6,10 +6,93 @@ export const liveFeedScript = ` // ── page: live-feed ──────────────────────────────────────────────── +// [wave:ops] shared helper. Declared here because live-feed.js is the +// EARLIEST-concatenated of the wave:ops page modules in ui/client.js +// (live-feed → approvals → alerts-guardrails), so the later ops modules can +// call it without leaning on whole-bundle function hoisting. Injects a +// section once into a page body: page modules own their panels but not +// ui/pages/index.js, so late-added panels mount themselves on first render. +function opsEnsureSection(pageId, markerId, html) { + if (document.getElementById(markerId)) return; + var page = document.getElementById('page-' + pageId); + if (!page) return; + page.insertAdjacentHTML('beforeend', html); +} + +// July harness vocabulary (#215) gets distinct glyphs so the audit story +// reads at a glance: checkpoints (CP), context pressure (CX), approval audit +// (AU), memory writes (MB), metrics drift (MX), retries (RT), guardrails +// (GR), goal loop (GL). Anything not listed falls back to the shared +// level-based feed row — unknown event types keep degrading gracefully. +var OPS_FEED_ICONS = { + stage_checkpoint_before_saved: 'CP', + stage_checkpoint_after_saved: 'CP', + context_pressure_warning: 'CX', + approval_audit_failed: 'AU', + episode_memory_written: 'MB', + episode_memory_skipped_untrusted: 'MB', + metrics_write_failed: 'MX', + retry_decision: 'RT', + task_retry_scheduled: 'RT', + task_retry_exhausted: 'RT', + task_human_context_required: 'RT', + guardrail_triggered: 'GR', + guardrail_overridden: 'GR', + goal_evaluated: 'GL' +}; + +// One compact detail chip per event type, read from the structured data the +// server feed attaches (state/feed.js). Empty string when the event carried +// no detail — nothing is fabricated. +function opsFeedMeta(item) { + var d = item.data || {}; + switch (item.type) { + case 'guardrail_triggered': + if (!d.limit_name) return ''; + return d.limit_name + (d.current_value != null && d.limit_value != null ? ': ' + d.current_value + ' of ' + d.limit_value : ''); + case 'context_pressure_warning': + if (d.size == null || d.threshold == null) return ''; + return d.size + ' vs ' + d.threshold + (d.metric ? ' ' + d.metric : ''); + case 'stage_checkpoint_before_saved': + case 'stage_checkpoint_after_saved': + return d.verified === true ? 'verified' : d.verified === false ? 'unverified' : ''; + case 'approval_audit_failed': + // Forged records can carry a non-array issues field — count only real lists. + return Array.isArray(d.issues) && d.issues.length ? d.issues.length + ' audit issue(s)' : ''; + case 'retry_decision': + // Pinned #123 shape: action ∈ complete|retry|exhausted|human_context|block, + // next_status is the task transition the harness made. + if (!d.action && !d.next_status) return ''; + return (d.action || '?') + (d.next_status ? ' → ' + d.next_status : ''); + case 'task_retry_scheduled': + case 'task_retry_exhausted': + case 'task_human_context_required': + return d.attempt != null && d.max_attempts != null ? 'attempt ' + d.attempt + '/' + d.max_attempts : ''; + case 'episode_memory_written': + return d.trusted === true ? 'trusted' : d.trusted === false ? 'untrusted' : ''; + case 'episode_memory_skipped_untrusted': + return d.write_policy || ''; + case 'goal_evaluated': + // Real pipeline-loop.js fields: iteration/max_iterations + failing_stages. + if (d.iteration != null) return 'iteration ' + d.iteration + (d.max_iterations != null ? '/' + d.max_iterations : ''); + return Array.isArray(d.failing_stages) && d.failing_stages.length ? d.failing_stages.length + ' failing stage(s)' : ''; + default: + return ''; + } +} + +function opsFeedRowHtml(item) { + var icon = OPS_FEED_ICONS[item.type]; + if (!icon) return feedRowHtml(item); + var level = item.level || 'info'; + var extra = opsFeedMeta(item); + return '
' + esc(icon) + '
' + esc(item.summary || '') + '
' + (item.runId ? '' + esc(item.runId.slice(-14)) + '' : '') + (item.projectRoot ? '' + esc(shortName(item.projectRoot)) + '' : '') + (item.type ? '' + esc(item.type) + '' : '') + (extra ? '' + esc(extra) + '' : '') + '
' + esc(fmtTime(item.ts)) + '
'; +} + function renderLiveFeed(s) { var feed = s.feed || []; setText('live-feed-count', feed.length + ' events'); - setHTML('live-feed-list', feed.length ? feed.map(feedRowHtml).join('') : emptyHtml('No events yet', 'Live event data appears here.')); + setHTML('live-feed-list', feed.length ? feed.map(opsFeedRowHtml).join('') : emptyHtml('No events yet', 'Live event data appears here.')); } registerPage('live-feed', { diff --git a/src/observability/dashboard/ui/styles.js b/src/observability/dashboard/ui/styles.js index 6d642ab..8fffa1a 100644 --- a/src/observability/dashboard/ui/styles.js +++ b/src/observability/dashboard/ui/styles.js @@ -1269,4 +1269,13 @@ tr.clickable:hover td { background: #f8fbff; } .kv-note { font-size: 11px; color: var(--muted); margin-top: 8px; font-style: italic; } @media (max-width: 900px) { .report-kpis { grid-template-columns: repeat(2, minmax(0, 1fr)); } } + +/* [wave:ops] — ops panels: retry state, guardrail depth, context pressure, audit rejections */ +.feed-icon.info { color: var(--blue); background: #eff6ff; } +.ops-meta { color: var(--amber); font-weight: 700; } +.ops-note { font-size: 11px; color: var(--muted); font-style: italic; margin-top: 8px; } +.ops-issues { margin: 6px 0 0; padding-left: 18px; font-size: 12px; color: var(--muted); } +.ops-issues li { padding: 2px 0; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } +.alert-card.fail { border-left: 4px solid var(--red); } +/* [/wave:ops] */ `; diff --git a/tests/dashboard-ops-signals.test.js b/tests/dashboard-ops-signals.test.js new file mode 100644 index 0000000..e0460cc --- /dev/null +++ b/tests/dashboard-ops-signals.test.js @@ -0,0 +1,357 @@ +/** + * Ops surfaces for the July harness signals (#156 remainder + #215): + * - state/feed.js renders plain-English lines (+ structured data) for the + * new event vocabulary: checkpoints, context pressure, approval audit + * rejections, memory write decisions, metrics drift, retry decisions and + * goal evaluations — while unknown event types keep degrading gracefully; + * - pages/alerts-guardrails.js renders the Retry State / Guardrail Triggers / + * Context Pressure panels from fixture-shaped state; + * - pages/approvals.js renders the Audit Rejections panel (a rejected or + * forged approval record must be visible, never silent); + * - every panel has an honest empty state. + * + * The page renderers are real client code living inside template literals, + * so these tests execute them against a minimal DOM stub instead of only + * string-matching the source. + * + * owner: RStack developed by Richardson Gunde + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { buildActivityFeed } from '../src/observability/dashboard/state/feed.js'; +import { libScript } from '../src/observability/dashboard/ui/lib.js'; +import { liveFeedScript } from '../src/observability/dashboard/ui/pages/live-feed.js'; +import { approvalsScript } from '../src/observability/dashboard/ui/pages/approvals.js'; +import { alertsGuardrailsScript } from '../src/observability/dashboard/ui/pages/alerts-guardrails.js'; + +// Fixture-shaped events — field-for-field the shapes the harness emits +// (mirrors .rstack/runs/…-ui-fixture-july-signals/events.jsonl + HARNESS.md). +const JULY_EVENTS = [ + { type: 'stage_checkpoint_before_saved', stage_id: '06-architecture', task_id: '003-architecture', verified: true, ts: '2026-07-06T12:01:05.000Z' }, + { type: 'stage_checkpoint_after_saved', stage_id: '06-architecture', task_id: '003-architecture', verified: false, ts: '2026-07-06T12:10:05.000Z' }, + { type: 'context_pressure_warning', source: 'memory_summary', metric: 'chars', size: 61000, threshold: 40000, blocking: false, task_id: '004-implementation', ts: '2026-07-06T12:13:00.000Z' }, + { type: 'task_retry_scheduled', task_id: '005-testing', attempt: 1, max_attempts: 2, reason: 'validator requested another attempt', ts: '2026-07-06T12:31:00.000Z' }, + { type: 'retry_decision', task_id: '005-testing', stage_id: '08-testing', attempt: 2, max_attempts: 2, retry_recommendation: 'retry_builder', action: 'exhausted', next_status: 'BLOCKED', reason: 'attempt budget exhausted', issues: [], ts: '2026-07-06T12:35:00.000Z' }, + { type: 'guardrail_triggered', task_id: '005-testing', limit_name: 'maxTaskAttempts', current_value: 2, limit_value: 2, ts: '2026-07-06T12:35:01.000Z' }, + { type: 'approval_audit_failed', record_id: 'forged-1', artifact: 'guardrail-override:005-testing', status: 'APPROVED', issues: ['approval_actor_present: approver missing or empty'], reason: 'approval record failed the consistency audit — treated as absent, gated work stays gated', ts: '2026-07-06T12:36:00.000Z' }, + { type: 'episode_memory_written', task_id: '003-architecture', trusted: true, ts: '2026-07-06T12:37:00.000Z' }, + { type: 'episode_memory_skipped_untrusted', task_id: '005-testing', reason: 'validator status FAIL under validator-approved-only', write_policy: 'validator-approved-only', ts: '2026-07-06T12:38:00.000Z' }, + { type: 'metrics_write_failed', task_id: '004-implementation', operation: 'telemetry_increment', error: 'EACCES', ts: '2026-07-06T12:39:00.000Z' }, + { type: 'goal_evaluated', iteration: 1, max_iterations: 3, goal_id: 'ship-the-mvp', status: 'FAIL', score: 0.5, critical_count: 1, failing_stages: ['08-testing'], recommended_rerun_stages: ['07-code', '08-testing'], reason: 'test coverage below target', ts: '2026-07-06T12:40:00.000Z' }, + // Unknown event type: must not throw and must not fabricate a feed line. + { type: 'mystery_signal_from_the_future', task_id: '005-testing', ts: '2026-07-06T12:41:00.000Z' }, +]; + +const FIXTURE_RUN = { + runId: 'run-july-signals', + projectRoot: '/tmp/project', + manifest: { goal: 'July signals fixture' }, + events: JULY_EVENTS, +}; + +function fixtureState() { + return { + runs: [{ + runId: 'run-july-signals', + projectRoot: '/tmp/project', + manifest: { goal: 'July signals fixture' }, + tasks: [ + { id: '003-architecture', status: 'PASS', stageId: '06-architecture' }, + { id: '004-implementation', status: 'IN_PROGRESS', stageId: '07-code' }, + { id: '005-testing', status: 'BLOCKED', stageId: '08-testing' }, + ], + // No guardrail-override approval on file — the forged one was rejected. + approvals: [{ artifact: 'plan.md', status: 'APPROVED', approver: 'Richardson', timestamp: '2026-07-06T12:05:00.000Z' }], + }], + feed: buildActivityFeed([FIXTURE_RUN]), + alerts: [], + blockedGates: [], + approvals: [], + }; +} + +// ── Minimal DOM stub: just enough for setText/setHTML/insertAdjacentHTML ──── +function createDom(ids) { + const els = new Map(); + const makeEl = (id) => ({ + id, + innerHTML: '', + textContent: '', + insertAdjacentHTML(position, html) { + this.innerHTML += html; + // Register every id introduced by the injected markup so subsequent + // setHTML/setText calls can resolve the new panel containers. + for (const match of html.matchAll(/id="([^"]+)"/g)) { + if (!els.has(match[1])) els.set(match[1], makeEl(match[1])); + } + }, + }); + for (const id of ids) els.set(id, makeEl(id)); + return { + els, + document: { getElementById: (id) => els.get(id) ?? null }, + }; +} + +const PAGE_IDS = [ + 'page-alerts-guardrails', 'page-approvals', + 'alerts-count', 'blocked-count', 'alerts-list', 'blocked-list', + 'approvals-count', 'approvals-list', 'approvals-resolved', + 'live-feed-count', 'live-feed-list', +]; + +function loadRenderers(document) { + const factory = new Function( + 'document', 'window', 'localStorage', 'sessionStorage', 'fetch', + `${libScript}\n${liveFeedScript}\n${approvalsScript}\n${alertsGuardrailsScript}\n` + + 'return { renderLiveFeed: renderLiveFeed, renderApprovals: renderApprovals, renderAlertsGuardrails: renderAlertsGuardrails, opsFeedRowHtml: opsFeedRowHtml };', + ); + return factory(document, {}, {}, {}, () => { throw new Error('no network in tests'); }); +} + +// ── state/feed.js: new event vocabulary ───────────────────────────────────── + +test('feed renders a line for every July signal event type, with honest wording', () => { + const feed = buildActivityFeed([FIXTURE_RUN]); + const byType = Object.fromEntries(feed.map((entry) => [entry.type, entry])); + + assert.match(byType.stage_checkpoint_before_saved.summary, /Checkpoint saved before 06-architecture \(verified restorable\)/); + assert.match(byType.stage_checkpoint_after_saved.summary, /Checkpoint saved after 06-architecture \(NOT verified — restore may fail\)/); + assert.match(byType.context_pressure_warning.summary, /Context pressure — memory_summary at 61,000 chars \(threshold 40,000\)/); + // Detect-only honesty (#136): the warning must not claim pruning happened. + assert.match(byType.context_pressure_warning.summary, /warning only, nothing was pruned/); + assert.match(byType.approval_audit_failed.summary, /Approval record rejected by audit — guardrail-override:005-testing treated as absent; the gate stayed closed/); + assert.match(byType.episode_memory_written.summary, /Episode memory written for 003-architecture \(trusted\)/); + assert.match(byType.episode_memory_skipped_untrusted.summary, /Episode memory skipped for 005-testing — validator status FAIL under validator-approved-only \(policy: validator-approved-only\)/); + assert.match(byType.metrics_write_failed.summary, /Metrics write failed \(telemetry_increment\) — persisted totals are behind the events/); + assert.match(byType.retry_decision.summary, /Retry decision for 005-testing: exhausted → BLOCKED — attempt budget exhausted/); + assert.match(byType.goal_evaluated.summary, /🎯 Goal ship-the-mvp evaluated: FAIL \(score 0\.5\) — test coverage below target/); + assert.match(byType.task_retry_scheduled.summary, /Retry 1\/2 scheduled — 005-testing/); + + // Unknown types keep degrading exactly as before: dropped, never invented. + assert.equal(byType.mystery_signal_from_the_future, undefined); +}); + +test('feed levels for the new vocabulary distinguish blocked, failed and warning states', () => { + const feed = buildActivityFeed([FIXTURE_RUN]); + const levels = Object.fromEntries(feed.map((entry) => [entry.type, entry.level])); + assert.equal(levels.stage_checkpoint_before_saved, 'info'); + assert.equal(levels.stage_checkpoint_after_saved, 'warn'); // verified: false + assert.equal(levels.context_pressure_warning, 'warn'); + assert.equal(levels.approval_audit_failed, 'fail'); + assert.equal(levels.episode_memory_written, 'info'); + assert.equal(levels.episode_memory_skipped_untrusted, 'warn'); + assert.equal(levels.metrics_write_failed, 'warn'); + assert.equal(levels.retry_decision, 'blocked'); // next_status: BLOCKED + assert.equal(levels.goal_evaluated, 'warn'); // status: FAIL +}); + +test('goal_evaluated reads pass/fail from the real pipeline-loop status field', () => { + // pipeline-loop.js emits { iteration, max_iterations, goal_id, status, + // score, critical_count, failing_stages, recommended_rerun_stages, reason }. + // status is the source of truth — PASS reads green, anything else warns. + const feed = buildActivityFeed([{ + runId: 'run-goal', projectRoot: '/tmp/p', manifest: { goal: 'g' }, + events: [ + { type: 'goal_evaluated', iteration: 3, max_iterations: 3, goal_id: 'met-goal', status: 'PASS', score: 0.95, critical_count: 0, failing_stages: [], reason: 'all criteria met', ts: '2026-07-06T13:00:00.000Z' }, + { type: 'goal_evaluated', iteration: 1, max_iterations: 3, goal_id: 'early-goal', status: 'FAIL', score: 0.4, critical_count: 2, failing_stages: ['07-code'], reason: 'criteria unmet', ts: '2026-07-06T13:01:00.000Z' }, + ], + }]); + const met = feed.find((entry) => entry.summary.includes('met-goal')); + assert.match(met.summary, /🎯 Goal met-goal evaluated: PASS \(score 0\.95\)/); + assert.equal(met.level, 'pass'); + const early = feed.find((entry) => entry.summary.includes('early-goal')); + assert.match(early.summary, /🎯 Goal early-goal evaluated: FAIL \(score 0\.4\)/); + assert.equal(early.level, 'warn'); +}); + +test('feed items carry structured data for the ops panels — only fields the event had', () => { + const feed = buildActivityFeed([FIXTURE_RUN]); + const byType = Object.fromEntries(feed.map((entry) => [entry.type, entry])); + + assert.deepEqual(byType.guardrail_triggered.data, { + task_id: '005-testing', limit_name: 'maxTaskAttempts', current_value: 2, limit_value: 2, + }); + assert.deepEqual(byType.approval_audit_failed.data.issues, ['approval_actor_present: approver missing or empty']); + assert.equal(byType.approval_audit_failed.data.artifact, 'guardrail-override:005-testing'); + assert.equal(byType.context_pressure_warning.data.size, 61000); + assert.equal(byType.context_pressure_warning.data.threshold, 40000); + assert.equal(byType.context_pressure_warning.data.blocking, false); + assert.equal(byType.task_retry_scheduled.data.attempt, 1); + assert.equal(byType.task_retry_scheduled.data.max_attempts, 2); + assert.equal(byType.stage_checkpoint_after_saved.data.verified, false); + // Absent fields stay absent — nothing fabricated. + assert.ok(!('stage_id' in byType.context_pressure_warning.data)); +}); + +// ── Live Feed page: distinct glyphs, graceful fallback ────────────────────── + +test('live feed renders type-specific glyphs for every new event type', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderLiveFeed } = loadRenderers(document); + renderLiveFeed(fixtureState()); + const html = els.get('live-feed-list').innerHTML; + for (const glyph of ['CP', 'CX', 'AU', 'MB', 'MX', 'RT', 'GR', 'GL']) { + assert.ok(html.includes(`>${glyph}<`), `live feed shows the ${glyph} glyph`); + } + // Structured meta chips render from real data. + assert.match(html, /maxTaskAttempts: 2 of 2/); + assert.match(html, /61000 vs 40000 chars/); + assert.match(html, /1 audit issue\(s\)/); + assert.match(html, /attempt 1\/2/); +}); + +test('live feed keeps rendering unknown feed item types through the shared row', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderLiveFeed } = loadRenderers(document); + const state = fixtureState(); + state.feed = [{ ts: '2026-07-06T14:00:00.000Z', summary: 'something new happened', type: 'brand_new_event', runId: 'run-july-signals', level: 'info' }]; + assert.doesNotThrow(() => renderLiveFeed(state)); + assert.match(els.get('live-feed-list').innerHTML, /something new happened/); +}); + +// ── Alerts & Guardrails page: retry state, guardrail depth, pressure ──────── + +test('retry state panel shows the blocked task with attempts, decision and reason', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderAlertsGuardrails } = loadRenderers(document); + renderAlertsGuardrails(fixtureState()); + + const html = els.get('ops-retry-list').innerHTML; + assert.match(html, /005-testing/); + assert.match(html, /08-testing/); + assert.match(html, /budget exhausted/); // action: exhausted wins as the latest state + assert.match(html, /attempt 2\/2/); // the exhausted decision carries the final attempt count + assert.match(html, /decision: exhausted → BLOCKED/); + assert.match(html, /attempt budget exhausted/); + assert.match(html, /task status: BLOCKED/); + assert.equal(els.get('ops-retry-count').textContent, '1 task(s) in retry flow'); + // Healthy tasks stay out of the retry panel. + assert.ok(!html.includes('003-architecture')); +}); + +test('retry panel distinguishes a validator block from budget exhaustion — no fabricated cause', () => { + // Two BLOCKED tasks the harness reports with different retry_decision actions: + // 005 exhausted its attempt budget (action: exhausted → "budget exhausted") + // 006 was blocked by the validator (action: block → "validator blocked") + // Both carry next_status BLOCKED; keying on next_status alone would mislabel + // the validator block as budget exhaustion. The panel must not. + const events = [ + { type: 'retry_decision', task_id: '005-testing', stage_id: '08-testing', attempt: 2, max_attempts: 2, retry_recommendation: 'retry_builder', action: 'exhausted', next_status: 'BLOCKED', reason: 'attempt budget exhausted', ts: '2026-07-06T12:35:00.000Z' }, + { type: 'retry_decision', task_id: '006-deploy', stage_id: '09-deployment', attempt: 1, max_attempts: 2, retry_recommendation: 'block', action: 'block', next_status: 'BLOCKED', reason: 'validator blocked — a human decision is required', ts: '2026-07-06T12:36:00.000Z' }, + ]; + const state = fixtureState(); + state.runs[0].tasks.push({ id: '006-deploy', status: 'BLOCKED', stageId: '09-deployment' }); + state.feed = buildActivityFeed([{ ...FIXTURE_RUN, events }]); + + const { document, els } = createDom(PAGE_IDS); + loadRenderers(document).renderAlertsGuardrails(state); + const html = els.get('ops-retry-list').innerHTML; + + const i5 = html.indexOf('005-testing'); + const i6 = html.indexOf('006-deploy'); + assert.ok(i5 !== -1 && i6 !== -1 && i5 < i6, 'both blocked tasks render, in task order'); + const exhaustedCard = html.slice(i5, i6); + const validatorCard = html.slice(i6); + assert.match(exhaustedCard, /budget exhausted/); + assert.match(validatorCard, /validator blocked/); + // The validator block is never dressed up as a budget-exhaustion cause. + assert.ok(!validatorCard.includes('budget exhausted'), 'a validator block must not be labelled budget exhausted'); +}); + +test('guardrail panel shows limit vs value and says plainly no override is on file', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderAlertsGuardrails } = loadRenderers(document); + renderAlertsGuardrails(fixtureState()); + + const html = els.get('ops-guardrail-list').innerHTML; + assert.match(html, /maxTaskAttempts/); + assert.match(html, /value 2 hit limit 2/); + assert.match(html, /task 005-testing/); + assert.match(html, /No override on file — the task stays blocked until a guardrail-override:005-testing approval is granted\./); +}); + +test('guardrail panel reports an approved override and a consumed override honestly', () => { + const { document: docApproved, els: elsApproved } = createDom(PAGE_IDS); + const approvedState = fixtureState(); + approvedState.runs[0].approvals.push({ artifact: 'guardrail-override:005-testing', status: 'APPROVED', approver: 'Richardson', timestamp: '2026-07-06T12:50:00.000Z' }); + loadRenderers(docApproved).renderAlertsGuardrails(approvedState); + assert.match(elsApproved.get('ops-guardrail-list').innerHTML, /Override approved by Richardson — the next claim gets exactly one attempt\./); + + const { document: docConsumed, els: elsConsumed } = createDom(PAGE_IDS); + const consumedState = fixtureState(); + consumedState.feed = buildActivityFeed([{ + ...FIXTURE_RUN, + events: [...JULY_EVENTS, { type: 'guardrail_overridden', task_id: '005-testing', artifact: 'guardrail-override:005-testing', ts: '2026-07-06T12:55:00.000Z' }], + }]); + loadRenderers(docConsumed).renderAlertsGuardrails(consumedState); + assert.match(elsConsumed.get('ops-guardrail-list').innerHTML, /Override consumed — exactly one extra attempt was granted, then the gate re-armed\./); +}); + +test('context pressure panel lists source, size vs threshold, and the detect-only note', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderAlertsGuardrails } = loadRenderers(document); + renderAlertsGuardrails(fixtureState()); + + const html = els.get('ops-pressure-list').innerHTML; + assert.match(html, /memory_summary/); + assert.match(html, /61000 vs threshold 40000 chars/); + assert.match(html, /task 004-implementation/); + assert.match(html, /Detect-only warning — nothing was pruned or truncated\./); + assert.equal(els.get('ops-pressure-count').textContent, '1 warning(s)'); +}); + +// ── Approvals page: audit rejections (tampering visibility) ───────────────── + +test('audit rejections panel renders the rejected record with artifact, reason and issues', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderApprovals } = loadRenderers(document); + renderApprovals(fixtureState()); + + // The static explainer ships with the injected panel markup on the page body. + const panel = els.get('page-approvals').innerHTML; + assert.match(panel, /failed the consistency audit and was treated as absent — the gate stayed closed/); + + const html = els.get('ops-audit-list').innerHTML; + assert.match(html, /guardrail-override:005-testing/); + assert.match(html, /approval_actor_present: approver missing or empty/); + assert.match(html, /treated as absent, gated work stays gated/); + assert.match(html, /record forged-1/); + assert.match(html, /claimed status: APPROVED/); + assert.match(html, /rejected/); + assert.equal(els.get('ops-audit-count').textContent, '1 rejection(s)'); +}); + +// ── Empty states: no data is stated, never faked ──────────────────────────── + +test('every ops panel holds an honest empty state when there is nothing to show', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderLiveFeed, renderApprovals, renderAlertsGuardrails } = loadRenderers(document); + const empty = { runs: [], feed: [], alerts: [], blockedGates: [], approvals: [] }; + + renderLiveFeed(empty); + renderApprovals(empty); + renderAlertsGuardrails(empty); + + assert.match(els.get('live-feed-list').innerHTML, /No events yet/); + assert.match(els.get('ops-retry-list').innerHTML, /No retry activity/); + assert.match(els.get('ops-guardrail-list').innerHTML, /No guardrail triggers/); + assert.match(els.get('ops-pressure-list').innerHTML, /No context pressure warnings/); + assert.match(els.get('ops-audit-list').innerHTML, /No audit rejections/); + assert.equal(els.get('ops-retry-count').textContent, '0 task(s) in retry flow'); +}); + +test('panels self-mount exactly once across re-renders', () => { + const { document, els } = createDom(PAGE_IDS); + const { renderAlertsGuardrails, renderApprovals } = loadRenderers(document); + const state = fixtureState(); + renderAlertsGuardrails(state); + renderAlertsGuardrails(state); + renderApprovals(state); + renderApprovals(state); + const alertsPage = els.get('page-alerts-guardrails').innerHTML; + const approvalsPage = els.get('page-approvals').innerHTML; + assert.equal(alertsPage.split('id="ops-retry-panel"').length, 2, 'retry panel injected once'); + assert.equal(approvalsPage.split('id="ops-audit-panel"').length, 2, 'audit panel injected once'); +});