diff --git a/docs/EXCEPTION_INBOX.md b/docs/EXCEPTION_INBOX.md index 41010cdc..c634a0f5 100644 --- a/docs/EXCEPTION_INBOX.md +++ b/docs/EXCEPTION_INBOX.md @@ -84,11 +84,14 @@ replayed, or undeclared cursor or receipt refuses instead of guessing. - Browser mutations also require a valid Host, same-origin request, `application/json`, and the session CSRF token. - The authenticated local OS account is recorded as the operator. -- Queue records use opaque IDs and category-derived copy. Workflow names, - parameters, raw halt reasons, observed text, local paths, and raw reports do - not enter the DTO. -- Artifact lookup accepts only report-referenced PNG IDs and never follows a - symlink. +- Queue and summary records use opaque IDs and category-derived copy. Workflow + names, parameters, raw halt reasons, observed text, local paths, and raw + reports do not enter those DTOs. Protected values appear only when the local + operator explicitly opens an admitted JSON/JSONL artifact. +- Artifact lookup accepts only report-referenced PNG IDs or bounded JSON + families selected by opaque IDs. It never accepts a filesystem path or + follows a symlink. The viewer reads only the configured local roots and has + no Cloud or other network transport. - The notification endpoint returns only a count and `#/attention`; it is the PHI-free seam for a desktop/tray OS notification. diff --git a/openadapt_flow/console/__init__.py b/openadapt_flow/console/__init__.py index 883ce5eb..6e0cbd93 100644 --- a/openadapt_flow/console/__init__.py +++ b/openadapt_flow/console/__init__.py @@ -1,9 +1,11 @@ """Authenticated loopback operator console over engine artifacts. The browser receives explicit redacted projections of compiled bundles, run -reports, durable pause/approval state, and versioned skill libraries. It does -not receive raw workflow/report models, protected labels, parameter values, or -local paths. The console invents no new engine semantics: +reports, durable pause/approval state, and versioned skill libraries. Protected +labels and parameter values appear only when the local operator explicitly +opens an admitted JSON/JSONL artifact in the bearer-authenticated viewer; +filesystem paths never cross the API boundary. The console invents no new engine +semantics: - Every number it shows is computed by the SAME callables the CLI uses (``policy.evaluate_policy`` / ``policy.lint_workflow`` / the identity- and diff --git a/openadapt_flow/console/app.py b/openadapt_flow/console/app.py index 5739f202..fc25ca9c 100644 --- a/openadapt_flow/console/app.py +++ b/openadapt_flow/console/app.py @@ -10,9 +10,10 @@ remove the normal console. Attended mutations additionally require an exact engine-issued pause capability and a deployment-bound action service. -Browser DTOs use opaque ids and explicit projections; protected workflow -labels, parameters, identity evidence, local paths, and raw reports do not -cross the API boundary. Run artifacts are limited to PNGs explicitly +Browser summary DTOs use opaque ids and explicit projections; protected +workflow labels, parameters, identity evidence, and local paths do not cross +that API boundary. Raw JSON is available only through the explicit protected +local-artifact viewer. Run screenshots remain limited to PNGs explicitly referenced by the protected report and resolved without following symlinks. """ @@ -24,7 +25,7 @@ import secrets import shlex from pathlib import Path -from typing import Any, Optional +from typing import Any, Literal, Optional from urllib.parse import urlsplit from fastapi import FastAPI, HTTPException, Query, Request @@ -34,7 +35,7 @@ from openadapt_flow import __version__ from openadapt_flow.console import actions as actions_mod from openadapt_flow.console import attention as attention_mod -from openadapt_flow.console import data +from openadapt_flow.console import data, json_artifacts from openadapt_flow.runtime.durable.approval import ResumeRefused from openadapt_flow.runtime.durable.attended import ( AttendedActionRefused, @@ -491,6 +492,64 @@ def run_detail(run_id: str) -> dict[str, Any]: run_dir = _resolve_run(runs, run_id) return data.run_detail(runs, run_dir) + # -- explicit local JSON artifacts ------------------------------------- + + def json_artifact_owner( + scope: str, owner_id: str + ) -> tuple[Path, Literal["runs", "workflows"]]: + if scope == "runs": + return _resolve_run(runs, owner_id), "runs" + if scope == "workflows": + return _resolve_bundle(bundles, owner_id), "workflows" + raise HTTPException(status_code=404, detail="no such artifact scope") + + def load_json_artifact( + scope: str, owner_id: str, artifact_id: str + ) -> json_artifacts.LoadedJsonArtifact: + owner, admitted_scope = json_artifact_owner(scope, owner_id) + artifact = json_artifacts.resolve_artifact( + owner, + artifact_id, + scope=admitted_scope, + ) + if artifact is None: + raise HTTPException(status_code=404, detail="no such JSON artifact") + try: + return json_artifacts.load_artifact(artifact) + except json_artifacts.JsonArtifactError as exc: + raise HTTPException(status_code=exc.status_code, detail=str(exc)) from exc + + @app.get("/api/artifacts/json/{scope}/{owner_id}") + def json_artifact_list(scope: str, owner_id: str) -> list[dict[str, Any]]: + owner, admitted_scope = json_artifact_owner(scope, owner_id) + return [ + artifact.public() + for artifact in json_artifacts.list_artifacts( + owner, + scope=admitted_scope, + ) + ] + + @app.get("/api/artifacts/json/{scope}/{owner_id}/{artifact_id}") + def json_artifact_document( + scope: str, owner_id: str, artifact_id: str + ) -> dict[str, Any]: + return load_json_artifact(scope, owner_id, artifact_id).public() + + @app.get("/api/artifacts/json/{scope}/{owner_id}/{artifact_id}/raw") + def json_artifact_raw(scope: str, owner_id: str, artifact_id: str) -> Response: + loaded = load_json_artifact(scope, owner_id, artifact_id) + media_type = ( + "application/json" + if loaded.artifact.format == "json" + else "application/x-ndjson" + ) + return Response( + loaded.raw, + media_type=media_type, + headers={"X-Content-SHA256": loaded.sha256}, + ) + # -- needs attention ---------------------------------------------------- @app.get("/api/attention") diff --git a/openadapt_flow/console/json_artifacts.py b/openadapt_flow/console/json_artifacts.py new file mode 100644 index 00000000..54368b19 --- /dev/null +++ b/openadapt_flow/console/json_artifacts.py @@ -0,0 +1,456 @@ +"""Bounded, local-only JSON artifacts for the operator console. + +The normal console DTOs deliberately redact workflow and run values. This +module powers the explicit "open local artifact" path: a bearer-authenticated +operator can inspect the exact JSON/JSONL bytes that already exist inside the +configured bundle or run root. It never accepts a caller-provided path and it +never performs network I/O. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, Optional + +MAX_ARTIFACT_BYTES = 5 * 1024 * 1024 +MAX_DOCUMENT_NODES = 20_000 +MAX_DOCUMENT_DEPTH = 100 +MAX_DISCOVERED_ARTIFACTS = 256 +MAX_DISCOVERY_DEPTH = 6 +MAX_DISCOVERY_ENTRIES = 4_096 + +ArtifactFormat = Literal["json", "jsonl"] + +_RUN_ROOT_FILES = { + "approval.json", + "attended_decisions.json", + "governed_policy.json", + "pending_escalation.json", + "report.json", +} +_BUNDLE_ROOT_FILES = { + "annotations.json", + "manifest.json", + "workflow.json", +} +_CHECKPOINT_PREFIXES = ("step_", "pstate_") +_HEAL_FILES = {"heal.json", "patch.json"} +_ROOT_LABELS = { + "annotations.json": "Workflow annotations", + "approval.json": "Operator approval", + "attended_decisions.json": "Attended decisions", + "governed_policy.json": "Governed policy", + "manifest.json": "Bundle manifest", + "pending_escalation.json": "Pending escalation", + "report.json": "Run report", + "workflow.json": "Compiled workflow", +} + + +class JsonArtifactError(ValueError): + """A local artifact cannot be admitted to the bounded viewer.""" + + def __init__(self, message: str, *, status_code: int = 422) -> None: + super().__init__(message) + self.status_code = status_code + + +@dataclass(frozen=True) +class JsonArtifact: + """A path admitted from a fixed local artifact family.""" + + id: str + relative_path: str + label: str + root: Path + path: Path + format: ArtifactFormat + size_bytes: int + + def public(self) -> dict[str, Any]: + return { + "id": self.id, + "label": self.label, + "format": self.format, + "size_bytes": self.size_bytes, + "download_name": f"openadapt-artifact-{self.id}.{self.format}", + } + + +@dataclass(frozen=True) +class LoadedJsonArtifact: + artifact: JsonArtifact + raw: bytes + document: Any + node_count: int + max_depth: int + sha256: str + + def public(self) -> dict[str, Any]: + out = self.artifact.public() + out.update( + { + "document": self.document, + "node_count": self.node_count, + "max_depth": self.max_depth, + "sha256": self.sha256, + } + ) + return out + + +def _artifact_id(relative_path: str) -> str: + return hashlib.sha256(relative_path.encode("utf-8")).hexdigest()[:24] + + +def _public_label(relative: Path, artifact_id: str) -> str: + if len(relative.parts) == 1: + return _ROOT_LABELS.get(relative.name, "Local JSON artifact") + if relative.parts[0] == "checkpoints": + if relative.name == "_manifest.json": + return "Checkpoint manifest" + kind = ( + "Program checkpoint" + if relative.name.startswith("pstate_") + else "Run checkpoint" + ) + return f"{kind} {artifact_id[:8]}" + if relative.parts[0] == "heals": + kind = "Repair patch" if relative.name == "patch.json" else "Repair event" + return f"{kind} {artifact_id[:8]}" + if relative.parts[0] == "qualification": + return f"Qualification evidence {artifact_id[:8]}" + return f"Run evidence {artifact_id[:8]}" + + +def _format(path: Path) -> Optional[ArtifactFormat]: + suffix = path.suffix.lower() + if suffix == ".json": + return "json" + if suffix == ".jsonl": + return "jsonl" + return None + + +def _is_admitted_run_path(relative: Path) -> bool: + parts = relative.parts + if len(parts) == 1: + return relative.name in _RUN_ROOT_FILES + if parts[0] == "checkpoints" and len(parts) == 2: + return relative.name == "_manifest.json" or ( + relative.name.endswith(".json") + and relative.name.startswith(_CHECKPOINT_PREFIXES) + ) + if parts[0] == "heals" and len(parts) == 3: + return relative.name in _HEAL_FILES + if parts[0] == "evidence": + return _format(relative) is not None + return False + + +def _is_admitted_bundle_path(relative: Path) -> bool: + if len(relative.parts) == 1: + return relative.name in _BUNDLE_ROOT_FILES + if relative.parts[0] in {"evidence", "qualification"}: + return _format(relative) is not None + return False + + +def _walk_regular_files(root: Path) -> list[Path]: + """Walk a shallow local tree without following any symlink.""" + out: list[Path] = [] + frontier: list[tuple[Path, int]] = [(root, 0)] + entries_seen = 0 + while frontier: + directory, depth = frontier.pop() + if depth > MAX_DISCOVERY_DEPTH: + continue + try: + entries = sorted(os.scandir(directory), key=lambda entry: entry.name) + except OSError: + continue + for entry in entries: + entries_seen += 1 + if entries_seen > MAX_DISCOVERY_ENTRIES: + return sorted(out) + try: + if entry.is_symlink(): + continue + if entry.is_dir(follow_symlinks=False): + frontier.append((Path(entry.path), depth + 1)) + elif entry.is_file(follow_symlinks=False) and _format(Path(entry.path)): + out.append(Path(entry.path)) + if len(out) >= MAX_DISCOVERED_ARTIFACTS: + return sorted(out) + except OSError: + continue + return sorted(out) + + +def _candidate_files(root: Path, *, scope: Literal["runs", "workflows"]) -> list[Path]: + """Inspect only directories that can contain an admitted artifact family.""" + root_names = _RUN_ROOT_FILES if scope == "runs" else _BUNDLE_ROOT_FILES + out = [root / name for name in sorted(root_names)] + subdirectories = ( + ("checkpoints", "heals", "evidence") + if scope == "runs" + else ("evidence", "qualification") + ) + for name in subdirectories: + directory = root / name + if directory.is_symlink() or not directory.is_dir(): + continue + out.extend(_walk_regular_files(directory)) + if len(out) >= MAX_DISCOVERED_ARTIFACTS: + break + ordered: list[Path] = [] + seen: set[Path] = set() + for candidate in out: + if candidate not in seen: + seen.add(candidate) + ordered.append(candidate) + return ordered[:MAX_DISCOVERED_ARTIFACTS] + + +def _contained_regular_file(root: Path, candidate: Path) -> Optional[Path]: + """Resolve a regular file beneath ``root`` with no symlink component.""" + lexical_root = root.absolute() + lexical = candidate.absolute() + if lexical_root.is_symlink(): + return None + try: + relative = lexical.relative_to(lexical_root) + except ValueError: + return None + current = lexical_root + for part in relative.parts: + current = current / part + if current.is_symlink(): + return None + try: + resolved_root = lexical_root.resolve(strict=True) + resolved = lexical.resolve(strict=True) + resolved.relative_to(resolved_root) + info = resolved.stat(follow_symlinks=False) + except (OSError, ValueError): + return None + return resolved if stat.S_ISREG(info.st_mode) else None + + +def list_artifacts( + root: Path, *, scope: Literal["runs", "workflows"] +) -> list[JsonArtifact]: + """List admitted JSON files in one already-resolved run or bundle dir.""" + predicate = _is_admitted_run_path if scope == "runs" else _is_admitted_bundle_path + out: list[JsonArtifact] = [] + for candidate in _candidate_files(root, scope=scope): + try: + relative = candidate.relative_to(root) + except ValueError: + continue + artifact_format = _format(relative) + if artifact_format is None or not predicate(relative): + continue + safe = _contained_regular_file(root, candidate) + if safe is None: + continue + try: + size = safe.stat(follow_symlinks=False).st_size + except OSError: + continue + relative_path = relative.as_posix() + artifact_id = _artifact_id(relative_path) + out.append( + JsonArtifact( + id=artifact_id, + relative_path=relative_path, + label=_public_label(relative, artifact_id), + root=root, + path=safe, + format=artifact_format, + size_bytes=size, + ) + ) + return sorted(out, key=lambda artifact: artifact.relative_path) + + +def resolve_artifact( + root: Path, + artifact_id: str, + *, + scope: Literal["runs", "workflows"], +) -> Optional[JsonArtifact]: + """Resolve only an opaque id returned by a fresh admitted listing.""" + return next( + ( + artifact + for artifact in list_artifacts(root, scope=scope) + if artifact.id == artifact_id + ), + None, + ) + + +def _read_bounded(artifact: JsonArtifact) -> bytes: + if artifact.size_bytes > MAX_ARTIFACT_BYTES: + raise JsonArtifactError( + f"artifact exceeds the {MAX_ARTIFACT_BYTES}-byte viewer limit", + status_code=413, + ) + safe = _contained_regular_file(artifact.root, artifact.path) + if safe is None: + raise JsonArtifactError("artifact is no longer available", status_code=404) + try: + before_open = safe.stat(follow_symlinks=False) + except OSError as exc: + raise JsonArtifactError( + "artifact is no longer available", status_code=404 + ) from exc + flags = os.O_RDONLY + if hasattr(os, "O_BINARY"): + flags |= os.O_BINARY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + fd = os.open(safe, flags) + except OSError as exc: + raise JsonArtifactError( + "artifact is no longer available", status_code=404 + ) from exc + try: + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode): + raise JsonArtifactError("artifact is not a regular file", status_code=404) + if (opened.st_dev, opened.st_ino) != (before_open.st_dev, before_open.st_ino): + raise JsonArtifactError("artifact changed while opening", status_code=404) + if opened.st_size > MAX_ARTIFACT_BYTES: + raise JsonArtifactError( + f"artifact exceeds the {MAX_ARTIFACT_BYTES}-byte viewer limit", + status_code=413, + ) + chunks: list[bytes] = [] + total = 0 + while True: + chunk = os.read(fd, min(64 * 1024, MAX_ARTIFACT_BYTES + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > MAX_ARTIFACT_BYTES: + raise JsonArtifactError( + f"artifact exceeds the {MAX_ARTIFACT_BYTES}-byte viewer limit", + status_code=413, + ) + return b"".join(chunks) + finally: + os.close(fd) + + +def _reject_nonstandard_constant(value: str) -> None: + raise ValueError(f"non-standard JSON constant {value}") + + +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for key, value in pairs: + if key in out: + raise ValueError("duplicate JSON object key") + out[key] = value + return out + + +def _parse_json(text: str) -> Any: + return json.loads( + text, + parse_constant=_reject_nonstandard_constant, + object_pairs_hook=_unique_object, + ) + + +def _measure(document: Any) -> tuple[int, int]: + """Bound the parsed tree before any UI recursively expands it.""" + nodes = 0 + max_depth = 0 + stack: list[tuple[Any, int]] = [(document, 0)] + while stack: + value, depth = stack.pop() + nodes += 1 + max_depth = max(max_depth, depth) + if nodes > MAX_DOCUMENT_NODES: + raise JsonArtifactError( + f"artifact exceeds the {MAX_DOCUMENT_NODES}-node viewer limit", + status_code=413, + ) + if depth > MAX_DOCUMENT_DEPTH: + raise JsonArtifactError( + f"artifact exceeds the {MAX_DOCUMENT_DEPTH}-level viewer limit", + status_code=413, + ) + if isinstance(value, dict): + stack.extend((child, depth + 1) for child in value.values()) + elif isinstance(value, list): + stack.extend((child, depth + 1) for child in value) + return nodes, max_depth + + +def load_artifact(artifact: JsonArtifact) -> LoadedJsonArtifact: + raw = _read_bounded(artifact) + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise JsonArtifactError("artifact is not valid UTF-8") from exc + try: + measured: Optional[tuple[int, int]] = None + if artifact.format == "json": + document = _parse_json(text) + else: + document = [] + node_count = 1 # the JSONL document is presented as an array + max_depth = 0 + for line_number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + try: + item = _parse_json(line) + except (json.JSONDecodeError, ValueError) as exc: + raise JsonArtifactError( + f"JSONL line {line_number} is invalid" + ) from exc + item_nodes, item_depth = _measure(item) + node_count += item_nodes + max_depth = max(max_depth, item_depth + 1) + if node_count > MAX_DOCUMENT_NODES: + raise JsonArtifactError( + f"artifact exceeds the {MAX_DOCUMENT_NODES}-node viewer limit", + status_code=413, + ) + if max_depth > MAX_DOCUMENT_DEPTH: + raise JsonArtifactError( + f"artifact exceeds the {MAX_DOCUMENT_DEPTH}-level viewer limit", + status_code=413, + ) + document.append(item) + measured = node_count, max_depth + except JsonArtifactError: + raise + except RecursionError as exc: + raise JsonArtifactError( + f"artifact exceeds the {MAX_DOCUMENT_DEPTH}-level viewer limit", + status_code=413, + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + raise JsonArtifactError("artifact is not valid JSON") from exc + node_count, max_depth = measured or _measure(document) + return LoadedJsonArtifact( + artifact=artifact, + raw=raw, + document=document, + node_count=node_count, + max_depth=max_depth, + sha256=hashlib.sha256(raw).hexdigest(), + ) diff --git a/openadapt_flow/console/static/console.css b/openadapt_flow/console/static/console.css index 9d0a5a19..9a06e66e 100644 --- a/openadapt_flow/console/static/console.css +++ b/openadapt_flow/console/static/console.css @@ -242,3 +242,130 @@ a { color: var(--accent); } text-decoration: none; } .button-link:hover { border-color: var(--accent); } + +.artifact-links { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 8px; +} +.artifact-link { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; + padding: 10px 12px; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--panel); + text-decoration: none; +} +.artifact-link:hover { border-color: var(--accent); } +.artifact-link .mono { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.json-viewer-head, +.json-viewer-meta, +.json-viewer-actions { + display: flex; + align-items: center; + gap: 9px; + flex-wrap: wrap; +} +.json-viewer-head { justify-content: space-between; } +.json-viewer-head h2 { margin-bottom: 3px; } +.json-viewer-head p { margin: 0 0 10px; overflow-wrap: anywhere; } +.json-viewer-meta { margin: 5px 0 15px; color: var(--muted); } +.json-search-label { + display: block; + margin: 12px 0 5px; + color: var(--muted); + font-size: 12px; +} +.json-search { + width: min(100%, 620px); + padding: 8px 10px; + background: var(--panel); + color: var(--text); + border: 1px solid var(--border); + border-radius: 5px; +} +.json-search:focus { outline: 1px solid var(--accent); border-color: var(--accent); } +.json-search-matches { + display: grid; + gap: 3px; + max-height: 220px; + margin: 8px 0; + overflow: auto; +} +.json-search-matches a { + display: flex; + gap: 10px; + padding: 5px 8px; + border-radius: 4px; + background: var(--panel); + text-decoration: none; +} +.json-tree, +.json-raw { + margin-top: 12px; + padding: 12px 14px; + overflow: auto; + border: 1px solid var(--border); + border-radius: 6px; + background: #0a0d10; + font: 12px/1.55 ui-monospace, Menlo, monospace; +} +.json-raw { max-height: 65vh; white-space: pre; } +.json-node { margin: 1px 0; } +.json-node > summary { + display: list-item; + color: var(--text); + white-space: nowrap; +} +.json-children { + margin-left: 14px; + padding-left: 10px; + border-left: 1px solid var(--border); +} +.json-leaf { + display: flex; + align-items: baseline; + gap: 5px; + min-height: 20px; + white-space: pre-wrap; + overflow-wrap: anywhere; +} +.json-root-label, +.json-key { color: #8fc7ff; } +.json-colon, +.json-preview { color: var(--muted); } +.json-value.string { color: #a8d68d; } +.json-value.number { color: #e4b672; } +.json-value.boolean { color: #c6a0f6; } +.json-value.null { color: var(--muted); } +.json-deep-link { + margin-left: 7px; + color: var(--muted); + opacity: 0; + text-decoration: none; +} +.json-node summary:hover .json-deep-link, +.json-leaf:hover .json-deep-link, +.json-deep-link:focus { opacity: 1; } +.json-target { + border-radius: 3px; + background: rgba(77, 163, 255, .12); + outline: 1px solid rgba(77, 163, 255, .35); +} + +@media (max-width: 640px) { + main { padding: 14px 12px; } + .json-viewer-head { align-items: flex-start; } + .json-viewer-actions { width: 100%; } + .json-viewer-actions button { flex: 1; } + .json-tree, + .json-raw { padding: 10px 8px; } + .json-children { margin-left: 7px; padding-left: 7px; } +} diff --git a/openadapt_flow/console/static/console.js b/openadapt_flow/console/static/console.js index b2ec81c1..cc5b2171 100644 --- a/openadapt_flow/console/static/console.js +++ b/openadapt_flow/console/static/console.js @@ -20,6 +20,7 @@ let currentActions = {list: [], postBase: ""}; let currentSkillActions = []; let pendingExec = null; let artifactObjectUrls = []; +let currentJsonViewer = null; function consumeBootstrapToken() { const rawFragment = window.location.hash.slice(1); @@ -72,6 +73,22 @@ const safeNumber = (value, fallback = "—") => { const number = Number(value); return Number.isFinite(number) ? esc(number) : fallback; }; +const humanBytes = (value) => { + const bytes = Number(value); + if (!Number.isFinite(bytes)) return "—"; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`; +}; + +function artifactLinks(artifacts, scope, ownerId) { + if (!artifacts.length) return '

no viewable local JSON artifacts

'; + return `
${artifacts.map((artifact) => ` + + ${esc(artifact.label)} + ${esc(artifact.format.toUpperCase())} · ${esc(humanBytes(artifact.size_bytes))} + `).join("")}
`; +} function statusChip(summary) { if (summary.load_error) { @@ -142,6 +159,7 @@ function covCard(label, numerator, denominator, unit) { async function viewWorkflowDetail(id, policy) { const detail = await api(`/api/workflows/${id}${policy ? `?policy=${enc(policy)}` : ""}`); const summary = detail.summary; + const artifacts = await api(`/api/artifacts/json/workflows/${id}`); if (detail.load_error) { return `

${esc(summary.id)}

unreadable

${esc(detail.load_error)}
@@ -237,6 +255,9 @@ async function viewWorkflowDetail(id, policy) { ${lintRows ? `${lintRows}
SeverityCode
` : '

no coverage gaps found

'}

Versions & diffs

${diffSelector} +

Local artifacts

+

Exact protected bundle JSON. It stays on this computer and is shown only in this authenticated console.

+ ${artifactLinks(artifacts, "workflows", summary.id)}

Actions

${renderActions(actions, `/api/workflows/${id}/actions/`)}`; } @@ -389,6 +410,7 @@ async function viewRunDetail(id) { return `

${esc(summary.id)}

${esc(summary.load_error)}
`; } const actions = await api(`/api/runs/${id}/actions`); + const artifacts = await api(`/api/artifacts/json/runs/${id}`); let haltHtml = ""; if (detail.halt) { @@ -466,9 +488,208 @@ async function viewRunDetail(id) { ${detail.checkpoints.length ? `
${safeNumber(detail.checkpoints.length, "0")} durable checkpoint(s) ${detail.checkpoints.map((checkpoint) => ``).join("")}
FileStepAt
protected${safeNumber(checkpoint.step_index)}${fmtTime(checkpoint.created_at)}
` : ""} +

Local evidence artifacts

+

Exact protected run JSON/JSONL. It stays on this computer and is shown only in this authenticated console.

+ ${artifactLinks(artifacts, "runs", summary.id)}

Actions

${renderActions(actions, `/api/runs/${id}/actions/`)}`; } +function pointerToken(value) { + return String(value).replaceAll("~", "~0").replaceAll("/", "~1"); +} + +function pointerTokens(pointer) { + if (pointer === "") return []; + if (!pointer.startsWith("/")) throw new Error("JSON Pointer must start with /"); + return pointer.slice(1).split("/").map((token) => { + if (/~(?:[^01]|$)/.test(token)) throw new Error("JSON Pointer escape is invalid"); + return token.replaceAll("~1", "/").replaceAll("~0", "~"); + }); +} + +function resolveJsonPointer(document, pointer) { + let value = document; + for (const token of pointerTokens(pointer)) { + if (Array.isArray(value)) { + if (!/^(0|[1-9][0-9]*)$/.test(token) || Number(token) >= value.length) { + throw new Error("JSON Pointer does not exist"); + } + value = value[Number(token)]; + } else if (value && typeof value === "object" && Object.hasOwn(value, token)) { + value = value[token]; + } else { + throw new Error("JSON Pointer does not exist"); + } + } + return value; +} + +function childPointer(parent, key) { + return `${parent}/${pointerToken(key)}`; +} + +function jsonValueClass(value) { + if (value === null) return "null"; + if (typeof value === "string") return "string"; + if (typeof value === "number") return "number"; + if (typeof value === "boolean") return "boolean"; + return "compound"; +} + +function jsonPrimitive(value) { + return value === null ? "null" : JSON.stringify(value); +} + +function pointerHref(baseRoute, pointer) { + return `${baseRoute}?pointer=${enc(pointer)}`; +} + +function renderJsonChildren(value, pointer, targetPointer, baseRoute) { + if (Array.isArray(value)) { + let html = ""; + for (let index = 0; index < value.length; index += 1) { + html += renderJsonNode(String(index), value[index], childPointer(pointer, index), targetPointer, baseRoute); + } + return html; + } + let html = ""; + for (const key of Object.keys(value)) { + html += renderJsonNode(key, value[key], childPointer(pointer, key), targetPointer, baseRoute); + } + return html; +} + +function renderJsonNode(key, value, pointer, targetPointer, baseRoute) { + const keyHtml = key === null + ? 'document' + : `${esc(JSON.stringify(key))}:`; + const deepLink = `#`; + if (value === null || typeof value !== "object") { + return `
${keyHtml} + ${esc(jsonPrimitive(value))}${deepLink}
`; + } + const count = Array.isArray(value) ? value.length : Object.keys(value).length; + const preview = Array.isArray(value) ? `Array(${count})` : `{${count}}`; + const open = pointer === "" || targetPointer === pointer || targetPointer.startsWith(`${pointer}/`); + return `
+ ${keyHtml}${esc(preview)}${deepLink} +
${open ? renderJsonChildren(value, pointer, targetPointer, baseRoute) : ""}
+
`; +} + +function searchJsonDocument(document, query, baseRoute) { + const needle = query.trim().toLocaleLowerCase(); + if (!needle) return ""; + const matches = []; + const stack = [{value: document, pointer: "", key: "document"}]; + while (stack.length && matches.length < 100) { + const item = stack.pop(); + const primitive = item.value === null || typeof item.value !== "object" + ? jsonPrimitive(item.value) + : ""; + if (`${item.key} ${item.pointer} ${primitive}`.toLocaleLowerCase().includes(needle)) { + matches.push(item); + } + if (Array.isArray(item.value)) { + for (let index = item.value.length - 1; index >= 0; index -= 1) { + stack.push({ + value: item.value[index], + pointer: childPointer(item.pointer, index), + key: String(index), + }); + } + } else if (item.value && typeof item.value === "object") { + const keys = Object.keys(item.value); + for (let index = keys.length - 1; index >= 0; index -= 1) { + const key = keys[index]; + stack.push({value: item.value[key], pointer: childPointer(item.pointer, key), key}); + } + } + } + if (!matches.length) return '

No matches.

'; + return `
${matches.map((match) => ` + + ${esc(match.pointer || "/")} + ${esc(match.key)} + ${match.value === null || typeof match.value !== "object" + ? `${esc(jsonPrimitive(match.value).slice(0, 120))}` + : ""} + `).join("")}
${matches.length === 100 ? '

Showing the first 100 matches.

' : ""}`; +} + +async function viewJsonArtifact(scope, ownerId, artifactId, queryParams) { + const artifact = await api(`/api/artifacts/json/${scope}/${ownerId}/${artifactId}`); + const section = scope === "runs" ? "runs" : "workflows"; + const backRoute = `#/${section}/${enc(ownerId)}`; + const baseRoute = `${backRoute}/artifacts/${enc(artifactId)}`; + const requestedPointer = queryParams.get("pointer") || ""; + let targetPointer = requestedPointer; + let pointerError = ""; + try { + resolveJsonPointer(artifact.document, requestedPointer); + } catch (error) { + targetPointer = ""; + pointerError = error.message || "JSON Pointer is invalid"; + } + currentJsonViewer = { + scope, + ownerId, + artifactId, + artifact, + baseRoute, + targetPointer, + }; + return `

← Back to ${section === "runs" ? "run" : "workflow"}

+
+

${esc(artifact.label)}

+

sha256 ${esc(artifact.sha256)}

+
+ + + + +
+
+
+ ${esc(artifact.format.toUpperCase())} + ${esc(humanBytes(artifact.size_bytes))} + ${safeNumber(artifact.node_count, "0")} nodes + depth ${safeNumber(artifact.max_depth, "0")} + local only +
+ ${pointerError ? `

${esc(pointerError)}

` : ""} + + +
+ +
${renderJsonNode(null, artifact.document, "", targetPointer, baseRoute)}
`; +} + +async function fetchCurrentJsonRaw() { + if (!currentJsonViewer) throw new Error("No JSON artifact is open"); + const {scope, ownerId, artifactId} = currentJsonViewer; + const response = await fetch( + `/api/artifacts/json/${scope}/${enc(ownerId)}/${enc(artifactId)}/raw`, + {headers: requestHeaders()}, + ); + if (!response.ok) throw new Error("The local artifact is no longer available"); + return response; +} + +function activateJsonViewer() { + if (!currentJsonViewer || !currentJsonViewer.targetPointer) return; + requestAnimationFrame(() => { + const target = [...document.querySelectorAll("[data-json-pointer]")].find( + (node) => node.dataset.jsonPointer === currentJsonViewer.targetPointer, + ); + if (target) { + target.classList.add("json-target"); + target.scrollIntoView({block: "center"}); + } + }); +} + async function viewSkills() { const libraries = await api("/api/skills"); currentSkillActions = []; @@ -639,19 +860,28 @@ async function route() { link.classList.toggle("active", link.dataset.nav === nav)); const main = $("#main"); clearArtifactObjectUrls(); + currentJsonViewer = null; main.innerHTML = '

Loading…

'; try { let html; const diffIndex = parts.indexOf("diff"); + const artifactIndex = parts.indexOf("artifacts"); + const queryParams = new URLSearchParams(query || ""); if (nav === "attention") { html = await viewAttention(); + } else if ((nav === "workflows" || nav === "runs") && artifactIndex > 0) { + html = await viewJsonArtifact( + nav, + enc(parts.slice(1, artifactIndex).join("/")), + enc(parts[artifactIndex + 1] || ""), + queryParams, + ); } else if (nav === "workflows" && diffIndex > 0) { html = await viewDiff( enc(parts.slice(1, diffIndex).join("/")), enc(parts.slice(diffIndex + 1).join("/")), ); } else if (nav === "workflows" && parts.length > 1) { - const queryParams = new URLSearchParams(query || ""); html = await viewWorkflowDetail( enc(parts.slice(1).join("/")), queryParams.get("policy"), @@ -667,12 +897,54 @@ async function route() { } main.innerHTML = html; await hydrateAuthenticatedImages(); + activateJsonViewer(); } catch (error) { main.innerHTML = `

error

${esc(JSON.stringify(error.body || String(error), null, 2))}
`; } } document.addEventListener("click", async (event) => { + const jsonAction = event.target.closest("[data-json-action]"); + if (jsonAction && currentJsonViewer) { + const action = jsonAction.dataset.jsonAction; + const actionStatus = $("#json-action-status"); + if (actionStatus) actionStatus.textContent = ""; + jsonAction.disabled = true; + const originalLabel = jsonAction.textContent; + try { + const response = await fetchCurrentJsonRaw(); + if (action === "copy") { + await navigator.clipboard.writeText(await response.text()); + jsonAction.textContent = "Copied"; + if (actionStatus) actionStatus.textContent = "Exact artifact copied."; + } else if (action === "raw") { + const raw = $("#json-raw"); + raw.textContent = await response.text(); + raw.hidden = !raw.hidden; + jsonAction.textContent = raw.hidden ? "Raw" : "Hide raw"; + } else if (action === "download") { + const objectUrl = URL.createObjectURL(await response.blob()); + const link = document.createElement("a"); + link.href = objectUrl; + link.download = currentJsonViewer.artifact.download_name || "openadapt-artifact.json"; + link.hidden = true; + document.body.append(link); + link.click(); + link.remove(); + window.setTimeout(() => URL.revokeObjectURL(objectUrl), 0); + if (actionStatus) actionStatus.textContent = "Download started."; + } + } catch (error) { + jsonAction.textContent = "Unavailable"; + if (actionStatus) actionStatus.textContent = "The protected local artifact could not be read."; + } finally { + jsonAction.disabled = false; + if (action !== "raw") { + window.setTimeout(() => { jsonAction.textContent = originalLabel; }, 1200); + } + } + return; + } const routeTarget = event.target.closest("[data-route]"); if (routeTarget && !event.target.closest("a")) { window.location.hash = routeTarget.dataset.route; @@ -710,6 +982,31 @@ document.addEventListener("click", async (event) => { } }); +document.addEventListener("input", (event) => { + if (event.target.id !== "json-search" || !currentJsonViewer) return; + const output = $("#json-search-results"); + output.innerHTML = searchJsonDocument( + currentJsonViewer.artifact.document, + event.target.value, + currentJsonViewer.baseRoute, + ); +}); + +document.addEventListener("toggle", (event) => { + const node = event.target; + if (!node.matches || !node.matches("details.json-node") || !node.open || !currentJsonViewer) return; + const children = node.querySelector(":scope > [data-json-children]"); + if (!children || children.dataset.hydrated === "true") return; + const pointer = node.dataset.jsonPointer || ""; + try { + const value = resolveJsonPointer(currentJsonViewer.artifact.document, pointer); + children.innerHTML = renderJsonChildren(value, pointer, "", currentJsonViewer.baseRoute); + children.dataset.hydrated = "true"; + } catch (error) { + children.textContent = "This JSON Pointer is no longer available."; + } +}, true); + $("#modal-cancel").addEventListener("click", closeModal); $("#modal-go").addEventListener("click", () => pendingExec && pendingExec()); window.addEventListener("hashchange", route); diff --git a/public-artifacts.json b/public-artifacts.json index ed6a8f1c..cf6b2d90 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -1773,11 +1773,11 @@ }, { "path": "openadapt_flow/console/static/console.css", - "sha256": "52ef899df24e3f98457326a2464a9ecc41ed01d7419fb8d088e4827c07c960aa" + "sha256": "15360619d73d65f83188d9fe0f2e52d4ddc7e6e6ce7c8dc4d58c2bb71cafc351" }, { "path": "openadapt_flow/console/static/console.js", - "sha256": "40f5b62e64c9677a63ad51498ac8c86b849b57dbf45327478e26ab1a5e8bcba0" + "sha256": "1ef048cdc535b022c12dd75b4a1cb8421f10d04221439884de0222261bc6aa1d" }, { "path": "openadapt_flow/console/static/index.html", diff --git a/tests/test_console.py b/tests/test_console.py index 233763b5..8d9207df 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -18,6 +18,7 @@ from __future__ import annotations +import hashlib import inspect import json import re @@ -286,6 +287,10 @@ def _run_id(client: TestClient, *, halted: bool = False, paused: bool = False) - ) +def _json_artifact_id(relative_path: str) -> str: + return hashlib.sha256(relative_path.encode("utf-8")).hexdigest()[:24] + + # --------------------------------------------------------------------------- # 1. read surface # --------------------------------------------------------------------------- @@ -465,6 +470,107 @@ def test_artifact_served_and_traversal_refused(console_env): assert r.status_code == 404, evil +def test_explicit_local_json_viewer_preserves_exact_run_and_bundle_artifacts( + console_env, +): + client = _client(console_env) + run_id = _run_id(client, halted=True) + evidence = console_env["halted"] / "evidence" + evidence.mkdir() + events = b'{"event":"resolve"}\n{"event":"verify"}\n' + (evidence / "events.jsonl").write_bytes(events) + + listed = client.get(f"/api/artifacts/json/runs/{run_id}").json() + by_id = {item["id"]: item for item in listed} + report_id = _json_artifact_id("report.json") + events_id = _json_artifact_id("evidence/events.jsonl") + assert by_id[report_id]["label"] == "Run report" + assert by_id[events_id]["label"].startswith("Run evidence ") + assert "evidence/events.jsonl" not in str(listed) + + report = client.get(f"/api/artifacts/json/runs/{run_id}/{report_id}") + assert report.status_code == 200 + body = report.json() + assert body["document"]["params"]["patient_id"] == "MRN-SECRET" + assert ( + body["sha256"] + == hashlib.sha256( + (console_env["halted"] / "report.json").read_bytes() + ).hexdigest() + ) + raw = client.get(f"/api/artifacts/json/runs/{run_id}/{report_id}/raw") + assert raw.content == (console_env["halted"] / "report.json").read_bytes() + assert raw.headers["x-content-sha256"] == body["sha256"] + + jsonl = client.get(f"/api/artifacts/json/runs/{run_id}/{events_id}").json() + assert jsonl["document"] == [{"event": "resolve"}, {"event": "verify"}] + assert ( + client.get(f"/api/artifacts/json/runs/{run_id}/{events_id}/raw").content + == events + ) + + bundle_id = _workflow_id(client, n_steps=3) + bundle_artifacts = client.get(f"/api/artifacts/json/workflows/{bundle_id}").json() + assert {item["label"] for item in bundle_artifacts} >= { + "Compiled workflow", + "Bundle manifest", + } + + +def test_local_json_viewer_refuses_secrets_symlinks_and_resource_abuse( + console_env, tmp_path +): + client = _client(console_env) + run_id = _run_id(client, halted=True) + run = console_env["halted"] + (run / "params.json").write_text('{"secret":"never-list"}') + (run / "attended_capability.json").write_text('{"capability":"never-list"}') + sensitive_step = "Jane-Roe-MRN-7788" + heal_dir = run / "heals" / sensitive_step + heal_dir.mkdir(parents=True) + (heal_dir / "heal.json").write_text('{"status":"proposed"}') + evidence = run / "evidence" + evidence.mkdir() + outside = tmp_path / "outside.json" + outside.write_text('{"outside":true}') + linked = evidence / "linked.json" + try: + linked.symlink_to(outside) + except OSError: + pass + (evidence / "too-large.json").write_bytes(b" " * (5 * 1024 * 1024 + 1)) + (evidence / "too-deep.json").write_text("[" * 102 + "0" + "]" * 102) + (evidence / "too-many.json").write_text("[" + ",".join(["0"] * 20_001) + "]") + (evidence / "invalid.json").write_text("{not json") + + listed = client.get(f"/api/artifacts/json/runs/{run_id}").json() + listed_ids = {item["id"] for item in listed} + assert _json_artifact_id("params.json") not in listed_ids + assert _json_artifact_id("attended_capability.json") not in listed_ids + assert _json_artifact_id("evidence/linked.json") not in listed_ids + assert "linked.json" not in str(listed) + assert sensitive_step not in str(listed) + assert _json_artifact_id(f"heals/{sensitive_step}/heal.json") in listed_ids + assert ( + client.get(f"/api/artifacts/json/runs/{run_id}/{'0' * 24}").status_code == 404 + ) + for label in ( + "evidence/too-large.json", + "evidence/too-deep.json", + "evidence/too-many.json", + ): + artifact_id = _json_artifact_id(label) + assert artifact_id in listed_ids + assert ( + client.get(f"/api/artifacts/json/runs/{run_id}/{artifact_id}").status_code + == 413 + ) + invalid_id = _json_artifact_id("evidence/invalid.json") + assert ( + client.get(f"/api/artifacts/json/runs/{run_id}/{invalid_id}").status_code == 422 + ) + + # --------------------------------------------------------------------------- # 3. governance actions: catalog + fail-closed read-only gate # --------------------------------------------------------------------------- @@ -824,7 +930,7 @@ def test_static_ui_has_no_inline_handlers_or_artifact_paths(console_env): assert "payload.approver" not in script -def test_sensitive_bundle_and_run_values_never_cross_api(console_env): +def test_sensitive_bundle_and_run_values_never_cross_summary_api(console_env): client = _client(console_env) bundle_id = _workflow_id(client, n_steps=3) workflow = client.get(f"/api/workflows/{bundle_id}").json()