diff --git a/BRIEFING.md b/BRIEFING.md new file mode 100644 index 00000000000..9e80d9e26e0 --- /dev/null +++ b/BRIEFING.md @@ -0,0 +1,371 @@ +# Inherent Capability Optimization Briefing + +## Objective + +Improve Aura's inherent harness capabilities without restoring runtime or Superpowers as visible skills. The immediate defect is the JVM dependency-analysis path; the broader goal is a smaller, clearer prompt contract with stronger, task-specific tool-selection telemetry. + +This plan starts from commit `f1efa00d36d4608a0cdbfa3d768d81dd6f0fa541` (`feat(coding-agent): make harness capabilities inherent`). The valid focused comparison against pinned vanilla OMP revision `06343fe4200c4e32d18f08df5a6a8bd84dcc710` established: + +- Both products passed all six focused trials. +- Aura used 28 tool calls versus vanilla's 34 and 280,194 input tokens versus 305,753. +- Aura selected `run` before `bash` in all TypeScript trials. +- Aura selected `jvm_deps` before `bash` in only two of three JVM trials. +- The failing JVM selection trace redundantly compiled `Report.java`, attempted to execute `Report.class` through `run`, then repeated dependency analysis. +- The focused benchmark incorrectly reported `INCONCLUSIVE` because the broad adoption gate treated unrepresented capability groups as 0% adoption. + +The performance problem is therefore model behavior at the JVM tool boundary, not runtime execution. Adapter microbenchmarks already show the runtime path itself is faster than the shell baseline for Java compile/run. + +## Non-negotiable architecture + +1. Runtime and core workflow policy remain inherent prompt/tool capabilities. They MUST NOT reappear in skill discovery, `/skill`, skill cards, promoted-skill telemetry, or bundled-skill materialization. +2. Cross-tool selection policy belongs in the high-order system prompt. Mechanical invocation rules belong in tool prompts and schemas. The same rule MUST NOT be repeated at all three layers. +3. `run` and `check` remain the default execution and validation capabilities. Specialized JVM tools own formatting, bytecode, JAR, and dependency workflows. +4. Successful tool output is evidence. The model MUST NOT rerun an equivalent shell or runtime command merely to reconfirm it. +5. Benchmark tasks remain deterministic, dependency-free, and task-specific. The focused loop stays small; the broad suite is a regression gate, not the tuning loop. +6. Historical vanilla OMP is a whole-product control only. The causal comparison for each optimization is the frozen pre-change Aura binary versus the changed Aura binary. +7. No new tool, alias, compatibility shim, runtime skill, or benchmark-only production branch is permitted. + +## Success criteria + +### Focused JVM/TypeScript gate + +Run three alternating attempts per task against the frozen pre-change Aura binary. The changed binary MUST satisfy all of the following: + +- 6/6 verifier passes and zero harness errors. +- TypeScript: `run` is the first capability tool in 3/3 trials. +- JVM dependencies: `jvm_deps` is the first capability tool in 3/3 trials. +- Zero promoted core runtime/Superpowers skill loads. +- Zero `bash` calls before the first successful expected capability call. +- Zero repeated calls to the expected capability after its first successful result unless intervening source changes invalidate that result. +- TypeScript median tool calls do not exceed the pre-change median. +- JVM median tool calls are at most 4, with no trial above 5. +- JVM median input tokens do not increase relative to the matched pre-change arm. +- The focused report returns `PASS`, not `INCONCLUSIVE`, when all represented tasks meet their gates. + +Duration and cost remain reported but are not standalone prompt-adoption gates because provider latency and pricing noise can move independently of tool choice. + +### Broad regression gate + +After the focused gate passes, run the full task matrix with three attempts per arm. The changed binary MUST satisfy: + +- Every task passes 3/3; no systematic runtime-only error class appears. +- Every eligible trial uses its declared expected capability. +- Expected capability selected first in at least 90% of eligible trials, with no represented group below 80%. +- Zero promoted core runtime/Superpowers skill loads. +- No task-level pass-rate regression. +- Existing paired efficiency gate passes: either meaningful duration improvement with token safety, or meaningful token improvement with duration safety. +- Groups with no eligible task are displayed as `n/a` and are excluded from adoption gates. + +`serve` remains outside Harbor's model-behavior suite because it requires a live hub supervisor and daemon lifecycle. Its real launch/readiness/stop contracts remain covered by coding-agent integration tests. + +## Phase 1 — Freeze controls and make benchmark telemetry truthful + +### 1. Preserve immutable comparison artifacts + +Before production edits: + +- Build and copy the coding-agent binary at `f1efa00d36d4608a0cdbfa3d768d81dd6f0fa541` to a source-mounted, ignored benchmark-artifact directory. +- Preserve the embedded runtime library used by that build. +- Preserve or rebuild vanilla OMP at revision `06343fe4200c4e32d18f08df5a6a8bd84dcc710` in the same source-mounted artifact directory. +- Record SHA-256 values for both binaries, the embedded library, task fixtures, system prompt, tool prompts, and tool registration in each benchmark manifest. +- Confirm both branded config trees, `~/.aura/agent` and `~/.omp/agent`, receive generated gateway configuration. Credentials remain host-side. + +No treatment prompt may be edited before these hashes are recorded. + +### 2. Add a shared ordered transcript analyzer + +Create `packages/metaharness/src/runtime-transcript.ts` and its focused test file. Move JSONL tool-event interpretation out of the two benchmark entrypoints into this module. + +The analyzer must: + +- Ignore malformed or truncated trailing JSONL records. +- Preserve tool-call order. +- Pair `tool_execution_start` and `tool_execution_end` by call ID. +- Record total calls, capability calls, first capability tool, and successful capability calls. +- Detect reads of promoted core runtime or Superpowers skills, including namespaced `skill://superpowers:` URLs. +- Given an expected capability, report: + - whether it was used; + - whether it was the first capability tool; + - whether `bash` ran before its first success; + - whether the same capability was repeated after success; + - whether a source-changing `write` or `edit` occurred between successful repeats. +- Treat an expected-tool repeat as redundant only when no intervening source mutation could invalidate the earlier result. + +Tests must use representative transcript events and assert behavior, not source text. Cover a truncated final line, failed-then-successful retry, successful repeat without mutation, and valid repeat after an edit. + +### 3. Declare task-specific expected capabilities + +Update `RuntimeTaskDefinition` in `packages/metaharness/src/runtime-benchmark-suite.ts` with an optional `expectedCapability` field. Populate it explicitly: + +| Task | Expected capability | +|---|---| +| `python-execution` | `run` | +| `typescript-execution` | `run` | +| `project-validation` | `check` | +| `runtime-debugging` | `run` | +| `instrumentation` | `insights` | +| `cpu-sampling` | `profile` | +| `call-tracing` | `insights` | +| `java-execution` | `run` | +| `bytecode-inspection` | `jvm_disassemble` | +| `executable-jar` | `jvm_jar` | +| `jvm-dependencies` | `jvm_deps` | + +Replace the stale `project-build` fixture, which no longer targets an inherent tool after `build` was removed, with a deterministic `jvm-formatting` fixture whose `runtimeTools` is `["jvm_format"]` and whose expected capability is `jvm_format`. Add `jvm_format` to `TaskRuntimeTool`. The fixture must supply malformed Java source and an exact expected formatted artifact, require `jvm_format` to write the artifact, then compile and execute it. Its verifier must compare the generated artifact with the expected artifact and assert the program's exact output. This checks files produced by the task, not implementation source. + +Keep `runtimeTools` as the tool-exposure list. `expectedCapability` is the measurement contract, not an inferred alias for the first element. + +### 4. Fix adoption denominators and report selection quality + +Update `packages/metaharness/src/runtime-benchmark.ts` and `packages/metaharness/src/inherent-capability-benchmark.ts` to consume the shared analyzer. + +Required metric changes: + +- Preserve `runtimeUsed` for backwards-compatible report context, but calculate adoption from `expectedCapabilityUsed` on eligible tasks. +- Exclude tasks without `expectedCapability` from adoption denominators. +- Represent groups with zero eligible trials as `null`/`n/a`, never `0%`. +- Gate only represented groups. +- Add overall and per-group first-capability selection rates. +- Add counts for promoted core skill loads, shell-before-capability, and redundant post-success repeats. +- Add the same facts to the historical report arm without treating historical deltas as causal. +- Include task-level tool-call and input-token rows so a group aggregate cannot hide a single regressing task. + +Add regression tests proving: + +- A two-task focused suite can pass without being penalized for absent project/debugging/profiling groups. +- An eligible task that uses the wrong runtime tool does not count as adopted. +- `n/a` groups render correctly. +- Shell-before-success and redundant-repeat failures produce explicit verdict reasons. +- Historical rows render but do not change the current-versus-baseline verdict. +- Existing bootstrap confidence intervals remain deterministic. + +## Phase 2 — Harden the JVM dependency boundary + +Use test-driven development: add failing observable-contract tests, run them, then change production code. + +### 5. Reject compiled artifacts through `run` + +The bad trace passed `Report.class` with `language: "java"`, bypassing extension inference and causing the runtime to read bytecode as source. Fix the shared validation boundary in `packages/coding-agent/src/runtime/protocol.ts`, inside `resolveRunTarget`, so every process, embedded, and Bun endpoint sees the same rule. + +Contract: + +- `run({ path: "Report.class", language: "java" })` fails with `invalid-params` before endpoint side effects. +- `.jar` paths fail identically. +- The error states that `run` executes source files, not compiled JVM artifacts, and directs dependency inspection to `jvm_deps`; execution of project-built artifacts remains a project toolchain command. +- Valid `.java`, `.kt`, `.js`, `.ts`, and `.py` paths remain unchanged. +- Inline Java/Kotlin source remains unchanged. + +Add protocol/endpoint tests that assert the surfaced error code and message and prove the endpoint was not called. Do not duplicate validation in individual adapters. + +### 6. Make the direct `jvm_deps` route unmistakable + +Tighten these surfaces together: + +- `packages/coding-agent/src/prompts/tools/jvm-deps.md` +- `packages/coding-agent/src/tools/jvm-deps.ts` +- Relevant rendered tool-schema/tool-prompt contract tests + +The contract must say, once and concisely: + +- A `.java` or `.kt` `path` is compiled in scratch space before `jdeps`; precompilation is unnecessary. +- A `.class`, `.jar`, or class-directory `path` is analyzed directly. +- `language` plus `code` is the inline-source alternative. +- `output` writes the report while the tool result also returns it; rereading the output solely to confirm a successful call is redundant. +- Prefer the source path when the task starts from source. + +Do not add this mechanical detail to the high-order system prompt. That prompt already owns the broader rules: use the specialized JVM tool and do not repeat successful equivalent work. + +Add or extend `packages/coding-agent/test/jvm-tools.test.ts` to prove a source path compiles and produces dependency output in one call, including `java.sql`, and that an output file receives the same successful report. Use the real runtime service seam already used by the JVM contract tests; no `mock.module()` and no source-grep assertions. + +### 7. Preserve concise prompt size + +Measure before and after with: + +```bash +bun scripts/tool-prompt-usage.ts --json \ + packages/coding-agent/src/prompts/tools/runtime-run.md \ + packages/coding-agent/src/prompts/tools/jvm-deps.md +``` + +The combined rendered token count must not increase. If the new JVM distinction requires added words, remove an equal or larger amount of schema-derivable or duplicated prose from the same two prompts. Record both measurements in the benchmark report or implementation notes, not in production comments. + +## Phase 3 — Run the focused causal comparison + +### 8. Build and preflight the treatment + +- Build the changed coding-agent binary. +- Run `bun check` in `packages/coding-agent` and `packages/metaharness`. +- Run focused tests for protocol resolution, JVM tools, runtime endpoint selection, system-prompt inventory, transcript analysis, and both benchmark analyzers. +- Run the embedded-runtime telemetry preflight and generated TypeScript verifier smoke. +- Confirm the treatment manifest hashes differ only where expected. + +### 9. Execute the matched focused benchmark + +Run `bun run bench:inherent` from `packages/metaharness` with: + +- model `openai-codex/gpt-5.6-sol`; +- thinking `high`; +- three attempts; +- tasks `typescript-execution` and `jvm-dependencies`; +- alternating treatment/control order; +- frozen pre-change Aura binary as the legacy/control arm; +- changed source build as the inherent/treatment arm; +- the same gateway, task fixtures, embedded library selection, and verifier image for both arms. + +Do not tune against partial trials. Let the complete six-pair campaign finish, then read the aggregate report and all JVM traces. + +If any focused criterion fails: + +1. Classify the failure as selection, schema misuse, tool-result misunderstanding, runtime defect, or verifier defect. +2. Fix the narrowest owning layer. +3. Add a contract test for any runtime defect. +4. Preserve a new control binary before the next prompt treatment. +5. Rerun the complete matched campaign under a new prefix; never overwrite or combine attempts across treatments. + +The phase is complete only when the focused report is `PASS` and the raw traces satisfy the zero-shell-before-success and zero-redundant-repeat criteria. + +## Phase 4 — General inherent-prompt simplification + +Only begin after Phase 3 passes. This prevents a broad prompt rewrite from hiding the JVM cause. + +### 10. Audit the inherent prompt layers + +Scope: + +- The inherent capability block in `packages/coding-agent/src/prompts/system/system-prompt.md`. +- `runtime-run.md`, `runtime-check.md`, `runtime-insights.md`, `runtime-profile.md`, `runtime-serve.md`. +- `jvm-disassemble.md`, `jvm-format.md`, `jvm-jar.md`, `jvm-deps.md`. +- Their tool schemas under `packages/coding-agent/src/tools/`. + +For every instruction, assign exactly one owner: + +- System prompt: precedence, selection, lifecycle, and cross-tool safety. +- Tool prompt: workflow-specific mechanics and output interpretation. +- Schema description: field meaning, allowed combinations, defaults, and units. + +Delete duplicates from lower-value layers. Keep the system block short enough to remain visible on every request, but do not trade away a decision rule for token savings. Never move inherent policy into a skill. + +Specific cleanup targets: + +- Remove shell-policy repetition from individual runtime prompts when the system prompt already states the rule. +- Keep source-versus-path and output semantics next to the affected tool. +- Keep hub lifecycle policy in the system/serve boundary; do not teach a second stop mechanism. +- Keep `run`/specialized-JVM precedence in one high-order sentence plus the concrete specialist prompt. +- Preserve exact failure guidance that prevents a plausible wrong tool call; remove marketing and implementation plumbing. + +Measure all changed prompts before and after with `scripts/tool-prompt-usage.ts` for both `o200k_base` and `cl100k_base`. Total rendered tokens across the changed prompt set MUST decrease, and no individual prompt may grow without an explicit behavior-protecting reason in the report. + +### 11. Protect implicit UI behavior + +Rerun the existing skill discovery, slash-command, skill-message, settings, and system-prompt inventory contracts. Add a test only if a real user-visible path is uncovered. Required result: + +- No runtime or core workflow pseudo-skill appears in discovery or UI. +- No removed bundled runtime skill is rematerialized. +- `skill://runtime` is not restored as a supported path. +- Tools remain discoverable through their schemas, system inventory, and `xd://` protocol surface. + +## Phase 5 — Broad behavior regression and vanilla comparison + +### 12. Run the full current-versus-baseline suite + +Run `bun run bench:runtime` with all task fixtures and three attempts per arm. The baseline arm receives file/shell tools; the runtime arm receives `run`, `check`, and only the specialist required by each task. Alternate arm order and preserve one job per task/arm. + +Evaluate: + +- verifier pass and error rates; +- expected-capability adoption and first selection; +- promoted skill loads; +- shell-before-success and redundant-repeat counts; +- paired duration, input, output, cache, cost, and tool-call deltas; +- task and group breakdowns; +- bootstrap confidence intervals; +- runtime telemetry success/failure durations. + +The report must use the revised eligibility-aware gate. A missing capability group is `n/a`; a represented low-adoption group is a real failure. + +### 13. Run the pinned vanilla whole-product control once + +After the current-versus-baseline gate passes, run the same task matrix with the pinned vanilla OMP binary as the historical arm. Use the already-correct dual branded config staging. Report: + +- pass/error rates; +- tool calls and token totals; +- expected first-capability selection; +- promoted skill loads; +- task-specific deltas; +- duration and cost as noisy observational metrics. + +Label this section `Historical whole-product control`. It MUST NOT alter the causal benchmark verdict because code revision, product branding, prompt architecture, and runtime availability differ simultaneously. + +## Phase 6 — Verification and cleanup + +### 14. Repository verification + +Run, in order: + +1. Focused coding-agent contract tests for every changed runtime/tool/prompt surface. +2. Focused metaharness tests for transcript facts, suite materialization, report formatting, and verdict logic. +3. `bun check` in `packages/coding-agent`. +4. `bun check` in `packages/metaharness`. +5. Root `bun check`. +6. Root `bun run test:ts`. + +If root tests expose an unrelated pre-existing failure, rerun the exact failing package/test to distinguish it and report the evidence. Do not weaken or skip a relevant failing test. + +### 15. Documentation and fork inventory + +After successful smoke and benchmark verification: + +- Update `packages/metaharness/README.md` with the expected-capability metrics, `n/a` adoption semantics, focused control procedure, and historical-control caveat. +- Update `docs/aura/FORK.md` for every newly touched upstream file and every new fork-owned benchmark file. +- Update `packages/coding-agent/CHANGELOG.md` under `[Unreleased]` for the user-visible compiled-artifact error and direct source-path dependency guidance. +- Remove obsolete benchmark fields, duplicated scanner logic, stale project-build fixture references, and temporary debug output. +- Keep run artifacts ignored; preserve report paths and hashes in the final evidence summary. +- Do not commit or push unless explicitly requested. + +## Expected file set + +Planned production changes: + +- `packages/coding-agent/src/runtime/protocol.ts` +- `packages/coding-agent/src/prompts/tools/runtime-run.md` +- `packages/coding-agent/src/prompts/tools/jvm-deps.md` +- `packages/coding-agent/src/tools/jvm-deps.ts` +- `packages/coding-agent/src/prompts/system/system-prompt.md` only if Phase 4 removes proven duplication + +Planned coding-agent tests: + +- `packages/coding-agent/test/runtime-embedded-endpoint.test.ts` +- `packages/coding-agent/test/jvm-tools.test.ts` +- `packages/coding-agent/test/system-prompt-inventory.test.ts` + +Metaharness changes: + +- `packages/metaharness/src/runtime-transcript.ts` (new) +- `packages/metaharness/src/runtime-transcript.test.ts` (new) +- `packages/metaharness/src/runtime-benchmark-suite.ts` +- `packages/metaharness/src/runtime-benchmark-suite.test.ts` +- `packages/metaharness/src/runtime-benchmark.ts` +- `packages/metaharness/src/runtime-benchmark.test.ts` +- `packages/metaharness/src/inherent-capability-benchmark.ts` +- `packages/metaharness/src/inherent-capability-benchmark.test.ts` +- `packages/metaharness/README.md` + +Cleanup/documentation: + +- `packages/coding-agent/CHANGELOG.md` +- `docs/aura/FORK.md` + +## Final evidence format + +The implementation handoff must contain: + +1. Changed contracts and their owning files. +2. Before/after rendered prompt token counts for both encodings. +3. Focused matched benchmark table and verdict. +4. Per-trial JVM tool sequences proving direct `jvm_deps` use. +5. Broad task/group adoption and first-selection table. +6. Historical vanilla comparison clearly labeled non-causal. +7. Exact test/check commands with pass/fail counts. +8. Remaining risks, limited to evidence-backed items. + +No success claim is valid without the focused raw traces, aggregate benchmark report, and repository verification output. \ No newline at end of file diff --git a/docs/aura/FORK.md b/docs/aura/FORK.md index f5c57b0d3f2..4694e19c98f 100644 --- a/docs/aura/FORK.md +++ b/docs/aura/FORK.md @@ -14,7 +14,7 @@ and after every upstream merge. | `packages/coding-agent/src/cli.ts` | env-profile bootstrap reads AURA_PROFILE (canonical) alongside legacy OMP_PROFILE/PI_PROFILE; handles the top-level `--check` flag (one-line clean-env health probe, no model/network/provisioning) and the top-level `--version`/`-v` (identity block: the runner's `/` line unchanged, plus a `runtime protocol v` line) before delegating; dispatches the embedded runtime execution/control Worker selectors through the canonical worker-host re-entry path (dynamic imports — upstream's startup-laziness tests forbid dotenv/native-addon loads in the entry graph, and `worker-core` pulls both), dispatches the isolated Bun run selector, and exercises all three runtime worker graphs in `--smoke-test` | | `packages/coding-agent/src/task/discovery.ts` | `TASK_AGENT_CONFIG_SOURCES` derives from CONFIG_DIR_NAME + LEGACY_CONFIG_DIR_NAME (was a single hardcoded `".omp"`; stale value filtered out all project/user agent dirs after rebrand). Project agent dirs are consumed in priority order so `.aura/agents` beats legacy `.omp/agents` on a name collision, and `projectAgentsDir` only ever reports a writable base | | `packages/coding-agent/package.json` | bin: aura alias alongside omp; runtime dependency `capnp-es@0.0.14` for checked-in embedded-protocol readers/writers, plus package-local `typescript@5.9.3` dev peer so that capnp-es codegen does not resolve the workspace's incompatible native-preview TypeScript 7 package | -| `packages/coding-agent/CHANGELOG.md` | records the opt-in embedded JavaScript/TypeScript/Python `run` adapter under Unreleased while preserving the process adapter as the default | +| `packages/coding-agent/CHANGELOG.md` | records the opt-in embedded JavaScript/TypeScript/Python `run` adapter, inherent runtime/Superpowers policy, compact tool surface, Java/Kotlin routing, and runtime telemetry under Unreleased while preserving the process adapter as the default | | `packages/coding-agent/src/modes/theme/theme.ts` | register built-in `aura` and `aura-light` themes in `BUILTIN_THEMES` (imports + entries), mirroring `dark`/`light` | | `packages/coding-agent/src/utils/title-generator.ts`, `src/modes/interactive-mode.ts`, `src/modes/theme/defaults/{dark,light}-poimandres.json`, `test/terminal-title-state.test.ts`, `test/title-generator.test.ts` | replace upstream's compact `π` prompt/title brand with Aura's `☉`; preserve `icon.pi` as the compatible custom-theme key, use `o` only for the explicit ASCII preset, and keep title-state behavior unchanged apart from the mark | | `packages/coding-agent/src/modes/setup-wizard/scenes/theme.ts` | the wizard is a SECOND source of theme defaults and the path most users actually take, so it is aligned to the brand: the "Match terminal" curated option commits `theme.dark: aura` / `theme.light: aura-light` (was `titanium`/`light`) via the `BRAND_DARK_THEME`/`BRAND_LIGHT_THEME` constants, which must stay equal to the `settings-schema.ts` defaults — a schema default the wizard overwrites is not a default. Its description names the pair; `Titanium`/`Light` remain as named non-default choices (redescribed "Neutral dark/light theme" since they no longer are the defaults) | @@ -22,15 +22,15 @@ and after every upstream merge. | `packages/coding-agent/src/modes/setup-wizard/scenes/splash.ts` | the splash hero renders the aura wordmark: `LARGE_LOGO` doubles `AURA_LOGO` (was `PI_LOGO`), the compact fallback picks `AURA_LOGO`, and the wordmark caption is `SPACED_APP_NAME` (`[...APP_NAME].join(" ")`, i.e. `a u r a`) instead of the literal `"O h M y P i"` | | `packages/coding-agent/src/modes/setup-wizard/scenes/outro.ts` | one symbol: the outro's fading logo sweep renders `AURA_LOGO` | | `packages/coding-agent/src/modes/setup-wizard/wizard-overlay.ts` | one symbol: the wizard scene header renders `AURA_LOGO` | -| `packages/coding-agent/src/tools/renderers.ts` | one import + one `...runtimeToolRenderers` spread at the head of `toolRenderers`, registering the thirteen runtime tool renderers. The spread is first so a future upstream entry with the same key would win rather than be silently shadowed; all renderer logic lives in the fork-owned `tools/runtime-renderer.ts`, so this row stays a two-line change through any merge | +| `packages/coding-agent/src/tools/renderers.ts` | one import + one `...runtimeToolRenderers` spread at the head of `toolRenderers`, registering the nine runtime tool renderers. The spread is first so a future upstream entry with the same key would win rather than be silently shadowed; all renderer logic lives in the fork-owned `tools/runtime-renderer.ts`, so this row stays a two-line change through any merge | | `packages/coding-agent/src/cli/gallery-fixtures/index.ts` | one import + one `...runtimeFixtures` spread, adding the runtime tool family's `omp gallery` sample data (the fixtures themselves are the fork-owned sibling module `gallery-fixtures/runtime.ts`). Without it the coverage test still passes — unfixtured tools fall back to a generic sample — but the runtime rows render as placeholder args | -| `packages/coding-agent/src/config/settings-schema.ts` | `theme.dark` default = `aura` (was `titanium`) and `theme.light` default = `aura-light` (was `light`), so the fork's terminal-background auto light/dark switching stays on-brand in both directions; `runtime.*` settings (`runtime.enabled`, `runtime.adapter` with process default and explicit-embedded no-fallback, `runtime.autoDownload`, `runtime.path`, `runtime.version`, `runtime.embeddedPath`) added to the `tools` tab, plus `skills.enableBundled` (default `true`, `tools`/`Runtime` tab-group) and its `enableBundled?: boolean` field on `SkillsSettings` — `settings.getGroup("skills")` derives the object from the `skills.*` keys, so the new toggle threads to `loadSkills` with no call-site special case; `DEFAULT_BASH_INTERCEPTOR_RULES` retains only the ordinary user-controlled dedicated-tool nudges | +| `packages/coding-agent/src/config/settings-schema.ts` | `theme.dark` default = `aura` (was `titanium`) and `theme.light` default = `aura-light` (was `light`), so the fork's terminal-background auto light/dark switching stays on-brand in both directions; `runtime.*` settings (`runtime.enabled`, `runtime.adapter` with process default and explicit-embedded no-fallback, `runtime.autoDownload`, `runtime.path`, `runtime.version`, `runtime.embeddedPath`) added to the `tools` tab; `DEFAULT_BASH_INTERCEPTOR_RULES` retains only the ordinary user-controlled dedicated-tool nudges | | `packages/coding-agent/src/tools/report-tool-issue.ts`, `src/cli/grievances-cli.ts`, `test/tools/report-tool-issue.test.ts` | default Auto-QA collector copy points at Aura's Elide-operated `qa.elide.dev` endpoint; tests pin the fork default and preserve explicit setting / `PI_AUTO_QA_PUSH_URL` precedence. Keep upstream batching, consent, local retention, and push behavior unchanged when resolving merges | | `packages/coding-agent/src/tools/bash.ts` | selects configured rules through `activeBashInterceptorRules(getBashInterceptorRules(), settings.get("bashInterceptor.enabled"))`; the toggle gates every rule, and direct runtime-binary commands are not intercepted | -| `packages/coding-agent/src/tools/index.ts` | `ToolSession.getRuntimeService?: () => RuntimeService \| undefined` accessor added beside `getMnemopiSessionState`, plus the root-owned `runtimeServiceScope` propagated into every subagent executor; imports the five `Runtime*Tool` classes and registers engine-aware `run` plus `check`/`build`/`insights`/`profile` in `BUILTIN_TOOLS` via their `createIf` gates; imports the five specialized `Jvm*Tool` classes and registers `jvm_disassemble`/`jvm_format`/`jvm_jar`/`jvm_deps`/`jvm_javadoc` on the same `runtime.enabled` gate as discoverable tools; plus `RuntimeDebugTool`/`RuntimeServeTool` registered as `runtime_debug`/`serve`, and `RuntimeAdviceTool` as read-only `project_advice` | +| `packages/coding-agent/src/tools/index.ts` | `ToolSession.getRuntimeService?: () => RuntimeService \| undefined` accessor added beside `getMnemopiSessionState`, plus the root-owned `runtimeServiceScope` propagated into every subagent executor; registers engine-aware `run` and validation-only `check` as essential, `insights`/`profile` plus four specialized JVM tools (`jvm_disassemble`, `jvm_format`, `jvm_jar`, `jvm_deps`) and hub-supervised `serve` as discoverable, all on the `runtime.enabled` gate | | `packages/coding-agent/src/tools/render-utils.ts` | keeps the upstream TUI renderer dependency graph free of the fork's runtime/worker path-boundary helper; runtime diagnostics import the fork-added dependency-light `utils/display-path.ts` directly | -| `packages/coding-agent/src/tools/builtin-names.ts` | appended `run`, `check`, `build`, `insights`, `profile`, the five specialized JVM names `jvm_disassemble`, `jvm_format`, `jvm_jar`, `jvm_deps`, `jvm_javadoc`, `runtime_debug`, `serve`, and `project_advice` to `BUILTIN_TOOL_NAMES`; Java/Kotlin execution is part of `run`, with no `jvm_run` alias or shim | -| `packages/coding-agent/src/tools/essential-tools.ts` | added `run`, `check`, `build` to `ESSENTIAL_BUILTIN_TOOL_NAMES` so they stay top-level and a re-register cannot demote them (issue #5764 guard); `insights`/`profile`, all five specialized `jvm_*` tools, and `project_advice` stay `discoverable` (they are reached through discovery, not the always-on schema) | +| `packages/coding-agent/src/tools/builtin-names.ts` | appended the nine runtime names `run`, `check`, `insights`, `profile`, `jvm_disassemble`, `jvm_format`, `jvm_jar`, `jvm_deps`, and `serve` to `BUILTIN_TOOL_NAMES`; Java/Kotlin execution is part of `run`, with no `jvm_run` alias or shim | +| `packages/coding-agent/src/tools/essential-tools.ts` | added `run` and `check` to `ESSENTIAL_BUILTIN_TOOL_NAMES` so they stay top-level and a re-register cannot demote them (issue #5764 guard); analysis, profiling, JVM, and serve tools stay discoverable | | `packages/coding-agent/src/config/settings.ts` | project config resolves to `/${CONFIG_DIR_NAME}/config.yml` (was a hardcoded `.omp`): `#projectConfigPath()` is the sole write target, `#loadProjectConfigYaml()` falls back to `/.omp/config.yml` for reads (narrowed to the `modelRoles` slice at the `#loadProjectSettings` call site) and seeds the first branded write so legacy keys carry forward. Still required after the legacy-base work — it is the only path that reads a legacy `config.yml` at all. Adapted to upstream's quarantine loader: branded reads go through `#loadYamlIfPresentForStartup`/`#loadYamlIfPresentForWriteLocked` (invalid branded files are moved aside), while legacy `.omp` reads use `#loadLegacyYamlIfPresent` — invalid legacy files error WITHOUT being quarantined, because nothing is ever written into `.omp`, not even a move-aside | | `packages/coding-agent/src/discovery/omp-extension-roots.ts` | project extension roots read `/${CONFIG_DIR_NAME}/settings.json` (was a hardcoded `.omp`) with a `.omp` read fallback via `readProjectSettingsExtensions`; `scopeDirs` folded into the call site. Still required after the legacy-base work: the legacy base contributes no settings documents, so this `extensions` slice remains the only `.omp/settings.json` compat | | `packages/coding-agent/src/config.ts` | `priorityList` entries are typed `ConfigBase` and carry `legacy?`; `.omp` added as a READ-ONLY project base directly below `.aura`. User level excludes it, and pre-rebrand user state under `~/.omp/agent` is consequently on NO read path — `PI_CONFIG_DIR=.omp` is the workaround, proper adoption belongs in a future `config migrate`. `ConfigDirEntry` gained `writable`, which every write path must honor | @@ -47,9 +47,9 @@ and after every upstream merge. | `packages/coding-agent/src/commands/launch.ts` | `prepend-system-prompt: Flags.string(...)` and `runtime: Flags.string(...)` declared for oclif's generated `--help`; the real parse lives in `cli/args.ts` (same pattern as `--auto-approve` / `--approval-mode`) | | `packages/coding-agent/src/cli-commands.ts` | register runtime and doctor commands | | `packages/coding-agent/src/sdk.ts` | `CreateAgentSessionOptions.prependSystemPrompt`, forwarded to `buildSystemPrompt` as `resolvedPrependSystemPrompt` and included in the fork-cache-shape check; wires selected/composite runtime settings onto the lazy `toolSession.getRuntimeService` accessor; creates a root-owned runtime cache/config scope, exposes the canonical settings snapshot reader for private top-level session factories, and propagates that scope into every descendant; every top-level session sharing that scope acquires an idempotent lease while subagents never acquire or release one; startup failure releases the lease; last release asynchronously evicts/closes only that scope | -| `packages/coding-agent/src/discovery/index.ts` | one line: `import "./builtin-skills";` in the provider side-effect import block, registering the bundled runtime-skills provider | -| `packages/coding-agent/src/capability/skill.ts` | added `BUILTIN_SKILLS_PROVIDER_ID = "builtin-skills"`, mirroring `BUILTIN_DEFAULTS_PROVIDER_ID` on the rule capability, so consumers can identify a bundled skill without importing (and registering) the provider | -| `packages/coding-agent/src/extensibility/skills.ts` | `loadSkills` destructures `enableBundled = true` and `isSourceEnabled` gains an explicit `BUILTIN_SKILLS_PROVIDER_ID` branch above the third-party toggles. Load-bearing: the fallback at the end of `isSourceEnabled` returns `anyThirdPartySkillToggleEnabled`, which would let the Codex/Claude/Pi toggles silently retire an agent-native bundled skill (the same class of bug as issue #2401 for managed skills) | +| `packages/coding-agent/src/prompts/system/system-prompt.md`, `src/discovery/claude-plugins.ts` | promotes runtime selection and the universal engineering method into the inherent system layer; routes standalone Java/Kotlin through runtime/JVM tools and project builds through declared project commands; canonical Superpowers workflow skills are filtered only within the canonical plugin provider, while domain skills and same-named user/project skills remain discoverable | +| `packages/coding-agent/src/telemetry/{events,metrics,sink-otlp}.ts` | adds bounded `runtime.call.completed` events, `aura.runtime.calls` and `aura.runtime.duration` instruments, and structured OTLP logs without source, arguments, output, paths, or exception messages | +| `packages/ai/src/providers/cowork-fetch.ts`, `src/providers/openai-codex-responses.ts` | keeps provider source compatible when a workspace consumer enables `lib.dom`: explicitly bridges Node's `Readable.toWeb()` declaration to the WHATWG response stream, avoids Bun's DOM-aware `MessageEvent` constructor/instance inference bug at the WebSocket boundary, and narrows zstd bytes to an `ArrayBuffer`-backed `Uint8Array` without changing the transmitted body shape | | `packages/coding-agent/src/session/agent-session-types.ts` | adds the optional root-owned `disposeRuntimeService` lifecycle callback and inherited `runtimeServiceScope` to `AgentSessionConfig` | | `packages/coding-agent/src/session/agent-session.ts` | retains the inherited runtime scope for descendant creation and accepts the runtime disposal callback only for main sessions; disposal first performs the bounded owned-`AsyncJobManager` drain/cancel so active descendants settle before final runtime release, then runs that release once with the remaining bounded parallel teardown | | `packages/coding-agent/src/commit/agentic/agent.ts` | creates or accepts one explicit root runtime scope for the private commit-agent session and passes the identical object to both the root SDK session and commit-tool factory | @@ -64,22 +64,22 @@ and after every upstream merge. | `packages/coding-agent/src/vibe/runtime.ts`, `packages/coding-agent/test/vibe/vibe-runtime.test.ts` | carries the originating top-level session's root runtime scope through background vibe worker spawns and pins exact scope identity so workers cannot create or dispose a competing runtime cache | | `packages/coding-agent/test/runtime-integration.test.ts` | when `AURA_RUNTIME_EMBEDDED_LIB` names a packaged shared library, derives the sibling packaged process binary so the same real-library command also exercises the existing process-adapter/JVM integration suite | | `packages/coding-agent/src/runtime/python.ts` | supplies CPython-compatible file-mode globals, argv, and sibling-import roots for Python path execution because the packaged runtime currently evaluates Python file input without defining `__file__` | -| `packages/coding-agent/src/runtime/transport/local.ts` | runs Python path requests through the shared file-mode bootstrap while retaining the process adapter's supervised workdir lifecycle | +| `packages/coding-agent/src/runtime/transport/local.ts` | runs Python path requests through the shared file-mode bootstrap; implements the compact runtime/JVM protocol, including Java/Kotlin source-path dependency analysis, guarded dependency-report output, and symlink-safe project write boundaries | | `packages/coding-agent/src/runtime/transport/embedded.ts` | runs Python path requests through a per-call temporary bootstrap file, cleaned after success, failure, timeout, or cancellation, so the embedded adapter preserves `__file__`, argv, and sibling imports without mutating user source | | `packages/coding-agent/test/runtime-embedded-integration.test.ts` | verifies real process and embedded adapters expose the original absolute `__file__` for Python path execution | -| `packages/metaharness/agent/omp_local.py` | Harbor's local-agent adapter stages generated gateway routing and benchmark config under the branded `~/.aura/agent` directory so Aura finds `models.yml` and does not fail before its first model request | +| `packages/metaharness/agent/omp_local.py` | Harbor's local-agent adapter stages generated gateway routing and benchmark config under both branded `~/.aura/agent` and vanilla `~/.omp/agent` directories, allowing current Aura and pinned upstream OMP binaries to share the host gateway without credentials entering task containers | | `packages/metaharness/src/runner.ts` | accepts `--path` for deterministic local Harbor capability tasks, installs source dependencies as the host uid/gid so the cache remains reusable, and strips host-only `PI_CONFIG_FILES` paths at both Harbor and task-container boundaries | | `packages/metaharness/src/runner.test.ts` | covers Aura's local Harbor `--path` launch contract alongside registry dataset launches | | `packages/metaharness/src/server.ts` | discovers externally launched CLI benchmark jobs during periodic sync and `/api/runs` reads, so completed runtime benchmark arms appear without restarting the dashboard | | `packages/metaharness/src/manager.test.ts` | covers periodic discovery of benchmark jobs launched outside the dashboard process | -| `packages/metaharness/src/runtime-benchmark-suite.ts` | defines and materializes Aura's deterministic 12-task Harbor runtime capability suite, copies the exact orchestrator Bun into every task image for fair TypeScript execution, generates Bun-based verifiers, and provides a model-free Docker smoke covering Node imports, TypeScript syntax, top-level await, and sorted BFS output | -| `packages/metaharness/src/runtime-benchmark-suite.test.ts` | verifies the runtime task catalog, generated Harbor task structure, task-owned executable Bun, Bun-only TypeScript verifier contract, and deterministic smoke-source feature coverage | -| `packages/metaharness/src/runtime-benchmark.ts` | orchestrates balanced matched agent arms with production-essential runtime tools plus task-specific discoverable tools, injects repository-packaged process and embedded runtime artifacts into source-mounted task containers, blocks model launches on the deterministic TypeScript verifier smoke, resumes interrupted frozen campaigns without rerunning completed Harbor jobs, reconstructs full arm elapsed time, aggregates paired effectiveness/latency/token/adoption evidence, and freezes the verifier Bun version/SHA-256 beside per-task tools and hashes | -| `packages/metaharness/src/runtime-benchmark.test.ts` | covers matched and historical arm construction, task-specific runtime tool exposure, packaged runtime path injection, resumable arm selection and persisted elapsed-time reconstruction, canonical per-trial measurements and tool-call counting, paired confidence/gate logic, manifest identity including verifier Bun metadata, report metrics, adapter sample ordering/output acceptance, percentile and speedup math, option precedence, and service cleanup | -| `packages/metaharness/package.json` | exposes the `bench:runtime` package script for the Aura runtime capability and microbenchmark suite | -| `packages/metaharness/README.md` | documents the one-command runtime benchmark, resumable frozen campaigns, its comparison contract, outputs, focused run modes, and opt-in 30-iteration process-vs-embedded decision run | -| `package.json` | exposes root `bench:runtime` as the one-command entrypoint for Aura's matched-arm runtime evaluation and `build:runtime-bundle` for the relocatable Aura + Elide archive builder; includes its contract test in `test:scripts` | -| `docs/settings.md`, `docs/environment-variables.md` | documents `runtime.enabled` / `runtime.adapter` / `runtime.autoDownload` / `runtime.path` / `runtime.version` / `runtime.embeddedPath`, including process-default adapter selection, the validated per-process `AURA_RUNTIME_ADAPTER` override, explicit-embedded no-fallback behavior, embedded-library resolution precedence, off-pin managed-version verification limits, and links to the runtime tool pages (the five core tools, the six `jvm_*` tools, `runtime_debug`, `serve`, and `project_advice`) | +| `packages/metaharness/src/runtime-benchmark-suite.ts` | defines and materializes Aura's deterministic 10-task Harbor runtime capability suite, copies the exact orchestrator Bun into every task image for fair TypeScript execution, generates Bun-based verifiers, and provides a model-free Docker smoke covering Node imports, TypeScript syntax, top-level await, and sorted BFS output | +| `packages/metaharness/src/runtime-benchmark-suite.test.ts` | verifies the compact runtime task catalog, generated Harbor task structure, task-owned executable Bun, Bun-only TypeScript verifier contract, and deterministic smoke-source feature coverage | +| `packages/metaharness/src/runtime-benchmark.ts` | orchestrates balanced matched agent arms with production-essential `run`/`check` plus task-specific discoverable tools, injects repository-packaged process and embedded runtime artifacts into source-mounted task containers, blocks model launches on the deterministic TypeScript verifier smoke, resumes interrupted frozen campaigns without rerunning completed Harbor jobs, reconstructs full arm elapsed time, aggregates paired effectiveness/latency/token/adoption evidence, and freezes the verifier Bun version/SHA-256 beside per-task tools and hashes | +| `packages/metaharness/src/runtime-benchmark.test.ts` | covers matched and historical arm construction, the compact task-specific runtime surface, packaged runtime path injection, resumable arm selection and persisted elapsed-time reconstruction, canonical per-trial measurements and tool-call counting, paired confidence/gate logic, manifest identity including verifier Bun metadata, report metrics, adapter sample ordering/output acceptance, percentile and speedup math, option precedence, and service cleanup | +| `packages/metaharness/package.json` | exposes `bench:runtime` for the broad runtime suite and `bench:inherent` for the focused two-task legacy-skills-vs-inherent-prompt comparison | +| `packages/metaharness/README.md` | documents the broad compact-surface runtime benchmark, pinned vanilla OMP historical controls, the focused inherent-capability comparison and gates, resumable frozen campaigns, report outputs, focused run modes, and opt-in process-vs-embedded decision run | +| `package.json` | exposes root `bench:runtime` and `bench:inherent` commands for the broad runtime suite and focused prompt comparison, plus `build:runtime-bundle` for the relocatable Aura + runtime archive; includes its contract test in `test:scripts` | +| `docs/settings.md`, `docs/environment-variables.md` | documents `runtime.enabled` / `runtime.adapter` / `runtime.autoDownload` / `runtime.path` / `runtime.version` / `runtime.embeddedPath`, including process-default adapter selection, the validated per-process `AURA_RUNTIME_ADAPTER` override, explicit-embedded no-fallback behavior, embedded-library resolution precedence, off-pin managed-version verification limits, and the compact ten-tool runtime surface | | `packages/coding-agent/src/cli/update-cli.ts` | distribution coordinates come from `pi-utils/distribution` (`DIST_*`) instead of upstream's can1357/npm constants; `getLatestRelease` is channel-aware (GitHub `releases/latest` on the fork repo, token-aware via `GITHUB_TOKEN`/`GH_TOKEN`, npm branch retained for a future publish) and exported with a `timeoutMs` param for the startup check; `resolveReleaseBinaryAsset` additionally returns the API asset URL and `updateViaBinaryAt` downloads through it with `Accept: application/octet-stream` + bearer auth when a token is present (the browser download URL 404s while the repo is private); the reinstall hint points at `DIST_INSTALL_URL` | | `packages/coding-agent/src/main.ts` (version check) | `checkForNewVersion` delegates to `getLatestRelease(5_000)` from update-cli so the startup notification and `aura update` consult the same channel; any check failure stays silent | | `packages/coding-agent/src/modes/utils/ui-helpers.ts` | the "Update Available" box advises `${APP_NAME} update` (was hardcoded `omp update`) | @@ -140,13 +140,6 @@ bytes/sha because the fork's `tool-views.generated.js` carries the runtime tool renderers — recompute those three constants whenever an upstream merge changes the template sources. -`packages/coding-agent/test/skills.test.ts` additionally carries one fork line: -`enableBundled: false` in its `DISABLE_ALL_BUILTIN_SKILLS` helper. That helper -means "every built-in skill source off", and the bundled runtime provider is a -new source; without it the tests asserting an exact custom-directory skill list -(and the "empty when all sources disabled" case) would see the five bundled -skills. Resolve an upstream conflict by keeping upstream's toggles and -re-appending this one. ## Fork-added files and directories (additive, no merge risk) @@ -161,15 +154,17 @@ re-appending this one. - `packages/coding-agent/src/runtime/` — runtime capability core; `index.ts` owns the selected-service cache key, atomic swap/retirement/disposal ordering, and `RuntimeSettingsValues` adapter/library mapping; `service.ts` owns the idempotent - endpoint-awaiting close boundary; `transport/selected.ts` implements the engine-aware - process/embedded/Bun routing and composed status matrix, including optimistic - embedded Java/Kotlin dispatch with process fallback on an unsupported-language - response; `transport/bun.ts` owns isolated JavaScript/TypeScript child execution + endpoint-awaiting close boundary and observes every protocol request exactly once + through `telemetry.ts`, which emits bounded outcomes and annotates the active tool + span without changing results or failures; `transport/selected.ts` implements the + engine-aware process/embedded/Bun routing and composed status matrix, including + optimistic embedded Java/Kotlin dispatch with process fallback on unsupported + language; `transport/bun.ts` owns isolated JavaScript/TypeScript child execution through the CLI worker-host re-entry path in `bun-run-entry.ts`; `transport/local.ts` - adds adjacent bundled Kotlin libraries to JVM run classpaths; `transport/embedded.ts` + owns process execution, adjacent bundled Kotlin classpaths, Java/Kotlin source-path + dependency analysis, and guarded project output writes; `transport/embedded.ts` owns validation, lazy open/reuse, serial execution, cancellation races, poisoning, - teardown, and postmortem fallback; - `resolve.ts` owns regular-file validation shared + teardown, and postmortem fallback; `resolve.ts` owns regular-file validation shared by binary and library resolution; and `src/runtime/embedded/` contains the exact-precedence shared-library resolver, handwritten embedded wire adapter, schema identity constants, the sole `bun:ffi` ABI owner, and the typed @@ -192,14 +187,19 @@ re-appending this one. services are closed without masking a primary probe failure - `packages/coding-agent/src/cli/version-identity.ts` — `--version` identity line (`/` + runtime protocol version) -- `packages/coding-agent/src/discovery/builtin-skills.ts`, - `src/discovery/builtin-skill-sources/*.md` — bundled runtime skills, materialized - into the agent dir under a `.bundled.json` manifest that is the sole prune authority - `scripts/build-relocatable-runtime-bundle.ts`, `scripts/build-relocatable-runtime-bundle.test.ts` — fork-owned Linux x64/glibc packager and behavioral contract tests for a relocatable standalone Aura binary, complete Elide distribution, embedded-library sidecars, runtime overlay, launcher, archive, checksum, and post-extraction verification; the launcher resolves relative, absolute, and chained installation symlinks before deriving its bundle root +- `packages/metaharness/src/inherent-capability-benchmark.ts`, + `src/inherent-capability-benchmark.test.ts`, and + `packages/coding-agent/scripts/runtime-telemetry-preflight.ts` — focused current + smoke and matched comparison of legacy skill-mediated prompts against inherent + capability policy on TypeScript execution and edit-plus-verification tasks, with + identical tools, per-attempt AB/BA pairing, treatment hashes, a deterministic + success/failure telemetry preflight, and gates for task success, runtime adoption, + first execution choice, promoted-skill loads, tool calls, and input tokens - `packages/coding-agent/src/tools/runtime-*.ts` (including `runtime-launch.ts`, which starts runtime launch descriptors through the upstream `hub` supervisor rather than keeping a process registry of its own), `src/prompts/tools/runtime-*.md`, @@ -236,22 +236,7 @@ re-appending this one. non-empty `.aura/` never masks a legacy single-file surface, the legacy executable surfaces (`hooks/`, `tools/`) DO load by design, legacy settings documents never become live, writes stay branded -- `docs/tools/{run,check,build,insights,profile,runtime_debug,serve}.md` — root docs pages required by the - `omp://` docs-coverage guard (`test/internal-urls/docs-tool-coverage.test.ts` asserts one - `docs/tools/.md` per entry in `BUILTIN_TOOL_NAMES`) -- `packages/coding-agent/src/discovery/builtin-skills.ts` + `src/discovery/builtin-skill-sources/` - (`runtime.md`, `insights.md`, `profiling.md`, `jvm.md`, `stateful-debugger.md`, `index.ts`) — - the bundled runtime skills, embedded via `with { type: "text" }` so they survive - `bun build --compile`, exactly as `discovery/builtin-rules/` does for rules. The provider - materializes them into `/builtin-skills//SKILL.md` before scanning: - unlike a `Rule` (whose body lives in memory and is served by `rule://`), a `Skill` is a - path, and `buildSkillPromptMessage` plus the `skill://` handler both re-read - `Skill.filePath` off disk. What it wrote is recorded in a `.bundled.json` manifest, - and that manifest is the only deletion authority — the same directory is a place a - user may park a skill of their own, and a bare user-authored `SKILL.md` is - shape-identical to one we wrote. Priority 3 — below managed auto-learn (5) and every - authored provider — so any same-named skill overrides a bundled one -- `packages/coding-agent/test/discovery/builtin-skills.test.ts` — fork-owned +- `docs/tools/{run,check,insights,profile,serve,jvm_disassemble,jvm_format,jvm_jar,jvm_deps}.md` — root docs pages required by the `omp://` docs-coverage guard (`test/internal-urls/docs-tool-coverage.test.ts` asserts one `docs/tools/.md` per entry in `BUILTIN_TOOL_NAMES`) - `docs/aura/`, `docs/superpowers/` ## Naming rule diff --git a/docs/settings.md b/docs/settings.md index 8e73a0553fc..affd7024a4d 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -498,7 +498,7 @@ Individual built-in tools are toggled by their own keys, e.g. `bash.enabled`, `l ### Runtime -The `run`, `check`, `build`, `insights`, and `profile` tools execute on a managed runtime binary, as do the six JVM tools (`jvm_run`, `jvm_disassemble`, `jvm_format`, `jvm_jar`, `jvm_deps`, `jvm_javadoc`), which compile and run Java/Kotlin on the embedded JVM, and the two long-running flows `runtime_debug` (a CDP/DAP debug endpoint) and `serve` (static files over HTTP), which are supervised as `hub` jobs rather than by the runtime layer. `project_advice` rides the same gate but is read-only: it asks the runtime for its build/run/test/install guidance for the current project and executes nothing. All fourteen are gated on `runtime.enabled`; when it is off, none of them register. +The `run`, `check`, `insights`, and `profile` tools execute on a managed runtime binary, as do four JVM specialists (`jvm_disassemble`, `jvm_format`, `jvm_jar`, and `jvm_deps`). Java and Kotlin execution is part of `run`. The `serve` long-running flow is supervised as a `hub` job rather than by the runtime layer. All nine tools are gated on `runtime.enabled`; when it is off, none register. ```yaml runtime: @@ -513,7 +513,7 @@ runtime: | Key | Type | Default | Notes | |---|---|---|---| -| `runtime.enabled` | boolean | `true` | Enable the innate `run`/`check`/`build`/`insights`/`profile`, `runtime_debug`/`serve`, and `jvm_*` tools executed on the managed runtime. Off disables the tools; `aura runtime status` still runs, reporting the disabled state with a nonzero exit. | +| `runtime.enabled` | boolean | `true` | Enable the innate `run`/`check`/`insights`/`profile`, `serve`, and four specialized `jvm_*` tools executed on the managed runtime. Off disables the tools; `aura runtime status` still runs, reporting the disabled state with a nonzero exit. | | `runtime.adapter` | `process` \| `embedded` \| `auto` | `process` | Select the runtime process, require the embedded runtime library, or choose the library automatically when it is available and compatible. Explicit `embedded` mode never falls back to the process adapter when the library is missing or incompatible. | | `runtime.autoDownload` | boolean | `true` | Fetch the pinned runtime into the config dir on first use when no binary is found. Ignored when `runtime.path` is set. | | `runtime.path` | string | `""` | Explicit runtime binary path; overrides discovery and disables auto-download. Also settable per-run with `--runtime `, which reports `source: flag` in `aura runtime status`. | @@ -524,36 +524,23 @@ The process adapter remains the default. `AURA_RUNTIME_ADAPTER=process|embedded| Embedded library resolution checks a nonblank `runtime.embeddedPath`, then a nonblank `AURA_RUNTIME_EMBEDDED_LIB`, then the pinned managed runtime version's `lib` directory, and finally the `lib` directory adjacent to an already-resolved real runtime binary. Candidates must be regular files. Resolution never scans `PATH` for shared libraries. -#### Bundled runtime skills +#### Inherent runtime guidance -Five skills ship with the agent and are discovered in every session: -`skill://runtime` (the `run`/`check`/`build` surface and when to prefer it over -`bash` or `eval`), `skill://insights`, `skill://profiling`, `skill://jvm`, and -`skill://stateful-debugger` (the `runtime_debug`/`serve` flows and their -`hub`-owned lifecycle). They carry the strategy the per-tool descriptions cannot -— when `cputracing` beats `cpusampling`, why a one-shot instrumented run emits no -`close` event, how the JVM main class is derived. +Runtime selection is part of Aura's system policy whenever runtime tools are +registered. The agent chooses direct execution (`run`), validation (`check`), +instrumentation (`insights`), profiling (`profile`), serving, and the four JVM +specialists from the available tool inventory. Project builds use the project's +declared build command rather than a separate runtime tool. Tool prompts remain +the argument and failure-shape reference. -| Key | Type | Default | Notes | -|---|---|---|---| -| `skills.enableBundled` | boolean | `true` | Discover the bundled runtime skills. Also retired automatically when `runtime.enabled` is off, since they document tools that are then unregistered. | - -They are materialized into `/agent/builtin-skills//SKILL.md` -(the skill machinery reads a skill's body back from its path) and rewritten from -the embedded copy whenever a file drifts, so edit them there and the change is -reverted on the next launch. Turning either toggle off removes that tree again. - -Only what the agent itself wrote is ever deleted: the directory carries a -`.bundled.json` manifest naming the skills it materialized, and a skill you place -in there yourself is left alone (it is discovered like any other). To override a -bundled skill, author one of the same name in any normal skills directory — the -bundled provider sits at the lowest skill priority, so yours wins. To drop one, -list its name in `skills.ignoredSkills`. +These capabilities are implicit: they are not materialized as skills, do not +appear in skill lists or `/skill:*` commands, and require no skill-load round +trip. `runtime.enabled` remains the single capability gate. `aura doctor` reports the resolved runtime alongside the rest of the install (identity and Bun version, native addon, registered tools, plugin health, terminal capabilities, memory backend); `aura doctor --json` emits the same report structurally, and `aura --check` collapses it to one line for clean-env CI probes. None of the three touch a model, the network, or provisioning — the runtime probe is read-only and never downloads a binary, whatever `runtime.autoDownload` says. All three exit nonzero only on a hard failure: a Bun older than the minimum, or a runtime that is enabled but unavailable. Optional misses (runtime disabled, no memory backend, missing plugin directory) are warnings and still exit 0. -See [run](./tools/run.md), [check](./tools/check.md), [build](./tools/build.md), [insights](./tools/insights.md), and [profile](./tools/profile.md) for per-tool behavior, and [jvm_run](./tools/jvm_run.md), [jvm_disassemble](./tools/jvm_disassemble.md), [jvm_format](./tools/jvm_format.md), [jvm_jar](./tools/jvm_jar.md), [jvm_deps](./tools/jvm_deps.md), [jvm_javadoc](./tools/jvm_javadoc.md) for the JVM suite, and [runtime_debug](./tools/runtime_debug.md) and [serve](./tools/serve.md) for the two supervised long-running flows (their handle is a `hub` job name, so `hub logs`/`hub stop` apply — there is no separate stop tool), and [project_advice](./tools/project_advice.md) for the read-only project-guidance tool. `jvm_jar` (create) and `jvm_javadoc` are the only runtime tools that write into your project: both require `output` to resolve *inside* the session cwd, both refuse an existing output unless `overwrite: true` is passed, and `jvm_javadoc`'s replace path additionally only accepts a previous docs output (empty, or carrying `index.html` plus one of javadoc's own scaffolding files). Containment is enforced after resolving symlinks, so a symlinked directory cannot redirect either write outside the cwd. +See [run](./tools/run.md), [check](./tools/check.md), [insights](./tools/insights.md), and [profile](./tools/profile.md) for core behavior; [jvm_disassemble](./tools/jvm_disassemble.md), [jvm_format](./tools/jvm_format.md), [jvm_jar](./tools/jvm_jar.md), and [jvm_deps](./tools/jvm_deps.md) for JVM specialists; and [serve](./tools/serve.md) for the supervised long-running flow. Its handle is a `hub` job name, so `hub logs` and `hub stop` apply; there is no separate stop tool. `jvm_jar` creation and `jvm_deps` with `output` are the only runtime flows that write into the project. Both require `output` to resolve inside the session cwd and refuse an existing output unless `overwrite: true` is passed. Containment is enforced after resolving symlinks, so a symlinked directory cannot redirect either write outside the cwd. ### Native computer use diff --git a/docs/superpowers/plans/2026-08-01-inherent-harness-capabilities.md b/docs/superpowers/plans/2026-08-01-inherent-harness-capabilities.md new file mode 100644 index 00000000000..4404925ac98 --- /dev/null +++ b/docs/superpowers/plans/2026-08-01-inherent-harness-capabilities.md @@ -0,0 +1,121 @@ +# Inherent Harness Capabilities Implementation Plan + +**Goal:** Make runtime execution and the universal engineering method inherent system behavior, remove their implicit skill/UI footprint, keep the common tool surface compact, and prove selection with a minimal benchmark. + +**Architecture:** Static Handlebars policy in the default system prompt is conditional on registered tools. Canonical Superpowers core skills are filtered only inside the canonical plugin provider. `run` and `check` stay essential; expensive analysis, profiling, debug, serving, and JVM operations stay discoverable. `RuntimeService.#call` provides one bounded telemetry boundary for the remaining runtime protocol. + +**Constraints:** + +- Work only in `.wt/inherent-capabilities`. +- Preserve same-named user/project skills and all domain skills. +- Never expose source, paths, arguments, output, or exception messages as telemetry dimensions. +- Keep the process adapter default and preserve existing runtime lifecycle behavior. +- Remove obsolete tools and protocol methods cleanly; no aliases or compatibility shims. +- Do not commit unless explicitly requested. + +## Task 1: Promote inherent policy and remove skill surfaces + +**Change** + +- Add compact engineering-method and runtime-selection policy to `src/prompts/system/system-prompt.md`. +- Remove the bundled runtime skill provider, setting, materialization, command, and provider-specific tests. +- Filter only the canonical Superpowers workflow names inside `src/discovery/claude-plugins.ts`. +- Preserve domain skills and same-named skills from every other provider. + +**Contract** + +- The prompt advertises only tools actually registered. +- Runtime and core workflow skills do not appear in the skill catalog or slash-command UI. +- No skill load is needed before an inherent runtime/tool action. + +## Task 2: Add runtime telemetry and status classification + +**Change** + +- Observe every remaining runtime protocol call once in `src/runtime/telemetry.ts`. +- Publish bounded `runtime.call.completed` events and OTLP counter/histogram data. +- Annotate the active tool span with method, action, language, outcome, duration, exit code, and killed state where available. +- Mark non-zero or killed execution results as tool errors through the shared classifier in `src/runtime/format.ts`. + +**Contract** + +- Success, process failure, timeout, cancellation, and protocol failure produce distinct bounded outcomes. +- Telemetry failures never alter runtime results. +- Generic tool status and runtime-specific status agree. + +## Task 3: Compact the runtime tool surface + +**Change** + +- Keep `run` and `check` essential. +- Keep `insights`, `profile`, `serve`, `jvm_disassemble`, `jvm_format`, `jvm_jar`, and `jvm_deps` discoverable. +- Route standalone Java/Kotlin through `run`; retain only four specialized JVM artifact/analysis tools. +- Remove `build`, `project_advice`, `runtime_debug` and its CDP/DAP launch mode, `jvm_javadoc`, and the `jvm_run` alias from tool registration, protocol, service, endpoint routing, telemetry, renderers, fixtures, docs, settings copy, and tests. +- Keep project artifact production on declared external build commands. +- Trim remaining tool prompts and schemas to decision-relevant guidance. + +**Contract** + +- Runtime-enabled default requests carry only the `run` and `check` schemas. +- Discoverable tools remain invocable through `xd://` or explicit `--tools`. +- The runtime service exposes no dead protocol methods. +- Provider payload tests cap the essential pair and the full discoverable family. + +## Task 4: Tighten JVM dependency workflows + +**Change** + +- Let `jvm_deps` accept either an existing `.java`/`.kt`/`.class`/`.jar`/class directory path or inline language plus source. +- Compile source in scratch space; analyze artifacts directly without redundant compilation. +- Allow an optional guarded cwd-relative report output, requiring `overwrite: true` to replace a file. +- Refuse directories and any real or symlinked destination outside the working directory. + +**Contract** + +- Source-path and inline-source calls report dependencies without modifying project files unless `output` is supplied. +- Artifact mode invokes `jdeps` directly. +- Output containment survives parent and leaf symlinks. + +## Task 5: Keep benchmarks minimal and task-specific + +**Change** + +- Keep the focused smoke to two tasks: `typescript-execution` and `jvm-dependencies`. +- Give each task six baseline file/shell tools, `run`/`check`, and only its required specialist. +- Retain the broader suite for regression coverage; the focused smoke remains the optimization loop. +- Hash the exact prompt, tool prompt, registration, and skill-filter inputs used by the inherent treatment. + +**Smoke command** + +```bash +bun run bench:inherent --prefix inherent-compact-smoke +``` + +**Smoke gates** + +- Both tasks pass. +- Every trial uses a runtime tool. +- `run` is selected before `bash`. +- No promoted runtime/core workflow skill is loaded. +- Telemetry preflight distinguishes success and intentional failure. + +**Comparison command** + +```bash +AURA_LEGACY_BINARY=/absolute/path/to/legacy-aura bun run bench:inherent \ + --attempts 3 --prefix inherent-compact-comparison +``` + +**Comparison gates** + +- All smoke gates pass. +- Median paired tool calls do not increase. +- Median paired input tokens do not increase. + +## Task 6: Verify and clean up + +1. Run focused prompt, registry, runtime, endpoint, renderer, JVM, benchmark, settings, doctor, and docs-coverage tests. +2. Run `bun check` in `packages/coding-agent` and `packages/metaharness`. +3. Run root `bun run check:ts`. +4. Run the inherent smoke against the source-mounted coding agent. +5. Update `docs/settings.md`, `docs/aura/FORK.md`, `packages/metaharness/README.md`, and the coding-agent Unreleased changelog. diff --git a/docs/superpowers/specs/2026-08-01-inherent-harness-capabilities-design.md b/docs/superpowers/specs/2026-08-01-inherent-harness-capabilities-design.md new file mode 100644 index 00000000000..b1beeca7e5d --- /dev/null +++ b/docs/superpowers/specs/2026-08-01-inherent-harness-capabilities-design.md @@ -0,0 +1,193 @@ +# Inherent Harness Capabilities Design + +## Goal + +Treat Aura-owned runtime execution and the universal Superpowers engineering method as inherent harness behavior, not optional skills. Remove their skill UI and skill-invocation telemetry footprint while preserving domain-specific authored skills. + +The change must also emit per-call runtime telemetry with reliable success/error classification, resolved language where applicable, and wall-clock duration through Aura telemetry and OpenTelemetry. + +## Current state + +Aura registers five bundled runtime skills (`runtime`, `insights`, `profiling`, `jvm`, and `stateful-debugger`). The provider materializes their Markdown into the agent directory, advertises them in the system prompt, and exposes them through `skill://` and `/skill:*`. The model must load a skill before using a capability already represented by a registered tool and its tool prompt. + +Canonical Superpowers core workflow skills are similarly advertised as optional domain knowledge even though design-before-editing, debugging, testing, delegation, verification, and completion are universal harness policy. This adds prompt catalog tokens, visible UI entries, skill-load round trips, and skill-use events without adding optional behavior. + +All agent tools already receive generic `execute_tool` spans. Those spans carry tool name, terminal tool status, and intrinsic span duration, but runtime results are not consistently classified as tool errors on non-zero exit. Runtime-specific language and call duration are not available as indexed attributes or metric dimensions. + +## Scope + +### In scope + +- Promote compact runtime selection policy into the main system prompt. +- Promote the universal Superpowers engineering method into the main system prompt. +- Remove Aura's bundled runtime skill provider, materialization, setting, commands, and tests. +- Exclude only canonical Superpowers core workflow skills from Claude marketplace discovery; preserve every domain-specific Superpowers skill and every same-named user/project skill. +- Add runtime call telemetry at the `RuntimeService` boundary. +- Classify non-zero, killed, cancelled, timed-out, and thrown runtime calls correctly in generic tool spans. +- Add the smallest repeatable behavior benchmark that exercises runtime selection and verification. +- Keep `run` and `check` essential; move high-cost runtime capabilities behind tool discovery. +- Collapse Java/Kotlin execution into `run`, retain four distinct JVM artifact/analysis tools, and remove redundant build, project-advice, and Javadoc runtime tools. + +### Out of scope + +- Changing runtime execution, adapter selection, or lifecycle behavior beyond deleting obsolete protocol methods. +- Hiding domain-specific skills. +- Adding a general prompt-fragment discovery framework. +- Replacing the compact broad runtime effectiveness suite. + +## Prompt architecture + +### Inherent capability position + +The default system prompt gains an `INHERENT CAPABILITIES` section immediately after role and engineering principles, before authored skills and rules. + +The runtime subsection renders only when at least one runtime tool is registered. It contains decision policy, not schemas: + +- direct JS/TS/Python/Java/Kotlin execution uses `run`; +- persistent incremental exploration uses `eval`; +- shell commands and installed CLIs use `bash`; +- validation without artifacts uses `check`; +- project artifact production uses declared external build commands; +- instrumentation, profiling, externally attachable debugging, serving, and four JVM artifact/analysis operations use their registered named tools; +- the runtime binary is never invoked through `bash`. + +Each statement is guarded by actual tool availability so the prompt never advertises an unavailable capability. Tool prompt Markdown remains the sole argument and failure-shape reference. + +The engineering-method subsection is unconditional and compact: + +- understand intent and choose a design before behavioral edits; +- reproduce bugs before changing code; +- test observable contracts before implementation where a regression boundary exists; +- delegate only independent work; +- verify the changed behavior before completion. + +These rules replace the need to load universal Superpowers workflow skills. They do not impose UI ceremony or require user approval for routine, already-specified work. + +### Authored skill contract + +The `` catalog is redefined as optional domain knowledge and workflows. It no longer contains Aura runtime capabilities or canonical Superpowers core workflow skills. + +Canonical Superpowers filtering occurs in the Claude marketplace provider while the provider still has plugin identity. The filter applies only when the plugin name is `superpowers` and only to the agreed core names: + +- `using-superpowers` +- `brainstorming` +- `writing-plans` +- `executing-plans` +- `test-driven-development` +- `systematic-debugging` +- `verification-before-completion` +- `dispatching-parallel-agents` +- `subagent-driven-development` +- `using-git-worktrees` +- `requesting-code-review` +- `receiving-code-review` +- `finishing-a-development-branch` + +A project or user skill with the same name remains loadable because it comes from another provider. Other Superpowers skills remain ordinary discoverable skills. + +## Runtime telemetry + +### Instrumentation boundary + +`RuntimeService.#call` is the single instrumentation boundary for every remaining runtime protocol request. It records exactly one completion event for success, protocol error, cancellation, timeout, or unexpected failure. This avoids per-tool drift and covers `run`, `check`, `insights`, `profile`, four JVM actions, debug/serve launch composition, and status probes. + +The event contains: + +- session ID when attributable; +- runtime method and JVM/spawn action where applicable; +- resolved language when applicable; +- outcome: `ok`, `error`, `timeout`, or `cancelled`; +- wall-clock `durationMs` measured around the complete request; +- exit code and killed flag for execution results; +- bounded error type/code, never source or program output. + +Language resolution uses the returned `RuntimeRunResult.language` for `run`; request language or the existing runtime target resolver for instrumentation/profile/JVM calls. Check, spawn, and status omit the dimension rather than emitting `unknown`. + +### OpenTelemetry spans + +Runtime calls made by agent tools execute inside the existing `execute_tool` active span. The service adds indexed attributes to that span: + +- `aura.runtime.method` +- `aura.runtime.action` when applicable +- `aura.runtime.language` when applicable +- `aura.runtime.outcome` +- `aura.runtime.duration_ms` +- `aura.runtime.exit_code` when applicable +- `aura.runtime.killed` when applicable + +Generic `gen_ai` tool attributes remain authoritative for tool name and call ID. Generic `aura.tool.status` and `error.type` remain authoritative for success/error classification. + +Every execution-style runtime tool returns `isError: true` for non-zero exit or killed execution. Thrown runtime errors continue through the agent loop's existing exception path. This makes generic spans, aggregate run summaries, and tool metrics agree with runtime-specific attributes. + +### Aura telemetry and metrics + +Add a typed `runtime.call.completed` event to the coding-agent telemetry bus. The OTLP sink records: + +- `aura.runtime.calls` counter by method, action, language, and outcome; +- `aura.runtime.duration` histogram in milliseconds with the same bounded dimensions. + +The event bus remains a no-op without subscribers. Telemetry failures never alter runtime results. No code, paths, arguments, stdout, stderr, or exception messages become metric attributes. + +## UI and settings cleanup + +Remove: + +- the `builtin-skills` capability provider and provider constant; +- bundled skill materialization and manifest files; +- `skills.enableBundled` from settings and settings types; +- bundled runtime entries from slash-command and skill UI surfaces; +- provider-specific tests that assert the five runtime skills exist. + +Update general skill tests to stop carrying the removed toggle. Remove obsolete files rather than leaving aliases or deprecated settings. + +The final runtime tool surface is deliberately asymmetric: + +- Essential on every runtime-enabled request: `run`, `check`. +- Discoverable through `xd://`: `insights`, `profile`, `serve`, `jvm_disassemble`, `jvm_format`, `jvm_jar`, `jvm_deps`. +- Removed entirely: `build`, `project_advice`, `runtime_debug` and its CDP/DAP launch protocol, `jvm_javadoc`, and the `jvm_run` alias. + +The remaining provider schemas must stay compact. `run` owns standalone language execution; specialized JVM tools own only bytecode, formatting, JAR, and dependency workflows. Project-declared builds remain external toolchain commands. + +## Minimal benchmark + +Reuse the existing deterministic metaharness fixtures, runner, measurements, and report primitives in a focused two-task orchestrator rather than creating a second benchmark framework. + +The default inherent-capability smoke runs: + +1. `typescript-execution` — validates direct runtime selection and successful execution. +2. `jvm-dependencies` — validates selection of `jvm_deps` over generic execution or shell commands. + +Run one attempt for a smoke gate and three alternating attempts when comparing revisions. Record existing pass, duration, token, tool-call, and runtime-adoption measurements plus the new runtime telemetry. + +Smoke gate: + +- both tasks pass; +- every completed task invokes a runtime tool; +- each task selects its expected capability (`run` or `jvm_deps`) before `bash`; +- success and intentional failure probes produce distinct telemetry outcomes; +- no runtime or core-workflow skill is loaded because those capabilities are absent from the skill catalog. + +The compact ten-task suite remains the broader regression run. The two-task mode is the optimization loop, not evidence of general model quality. + +## Verification + +1. System-prompt rendering tests cover runtime tools present and absent, and prove promoted names are absent from ``. +2. Claude plugin discovery tests prove only canonical core Superpowers skills are filtered. +3. Runtime service tests cover success, non-zero exit, timeout/cancellation, thrown protocol errors, language resolution, and duration. +4. Tool tests prove non-zero and killed results set `isError`. +5. Telemetry event and OTLP metric tests prove method, outcome, language, and duration propagation without payload leakage. +6. OpenTelemetry tests prove runtime attributes coexist with generic tool status/error attributes. +7. Run the two-task inherent-capability benchmark. +8. Run focused coding-agent and agent tests, then repository type checks. + +## Risks and controls + +- Prompt duplication can increase context. Control: remove bundled skill descriptions and keep inherent policy shorter than the removed catalog entries. +- Filtering by skill name alone could hide user behavior. Control: filter only inside the canonical Superpowers plugin provider branch. +- Runtime results can disagree with tool status. Control: one shared execution-result classifier used by all runtime tools. +- Metric cardinality can grow from free-form values. Control: method, action, language, and outcome are closed vocabularies; errors use protocol/status classes only. +- Instrumentation can alter failures. Control: telemetry publication and span annotation are non-throwing and never replace the runtime result. + +## Work isolation + +Implementation occurs in `.wt/inherent-capabilities` on branch `inherent-capabilities`, isolated from the ongoing Bazel RBE work in the primary checkout. diff --git a/docs/tools/build.md b/docs/tools/build.md deleted file mode 100644 index c9188ea8136..00000000000 --- a/docs/tools/build.md +++ /dev/null @@ -1,59 +0,0 @@ -# build - -> Assemble project artifacts on the managed runtime's build system. - -## Source -- Entry: `packages/coding-agent/src/tools/runtime-build.ts` -- Model-facing prompt: `packages/coding-agent/src/prompts/tools/runtime-build.md` -- Key collaborators: - - `packages/coding-agent/src/runtime/service.ts` — `RuntimeService.build()`. - - `packages/coding-agent/src/runtime/transport/local.ts` — spawns the runtime `build` subcommand with the caller's targets. - - `packages/coding-agent/src/runtime/format.ts` — `formatExecResult()`. - - `packages/coding-agent/src/tools/index.ts` — registers the built-in via `RuntimeBuildTool.createIf`. - -## Inputs - -| Field | Type | Required | Description | -|---|---|---:|---| -| `targets` | `string[]` | No | `':'`-prefixed build targets with interleaved per-target options, passed through verbatim (e.g. `[":deps", "--fresh", ":compile"]`). Omit for the default build. | -| `cwd` | `string` | No | Project directory. Defaults to the session cwd. | -| `timeoutMs` | `number` | No | Kill the build after this many milliseconds. | - -## Outputs -A single text block plus `details` carrying the raw `RuntimeExecResult`. - -- Text is built by `formatExecResult()`: stdout, then `--- stderr ---` plus stderr when non-empty, then a kill notice when applicable, then `(exit code N)` for a non-zero exit; `(no output, exit code N)` when both streams are empty. -- `details`: `{ exitCode, stdout, stderr, durationMs, killed }`. - -## Flow -1. `RuntimeBuildTool.createIf(session)` returns `null` unless `runtime.enabled` is truthy. -2. `execute()` requires `session.getRuntimeService?.()`; a missing service throws `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` -3. Params are forwarded with `cwd` defaulted to `session.cwd` as a `runtime/build` request. -4. `LocalRuntimeEndpoint` resolves (and may auto-provision) the runtime binary, then spawns ` build --no-color <...targets>`. `targets` are appended verbatim with no parsing or validation on the agent side. -5. `timeoutMs` and the caller abort signal both kill the process and set `killed`. - -## Modes / Variants -- **Default build**: `targets` omitted; the runtime picks the project's default target set. -- **Explicit targets**: `targets` selects specific targets and per-target flags, in order. - -## Side Effects -- Filesystem: writes build artifacts and caches into the project directory. -- Subprocesses: one runtime binary spawn per call. -- Network: dependency resolution may fetch; first use may download the managed runtime when `runtime.autoDownload` is on. -- Approval: `approval = "exec"`. - -## Limits & Caps -- Output is not truncated by the tool. Text past `tools.artifactSpillThreshold` (default 50KB) is saved in full as a session artifact by the central spill, and the inline content becomes a head/tail preview plus a `Read artifact:// for full output` reference — the same head/tail-plus-artifact convention `bash` presents. There is no per-stream character cap. -- No implicit timeout; builds are unbounded unless `timeoutMs` is supplied. -- Requires runtime >= 1.4. - -## Errors -- `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` when no runtime service is wired. -- `runtime-missing` with installation guidance when the binary cannot be found or provisioned. -- `cancelled` on abort. -- Build failures are reported as a non-zero `exitCode` with diagnostics in the text, not as a thrown error. -- Unknown or malformed targets are rejected by the runtime, not by the tool. - -## Notes -- `loadMode = "essential"`, so `build` stays top-level in the callable schema. -- `check` and `build` share one build driver: `check` passes an empty target list, `build` passes the caller's. diff --git a/docs/tools/check.md b/docs/tools/check.md index 89412e51567..6ffadf40591 100644 --- a/docs/tools/check.md +++ b/docs/tools/check.md @@ -32,11 +32,11 @@ A single text block plus `details` carrying the raw `RuntimeExecResult`. 5. `timeoutMs` and the caller abort signal both kill the process and set `killed`. ## Modes / Variants -- Single mode. `check` is the validation-only sibling of `build`: same underlying build driver, empty target list. Use `build` when artifacts are the goal. +- Single mode: resolve dependencies and compile source sets without producing artifacts. Use the project's declared build command when artifacts are required. ## Side Effects - Filesystem: the runtime's own dependency resolution may populate its caches and lockfiles in the project directory; no build artifacts are produced. -- Subprocesses: one runtime binary spawn per call — the runtime itself. Compilation never runs user code; use `run` (or `build` with a target that has a build script) when execution is intended. +- Subprocesses: one runtime binary spawn per call. Compilation never runs user code; use `run` when execution is intended. - Network: dependency resolution may fetch; first use may download the managed runtime when `runtime.autoDownload` is on. - Approval: `approval = "exec"`. @@ -53,4 +53,4 @@ A single text block plus `details` carrying the raw `RuntimeExecResult`. ## Notes - `loadMode = "essential"`, so `check` stays top-level in the callable schema. -- Use `check` as the fast "does the project still hold together" gate after edits; reach for `build` only when artifacts are the goal. +- Use `check` as the fast "does the project still hold together" gate after edits. This is not project-specific static analysis or a TypeScript typecheck. diff --git a/docs/tools/jvm_deps.md b/docs/tools/jvm_deps.md index ec11f34fe57..aa8aa0ed9e1 100644 --- a/docs/tools/jvm_deps.md +++ b/docs/tools/jvm_deps.md @@ -16,51 +16,58 @@ | Field | Type | Required | Description | |---|---|---:|---| -| `path` | `string` | No | Existing `.class`, `.jar`, or class directory, resolved against the session cwd. Selects artifact mode. | -| `language` | `"java" \| "kotlin"` | No | Source language (source mode). | -| `code` | `string` | No | Source to compile and analyze (source mode). | +| `path` | `string` | No | Existing Java/Kotlin source, `.class`, `.jar`, or class directory, resolved against the session cwd. Selects path mode. | +| `language` | `"java" \| "kotlin"` | No | Source language (inline-source mode). | +| `code` | `string` | No | Source to compile and analyze (inline-source mode). | | `mainClass` | `string` | No | Entrypoint class. Must be a class name (`/^[\w.$]+$/`). Defaults to the derived class. | +| `output` | `string` | No | Optional cwd-relative file to receive the dependency report. | +| `overwrite` | `boolean` | No | Required to replace an existing `output`. | | `timeoutMs` | `number` | No | Kills the compile or the analysis after this many milliseconds. | -Exactly one mode must be satisfiable: `path`, or `language` + `code`. A non-empty `path` wins when both are present; an empty `path` is treated as absent and takes source mode (it must never resolve to the working directory and analyze the whole project). +Exactly one input mode must be satisfiable: `path`, or `language` + `code`. A non-empty `path` wins when both are present; an empty `path` is treated as absent and takes inline-source mode. Java/Kotlin source paths compile in scratch space before analysis; `.class`, JAR, and class-directory paths are analyzed directly. ## Outputs A single text block plus `details` carrying the raw `RuntimeJvmResult`. -- Text is `formatExecResult(result)` — `jdeps` writes its report to stdout, and its complaints to stderr, so both are shown. -- `details`: `{ exitCode, stdout, stderr, durationMs, killed, action: "deps", phase, language?, className? }`. `language`/`className` are present only in source mode. +- Text is the dependency report on stdout plus any stderr or exit annotation. +- With `output`, a successful report is also written to the requested file and the text names that absolute path. +- `details`: `{ exitCode, stdout, stderr, durationMs, killed, action: "deps", phase, language?, className?, output? }`. `language`/`className` are present after source compilation; `output` is the absolute path written. -## Flow (artifact mode) +## Flow (path mode) 1. Params are sent as `runtime/jvm` with `action: "deps"` and `cwd` = the session cwd. -2. `path` is resolved against `cwd` and must exist. -3. ` jdeps -- ` runs in `cwd`. No workdir, nothing compiled, nothing written. +2. A `.java` or `.kt` path is read, compiled in a temp workdir, and analyzed as source. Other paths run ` jdeps -- ` directly in `cwd`. -## Flow (source mode) +## Flow (inline-source mode) 1. `language` and `code` are required; without them the call is `invalid-params`. 2. The endpoint opens one temp workdir, derives the class name, writes the source, and compiles (`javac -- --release 17 .java`, or `kotlinc -- Main.kt -cp . -d out`). A failed compile returns with `phase: "compile"`. 3. ` jdeps -- .class` (Java) or ` jdeps -- out` (Kotlin). 4. The workdir is removed in a `finally` block. `JAVA_HOME`/`JDK_HOME` are stripped from the spawn environment. +5. If `output` was requested and analysis succeeded, the report is written only after the temp workdir has closed. ## Modes / Variants -- **artifact**: read-only analysis of something already on disk. -- **source**: compile-then-analyze, useful for "what would this code pull in". +- **artifact path**: read-only analysis of an existing `.class`, JAR, or class directory. +- **source path**: compile-then-analyze an existing `.java` or `.kt` file. +- **inline source**: compile-then-analyze provided code. ## Side Effects -- Filesystem: source mode creates and removes one temp dir; artifact mode touches nothing. -- Subprocesses: two runtime spawns in source mode, one in artifact mode. +- Filesystem: source modes create and remove one temp dir. `output` writes one report file strictly inside the session cwd; without it, no project file changes. +- Subprocesses: two runtime spawns in source modes, one in artifact mode. - Network: first use may download the managed runtime when `runtime.autoDownload` is on. - Approval: `approval = "exec"`. ## Limits & Caps -- No `--multi-release`, `--module-path`, or summary/dot-output flags are exposed; the report is `jdeps`' default form. +- No `--multi-release`, `--module-path`, or summary/dot-output flags are exposed; the report is `jdeps`' default form. Use `output` when another project command needs the report as a file. - The report is not truncated by the tool; past `tools.artifactSpillThreshold` the central artifact spill preserves it in full and keeps a head/tail preview inline. - Requires runtime >= 1.4. ## Errors - `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` when no runtime service is wired. -- `jvm_deps requires either \`path\` (existing .class/.jar) or \`language\` + \`code\`.` — `invalid-params`. -- `No class file or jar found at .` — `invalid-params`, with `data.path`. -- `mainClass must be a class name (letters, digits, "_", "$", "."), got: ` — `invalid-params`. The derived class becomes a bare argv element for `java`/`javap`, so a value that could read as a flag is refused. +- `jvm_deps requires \`path\` (source, .class, .jar, or class directory) or \`language\` + \`code\`.` — `invalid-params`. +- `No JVM source, class, JAR, or class directory found at .` — `invalid-params`, with `data.path`. +- `Refusing to overwrite — pass overwrite: true to replace it.` — `invalid-params`, with `data.output`. +- `Refusing to write output to — output must be a path inside the working directory (), not the directory itself or one of its parents.` — `invalid-params`, with `data.output`. +- `Refusing to write the dependency report to — it is an existing directory.` — `invalid-params`, with `data.output`. +- `mainClass must be a class name (letters, digits, "_", "$", "."), got: ` — `invalid-params`. - `runtime-missing` with installation guidance; `cancelled` on abort. ## Notes diff --git a/docs/tools/jvm_jar.md b/docs/tools/jvm_jar.md index 8fb8c01075e..36b28f4d56f 100644 --- a/docs/tools/jvm_jar.md +++ b/docs/tools/jvm_jar.md @@ -36,7 +36,7 @@ A single text block plus `details` carrying the raw `RuntimeJvmResult`. ## Flow (create) 1. Params are sent as `runtime/jvm` with `action: "jar"`, `mode: "create"`, and `cwd` = the session cwd. 2. `language`, `code`, and `output` are all required; a missing one is `invalid-params`. -3. `output` is resolved against `cwd` and must land strictly inside it — `.`, `..`, an ancestor, or an absolute path elsewhere is refused, whatever `overwrite` says. The bound is checked lexically and again after resolving symlinks in `cwd` and in the destination's parent chain, so a symlinked directory cannot place the artifact outside the project. +3. `output` is resolved against `cwd` and must land strictly inside it — `.`, `..`, an ancestor, an absolute path elsewhere, or a destination symlink is refused, whatever `overwrite` says. The bound is checked lexically and again after resolving symlinks in `cwd` and in the destination's parent chain, so a symlinked directory cannot place the artifact outside the project. 4. **If it exists and `overwrite` is not `true` the call fails before anything is spawned** and the existing file is untouched. An existing *directory* at `output` is refused even with `overwrite: true` — a jar is a file. 5. The endpoint opens one temp workdir, derives the class name, writes the source, and compiles (`javac -- --release 17 .java`, or `kotlinc -- Main.kt -cp . -d out`). A failed compile returns with `phase: "compile"`. 6. Jar: ` jar -- --create --file aura-out.jar --main-class ` followed by the sorted `*.class` files in the workdir (Java) or `-C out .` (Kotlin). diff --git a/docs/tools/jvm_javadoc.md b/docs/tools/jvm_javadoc.md deleted file mode 100644 index ca5c92f35ac..00000000000 --- a/docs/tools/jvm_javadoc.md +++ /dev/null @@ -1,73 +0,0 @@ -# jvm_javadoc - -> Generate Javadoc HTML API docs from Java source into a project directory. - -## Source -- Entry: `packages/coding-agent/src/tools/jvm-javadoc.ts` -- Model-facing prompt: `packages/coding-agent/src/prompts/tools/jvm-javadoc.md` -- Key collaborators: - - `packages/coding-agent/src/runtime/service.ts` — `RuntimeService.jvm()`. - - `packages/coding-agent/src/runtime/transport/local.ts` — the `javadoc` flow of `runtime/jvm` plus `refuseExistingOutput()`. - - `packages/coding-agent/src/runtime/jvm.ts` — `deriveJvmMainClass()`. - - `packages/coding-agent/src/runtime/format.ts` — `formatExecResult()` on failure. - - `packages/coding-agent/src/tools/index.ts` — registers the built-in via `JvmJavadocTool.createIf`. - -## Inputs - -| Field | Type | Required | Description | -|---|---|---:|---| -| `code` | `string` | Yes | Java source to document. | -| `output` | `string` | No | Output directory, resolved against the session cwd and required to be **inside** it. Defaults to `javadoc-out`. | -| `overwrite` | `boolean` | No | Required to replace an existing `output`, and only ever replaces a **previous docs output** (see Flow). | -| `timeoutMs` | `number` | No | Kills the generator after this many milliseconds. | - -Java only — there is no Kotlin (Dokka) variant. `mainClass` is not a tool parameter; the documented class name is derived from `code`. - -## Outputs -A single text block plus `details` carrying the raw `RuntimeJvmResult`. - -- On success, three lines: - - `Generated API docs for ( entries).` - - `Top-level: ` - - `Tip: open /index.html to browse them.` -- On failure the text is `formatExecResult(result)`. -- `details`: `{ exitCode, stdout, stderr, durationMs, killed, action: "javadoc", phase: "javadoc", language: "java", className, output?, entryCount?, topLevel? }`. - -## Flow -1. `JvmJavadocTool.createIf(session)` returns `null` unless `runtime.enabled` is truthy. -2. Params are sent as `runtime/jvm` with `action: "javadoc"` and `cwd` = the session cwd. -3. `output` (default `javadoc-out`) is resolved against `cwd` and must land strictly inside it — `.`, `..`, an ancestor, or an absolute path elsewhere is refused, whatever `overwrite` says. The bound is checked twice: lexically, and again after resolving symlinks in `cwd` and in the destination's parent chain, so a symlinked directory inside the project cannot point the write (and the recursive remove) somewhere outside it. The final path component is deliberately left unresolved — a leaf symlink is unlinked, not followed. This bound is what keeps step 8's recursive remove from ever reaching the project root. -4. **If it exists and `overwrite` is not `true` the call fails before anything is spawned** and the existing directory is untouched. -5. With `overwrite: true` the destination must additionally *look like* a previous docs output — absent, an empty directory, or a directory containing `index.html` **and** one of javadoc's own scaffolding files (`element-list`, `help-doc.html`, `member-search-index.js`). `index.html` alone is not evidence: a static site has one too. A directory holding anything else, or a plain file, is refused before anything is spawned. -6. The endpoint opens one temp workdir, derives the class name from `code`, and writes `.java`. -7. Generate: ` javadoc -- -d apidocs .java`, with the workdir as cwd and `JAVA_HOME`/`JDK_HOME` stripped. A nonzero or killed run returns without writing anything. -8. On success the existing `output` is removed, its parent is created, and `apidocs` is copied recursively to `output`. -9. `entryCount` is the recursive entry count of `output`; `topLevel` is its first 12 top-level entries, sorted. -10. The workdir is removed in a `finally` block. - -## Modes / Variants -None — one flow, Java source in, an HTML tree out. - -## Side Effects -- Filesystem: writes `output` (and its parent directories) inside the session cwd — one of only two runtime tools that write outside a temp dir, and paths outside the cwd are refused. With `overwrite: true` the directory is **replaced wholesale** (recursive remove, then copy), not merged — which is why the replace path only accepts a previous docs output. The temp workdir is removed afterwards. -- Subprocesses: one runtime spawn. -- Network: first use may download the managed runtime when `runtime.autoDownload` is on. -- Approval: `approval = "exec"`. - -## Limits & Caps -- One compilation unit per call; no package trees, no `-link`/`-doclet` options. -- Javadoc warnings do not fail the run; a nonzero exit does. -- Requires runtime >= 1.4. - -## Errors -- `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` when no runtime service is wired. -- `jvm_javadoc requires \`code\` (Java source to document).` — `invalid-params`. -- `Refusing to overwrite — pass overwrite: true to replace it.` — `invalid-params`, with `data.output`. -- `Refusing to write output to — output must be a path inside the working directory (), not the directory itself or one of its parents.` — `invalid-params`, with `data.output`. -- `Refusing to replace — it does not look like a previous jvm_javadoc output (needs index.html plus one of element-list, help-doc.html, member-search-index.js). Choose a fresh directory, or the output directory of a previous run.` — `invalid-params`, with `data.output`. -- `runtime-missing` with installation guidance; `cancelled` on abort. - -## Notes -- `loadMode = "discoverable"`. -- A generated tree is a few dozen files even for one class, which is why the result reports a count and a sample rather than a listing. -- The two guards on `output` exist because this is the only runtime flow that *deletes* user files: `output: "."` with `overwrite: true` would otherwise remove the project, and `output: "link/docs"` through a symlink would remove a directory outside it. diff --git a/docs/tools/project_advice.md b/docs/tools/project_advice.md deleted file mode 100644 index 3f2fe407d88..00000000000 --- a/docs/tools/project_advice.md +++ /dev/null @@ -1,77 +0,0 @@ -# project_advice - -> Ask the runtime for its own build/run/test/install guidance for the current project. Read-only. - -## Source -- Entry: `packages/coding-agent/src/tools/runtime-advice.ts` -- Model-facing prompt: `packages/coding-agent/src/prompts/tools/runtime-advice.md` -- Key collaborators: - - `packages/coding-agent/src/runtime/service.ts` — `RuntimeService.advice()`. - - `packages/coding-agent/src/runtime/transport/local.ts` — `execAdvice()` spawns `project advice` in the real directory. - - `packages/coding-agent/src/runtime/format.ts` — `formatExecResult()`. - - `packages/coding-agent/src/tools/index.ts` — registers the built-in via `RuntimeAdviceTool.createIf`. - -## Inputs - -| Field | Type | Required | Description | -|---|---|---:|---| -| `cwd` | `string` | No | Project directory to inspect. Defaults to the session cwd. | -| `timeoutMs` | `number` | No | Kill the invocation after this many milliseconds. | - -There are no other inputs: the guidance is derived entirely from what the -directory contains, so there is nothing to configure but where to look. - -## Outputs -A single text block plus `details` carrying the raw `RuntimeExecResult`. - -- The guidance report is one blob of text: what the runtime's commands are, whether an - `elide.pkl` project configuration is present, the declared project name/version, and - the project's declared dependencies. -- `details`: `{ exitCode, stdout, stderr, durationMs, killed }`. - -## Flow -1. `RuntimeAdviceTool.createIf(session)` returns `null` unless `runtime.enabled` is truthy. -2. `execute()` requires `session.getRuntimeService?.()`; a missing service throws `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` -3. Params are forwarded with `cwd` defaulted to `session.cwd` as a `runtime/advice` request. -4. `LocalRuntimeEndpoint` resolves (and may auto-provision) the binary. -5. The endpoint spawns ` project advice --error-format=plain --no-color` with `cwd` as the working directory and `NO_COLOR=1` in the environment. -6. `formatExecResult()` renders stdout, then stderr, then an exit annotation. - -## Modes / Variants -None. One fixed invocation. - -## Side Effects -- Filesystem: reads only. No temp directory is created and nothing is written — unlike - every other runtime flow, this one has **no request workdir**, because the guidance - comes from *detecting* `elide.pkl` and package manifests and a temp directory would - always look like an empty project. -- Subprocesses: one runtime binary spawn per call, with a fixed argv. -- Network: first use may download the managed runtime when `runtime.autoDownload` is on. -- Approval: `approval = "read"` — the argv is fixed, the caller supplies no code, no - arguments and no output path, and the flow only inspects a directory the session can - already read. The other runtime tools are `"exec"` because they run code; this one - does not. - -## Limits & Caps -- The report is not truncated by the tool: past `tools.artifactSpillThreshold` (default - 50KB) the central spill saves the complete report as a session artifact and keeps a - head/tail preview inline with a `Read artifact:// for full output` reference. It - spills as a single blob, not per stream. -- No implicit timeout; unbounded unless `timeoutMs` is supplied. -- Requires runtime >= 1.4. - -## Errors -- `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` when no runtime service is wired. -- `runtime-missing` with installation guidance when no binary can be resolved or provisioned. -- `cancelled` on abort. -- A runtime that cannot produce advice is reported as it reported itself: its stderr and - nonzero exit code are surfaced verbatim rather than reinterpreted. Availability of this - guidance depends on the installed runtime build — it is known to have crashed on some - 1.4.0 nightlies, and works on the pinned 1.4.x line. - -## Notes -- `loadMode = "discoverable"`, so `project_advice` is reached through tool discovery - rather than the always-on schema, and it is not an essential tool. -- The runtime's own output is already plain: it is invoked with `--no-color` *and* - `NO_COLOR=1`, and the pinned runtime emits no ANSI escapes under either, so no - escape-stripping pass is applied on the way out. diff --git a/docs/tools/runtime_debug.md b/docs/tools/runtime_debug.md deleted file mode 100644 index 3a49cad6236..00000000000 --- a/docs/tools/runtime_debug.md +++ /dev/null @@ -1,82 +0,0 @@ -# runtime_debug - -> Start a CDP or DAP debug endpoint for a JS/TS/Python program on the managed runtime, supervised as a `hub` job. - -## Source -- Entry: `packages/coding-agent/src/tools/runtime-debug.ts` -- Model-facing prompt: `packages/coding-agent/src/prompts/tools/runtime-debug.md` -- Key collaborators: - - `packages/coding-agent/src/runtime/service.ts` — `RuntimeService.spawn()`. - - `packages/coding-agent/src/runtime/transport/local.ts` — composes the launch descriptor (`describeSpawn`); starts nothing. - - `packages/coding-agent/src/tools/runtime-launch.ts` — starts the descriptor through hub, scrapes the endpoint, formats the fallback. - - `packages/coding-agent/src/tools/hub/launch.ts` — `executeLaunch()`; owns the process lifecycle. - - `packages/coding-agent/src/tools/index.ts` — registers the built-in via `RuntimeDebugTool.createIf`. - -## Why the name is `runtime_debug` and not `debug` -`debug` is already a built-in: the interactive stepping debugger this agent drives -itself (`packages/coding-agent/src/tools/debug.ts`, `DebugTool` — breakpoints, -stepping, variable inspection, documented in `docs/tools/debug.md`). The two are -different tools, not two spellings of one: `debug` steps through code on the -agent's behalf, while `runtime_debug` publishes an endpoint for an *external* -debugger (Chrome DevTools, VS Code) to attach to. Registering this tool as `debug` -would have replaced the stepping debugger in `BUILTIN_TOOLS`, so it takes the -qualified name. The companion tool keeps the short name `serve`, which was free. - -## Inputs - -| Field | Type | Required | Description | -|---|---|---:|---| -| `path` | `string` | Yes | Program file to debug, resolved against `cwd`. There is no inline-code mode — see Limits. | -| `protocol` | `"cdp" \| "dap"` | No | Debug wire protocol. Default `cdp` (Chrome DevTools). | -| `language` | `"js" \| "ts" \| "python"` | No | Program language. Inferred from `path`'s extension otherwise (`.py` → python, `.js`/`.mjs`/`.cjs` → js, else ts). | -| `args` | `string[]` | No | Arguments passed to the program after `--`. | -| `cwd` | `string` | No | Working directory for the process and base for `path`. Defaults to the session cwd. | -| `timeoutMs` | `number` | No | Guest execution timeout, passed to the runtime as `--timeout ms`. | -| `waitSeconds` | `number` | No | How long hub waits for the endpoint banner. Default 15, clamped to 1–300. | - -## Outputs -A single text block plus `details` (`RuntimeJobDetails`). - -- With an endpoint: ` debugger listening at `, the attach hint for that protocol, a note that the program is suspended until a client attaches, and the job handle line naming the `hub` calls that read and stop it. -- Without one: the wait-window fallback (see Modes). -- `details`: `{ mode: "debug", jobName, endpoint?, state?, timedOut, readyMatch?, startupOutput, argv, cwd }`. `timedOut` is hub's own verdict on the readiness window, not a re-derivation from `endpoint`; `readyMatch` is the startup line hub matched. `jobName` is the hub job name — the handle for `hub {op:"logs"|"stop"|"restart", name}`. - -## Flow -1. `RuntimeDebugTool.createIf(session)` returns `null` unless `runtime.enabled` is truthy. -2. `execute()` requires `session.getRuntimeService?.()`; a missing service throws `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` -3. A `runtime/spawn` request returns a launch descriptor. The endpoint validates the parameters *first* (a bad mode or a nonexistent `path` must not trigger a runtime download just to be told `invalid-params`), then resolves (and may auto-provision) the binary, and composes ` run --debugger= --error-format=plain --no-color [--timeout ms] -l [-- ]`. No process is started and no temp directory is created. -4. The descriptor is started through `hub` with `op: "start"`, `pty: false`, `env: { NO_COLOR: "1" }` (an overlay on the broker's environment, not a snapshot of this session's), and `ready.log` set to the descriptor's endpoint pattern with `timeout: waitSeconds` — so hub's own readiness machinery does the waiting. -5. The endpoint is extracted from `readyMatch` — the line the broker's own readiness buffer matched, so it is found even when the banner has scrolled past the startup lines. A `hub` `logs` read (first 200 lines) supplies the startup output quoted in the fallbacks, and is the extraction fallback when there was no readiness pattern. -6. The job name is minted as `runtime-debug--<8 hex>`, so concurrent debuggers never collide and no name is reused. - -## Modes / Variants -- **`cdp`** — matches `ws://\S+` in the startup output and returns the inspector URL whole (1.4.2 prints `Debugger listening on ws://127.0.0.1:9229//inspect`). Attach with Chrome DevTools. -- **`dap`** — matches `listening on\s+/?(\S+)` and returns the captured `host:port`. The optional `/` is consumed rather than captured: 1.4.2 prints `[Graal DAP] Starting server and listening on /0.0.0.0:4711` (Java's socket-address formatting), and `/0.0.0.0:4711` is not an address a DAP client can attach to. Attach a DAP client such as VS Code. -- **Wait-window fallback** — when the banner never appears within `waitSeconds`, the result is *not* an error: the job is real and may still be starting. The text says `did not report an endpoint within the wait window (s); it may still be starting`, gives the job handle, and quotes the startup output so a changed startup banner is diagnosable rather than mysterious. Poll `hub {op:"logs", name}` before concluding anything failed. -- **Stale-rule fallback** — a distinct case: the banner *did* match but no endpoint could be extracted from it. The text says so, quotes the matched line, and names the likely cause (the scraping rule is out of date) rather than blaming a timeout that did not happen. `details.timedOut` is `false` here, which is how the two are told apart. -- **Failed launch** — when hub reports state `failed`, the result carries `isError: true`, hub's own failure summary, and the job name. - -## Side Effects -- Subprocesses: one long-running runtime process per call, owned by the hub broker — it outlives this tool call and must be stopped with `hub {op:"stop", name}`. -- Network: the debug endpoint listens on a local port. First use may download the managed runtime when `runtime.autoDownload` is on. -- Filesystem: none of its own; the debugged program may write anywhere it is permitted to. -- Approval: `approval = "exec"`. - -## Limits & Caps -- No inline-code mode. `runtime/run` can write inline source to a request-scoped temp directory because the request owns it; a supervised process outlives the request that started it, so the file would be deleted underneath it. Write the program with `write` first, then debug the path. -- `waitSeconds` is clamped to 1–300; the default 15 covers a cold runtime start. -- Startup output is read as the first 200 log lines. Everything after that is still in `hub logs`. -- The job is not `persist`ent or `detached`: it does not survive the broker exiting. -- Requires runtime >= 1.4. - -## Errors -- `The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).` when no runtime service is wired. -- `invalid-params`: `debug \`path\` (the program to debug) is required.` / `... does not exist: ` / `... is not a file: `; `timeoutMs must be a positive number of milliseconds.` -- `runtime-missing` with installation guidance; `cancelled` on abort. -- A failed launch is reported as an error *result* (with the job name), not a thrown error. - -## Notes -- `loadMode = "discoverable"`, so this tool is reached through tool discovery rather than the always-on schema, and it is not an essential tool. -- In a session without the `hub` tool (`--tools `, or a subagent with IRC disabled) the job still starts, but the guidance says so and offers no in-session route, because there is none: broker-supervised daemons are reachable only through `daemonClientForProject`, whose sole caller is the hub tool, and `/jobs` reads `getAsyncJobSnapshot` (async *tool* jobs — background bash, subagents) read-only, so it neither lists this job nor could stop it. What the guidance does say is true: the job is started without `persist` or `detached`, so it ends when the project's background broker exits; until then it must be stopped out of band, or re-run in a session that has hub. Availability is read from the registry's own active-tool set, not re-derived from hub's gate. -- There is no `stop_runtime_process` tool: `hub {op:"stop", name}` already takes a name, and routing through hub means `hub logs`, `hub wait`, `hub restart`, and `hub describe` all work on a debugger for free. (Not `/jobs` — that lists async *tool* jobs, not broker daemons.) -- When the binary was resolved from `PATH` (`source: "path"`), the result appends a `Note:` warning that it may be a wrapper script running the real binary as a child. The broker terminates the process *group* on stop, which normally covers that, but a surviving listener is worth checking for. The remedy the note gives is to point `runtime.path` / `AURA_RUNTIME_BIN` at a real binary, or to take the wrapper off `PATH` so the managed install is used — resolution prefers a `PATH` binary over auto-downloading, so a wrapper there always wins. The managed install carries no such note. diff --git a/docs/tools/serve.md b/docs/tools/serve.md index a5905a7182e..dc2553613b1 100644 --- a/docs/tools/serve.md +++ b/docs/tools/serve.md @@ -27,7 +27,7 @@ A single text block plus `details` (`RuntimeJobDetails`). - With a URL: `Serving at ` plus the job handle line naming the `hub` calls that read and stop it. - Without one: the wait-window fallback (see Modes). -- `details`: `{ mode: "serve", jobName, endpoint?, state?, timedOut, readyMatch?, startupOutput, argv, cwd }`. `timedOut` is hub's own verdict on the readiness window, not a re-derivation from `endpoint`; `readyMatch` is the startup line hub matched. `jobName` is the hub job name — the handle for `hub {op:"logs"|"stop"|"restart", name}`. +- `details`: `{ jobName, endpoint?, state?, timedOut, readyMatch?, startupOutput, argv, cwd }`. `timedOut` is hub's own verdict on the readiness window, not a re-derivation from `endpoint`; `readyMatch` is the startup line hub matched. `jobName` is the hub job name — the handle for `hub {op:"logs"|"stop"|"restart", name}`. ## Flow 1. `RuntimeServeTool.createIf(session)` returns `null` unless `runtime.enabled` is truthy. @@ -64,8 +64,6 @@ A single text block plus `details` (`RuntimeJobDetails`). ## Notes - `loadMode = "discoverable"`, so this tool is reached through tool discovery rather than the always-on schema, and it is not an essential tool. -- The short name is kept (unlike its companion `runtime_debug`, which is qualified because `debug` is the built-in stepping debugger) — it matches the other runtime tools `run`, `check`, `build`. - There is no `stop_runtime_process` tool: `hub {op:"stop", name}` already takes a name, and routing through hub means `hub logs`, `hub wait`, `hub restart`, and `hub describe` all work on a static server for free. (Not `/jobs` — that lists async *tool* jobs, not broker daemons.) - When the binary was resolved from `PATH` (`source: "path"`), the result appends a `Note:` warning that it may be a wrapper script running the real binary as a child, so a stop could leave a listener behind. The broker terminates the process *group*, which normally covers that. The remedy the note gives is to point `runtime.path` / `AURA_RUNTIME_BIN` at a real binary, or to take the wrapper off `PATH` so the managed install is used — resolution prefers a `PATH` binary over auto-downloading, so a wrapper there always wins. The managed install carries no such note. - In a session without the `hub` tool (`--tools serve`, or a subagent with IRC disabled) the server still starts, but the guidance says so and offers no in-session route, because there is none: broker-supervised daemons are reachable only through `daemonClientForProject`, whose sole caller is the hub tool, and `/jobs` reads `getAsyncJobSnapshot` (async *tool* jobs — background bash, subagents) read-only, so it neither lists this server nor could stop it. What the guidance does say is true: the job is started without `persist` or `detached`, so it ends when the project's background broker exits — until then it holds the port and must be stopped out of band, or re-run in a session that has hub. Availability is read from the registry's own active-tool set, not re-derived from hub's gate. -- Pairs with `jvm_javadoc`: generate docs, then serve the output directory to browse them. diff --git a/package.json b/package.json index dde8449fbdc..a2c27f72c3f 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,7 @@ "collab:web:build": "bun --cwd=packages/collab-web run build", "meta": "bun --cwd=packages/metaharness run dev", "bench:runtime": "bun --cwd=packages/metaharness run bench:runtime", + "bench:inherent": "bun --cwd=packages/metaharness run bench:inherent", "claude:trace": "bun scripts/claude-trace.ts", "build": "bun run --workspaces --if-present build", "build:native": "bun --cwd=packages/natives run build", diff --git a/packages/agent/test/otel.test.ts b/packages/agent/test/otel.test.ts index e59fdf8d66d..2513de662d5 100644 --- a/packages/agent/test/otel.test.ts +++ b/packages/agent/test/otel.test.ts @@ -286,6 +286,40 @@ describe("agent-loop OTEL instrumentation", () => { expect(userInner?.parentSpanContext?.spanId).toBe(tool?.spanContext().spanId); }); + it("records ERROR status when a tool returns a non-throwing failure result", async () => { + const mock = createMockModel({ + ...MOCK_IDENT, + responses: [ + { content: [{ type: "toolCall", id: "tc-1", name: "reject", arguments: { value: "x" } }] }, + { content: ["done"] }, + ], + }); + const config: AgentLoopConfig = { + model: mock.model, + convertToLlm: identityConverter, + telemetry: {}, + }; + const rejectSchema = z.object({ value: z.string() }); + const rejectTool: AgentTool = { + name: "reject", + label: "Reject", + description: "returns a handled failure", + parameters: rejectSchema, + execute: async () => ({ + content: [{ type: "text", text: "request rejected" }], + details: {}, + isError: true, + }), + }; + const ctx: AgentContext = { systemPrompt: [], messages: [], tools: [rejectTool] }; + await runAndDrain(agentLoop([createUserMessage("hi")], ctx, config, undefined, mock.stream)); + + const tool = findSpan(exporter.getFinishedSpans(), "execute_tool reject"); + expect(tool?.status.code).toBe(SpanStatusCode.ERROR); + expect(tool?.attributes[GenAIAttr.ErrorType]).toBe("tool_error"); + expect(tool?.events.some(event => event.name === "exception")).toBe(false); + }); + it("records ERROR status + exception when a tool throws", async () => { const mock = createMockModel({ ...MOCK_IDENT, diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 0dc86d6e0d9..228585c5915 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed provider source typechecking for DOM-enabled workspace consumers while preserving Cowork response streaming and Codex WebSocket/zstd request behavior. + ## [17.2.2] - 2026-07-31 ### Added diff --git a/packages/ai/src/providers/cowork-fetch.ts b/packages/ai/src/providers/cowork-fetch.ts index 896108e6e44..3da78214fa0 100644 --- a/packages/ai/src/providers/cowork-fetch.ts +++ b/packages/ai/src/providers/cowork-fetch.ts @@ -118,7 +118,11 @@ function createResponse(message: IncomingMessage, method: string): Response { const status = message.statusCode; if (status === undefined) throw new Error("Cowork transport received a response without an HTTP status."); const hasBody = method !== "HEAD" && status !== 204 && status !== 304; - const body = hasBody ? stream.Readable.toWeb(decodedResponseStream(message)) : null; + // Node and DOM expose structurally distinct stream declarations, but toWeb() + // returns the WHATWG stream accepted by Response at runtime. + const body = hasBody + ? (stream.Readable.toWeb(decodedResponseStream(message)) as unknown as ReadableStream) + : null; return new Response(body, { status, statusText: message.statusMessage, diff --git a/packages/ai/src/providers/openai-codex-responses.ts b/packages/ai/src/providers/openai-codex-responses.ts index 20751703c29..2e2fc54f215 100644 --- a/packages/ai/src/providers/openai-codex-responses.ts +++ b/packages/ai/src/providers/openai-codex-responses.ts @@ -1158,6 +1158,15 @@ function notifyCodexWebSocketMalformed( notifyRawSseEvent(observer, { event: "parse_error", data: text, raw }); } +function decodeCodexWebSocketFrame(data: unknown): string { + if (typeof data === "string") return data; + if (data instanceof ArrayBuffer) return Buffer.from(data).toString("utf-8"); + if (ArrayBuffer.isView(data)) { + return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("utf-8"); + } + throw new TypeError("unsupported WebSocket message frame"); +} + /** @internal Exported for tests. */ export function normalizeCodexToolChoice( choice: ToolChoice | undefined, @@ -3474,13 +3483,16 @@ class CodexWebSocketConnection { this.#push(null); }; socket.onmessage = event => { + // Bun's DOM-aware types currently infer this callback parameter as + // the MessageEvent constructor rather than an event instance. + const data = asRecord(event)?.data; // Stamp inbound activity before parsing so even malformed frames refresh // the liveness clock — what matters for reuse health is that the upstream // is still talking to us, not that every frame is well-formed. this.#lastInboundAt = Date.now(); - this.#writeDebugWebSocketFrame(event.data); + this.#writeDebugWebSocketFrame(data); try { - const text = typeof event.data === "string" ? event.data : Buffer.from(event.data).toString("utf-8"); + const text = decodeCodexWebSocketFrame(data); if (!text) return; const parsed = JSON.parse(text) as Record; if (parsed.type === "error" && typeof parsed.error === "object" && parsed.error) { @@ -3495,7 +3507,7 @@ class CodexWebSocketConnection { notifyCodexWebSocketInbound(this.#streamObserver, parsed, text); this.#push(parsed); } catch (error) { - notifyCodexWebSocketMalformed(this.#streamObserver, event.data, error); + notifyCodexWebSocketMalformed(this.#streamObserver, data, error); this.#push(new CodexWebSocketTransportError(`${String(error)}`)); } }; @@ -3945,10 +3957,12 @@ async function getOrCreateCodexWebSocketConnection( * compression is disabled or fails, in which case the caller sends the * plain JSON string without a `content-encoding` header. */ -function compressCodexRequestBody(bodyJson: string, baseUrl: string): Uint8Array | undefined { +function compressCodexRequestBody(bodyJson: string, baseUrl: string): Uint8Array | undefined { if (!isOfficialCodexApiUrl(baseUrl) || !$flag("PI_CODEX_ZSTD", true)) return undefined; try { - return Bun.zstdCompressSync(bodyJson, { level: 3 }); + const compressed = Bun.zstdCompressSync(bodyJson, { level: 3 }); + if (!(compressed.buffer instanceof ArrayBuffer)) return Uint8Array.from(compressed); + return new Uint8Array(compressed.buffer, compressed.byteOffset, compressed.byteLength); } catch (error) { CODEX_DEBUG && logger.debug("[codex] codex request body compression failed", { @@ -4014,7 +4028,7 @@ async function openCodexSseEventStream( sentModelsEtagHeader: headers.has(X_MODELS_ETAG_HEADER), }); - const send = (requestBody: string | Uint8Array): Promise => + const send = (requestBody: string | Uint8Array): Promise => fetchWithRetry(url, { method: "POST", headers, diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 02c76b89746..0a7e3c0f4ec 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes - Removed the public `jvm_run` tool; execute Java and Kotlin through the unified `run` tool with `language`, optional `mainClass`, and the existing args/stdin/cwd controls. +- Removed the `runtime_debug` tool and CDP/DAP launch protocol; use Aura's interactive `debug` tool until runtime-backed debugging is integrated there. ### Added @@ -18,6 +19,7 @@ - Clarified that managed `check` validates supported runtime project builds but does not replace project-declared TypeScript static typechecking. - Changed JavaScript and TypeScript `run` calls to use an isolated Bun child by default while keeping the embedded engine selectable; Python, Java, and Kotlin use the embedded engine. - Reduced persistent system and runtime tool prompt text while preserving tool-selection and safety contracts. +- Promoted managed runtime selection and core Superpowers workflows into inherent system policy; removed their implicit skill/UI surface; kept only `run` and `check` essential; moved insights, profiling, serving, and four JVM operations behind discovery; removed redundant runtime build/advice/Javadoc tools; and added bounded per-call telemetry plus a two-task adoption/prompt-efficiency benchmark. ### Fixed diff --git a/packages/coding-agent/scripts/runtime-telemetry-preflight.ts b/packages/coding-agent/scripts/runtime-telemetry-preflight.ts new file mode 100755 index 00000000000..e9c2ad9e0d7 --- /dev/null +++ b/packages/coding-agent/scripts/runtime-telemetry-preflight.ts @@ -0,0 +1,58 @@ +#!/usr/bin/env bun +import { okResponse } from "../src/runtime/protocol"; +import { RuntimeService } from "../src/runtime/service"; +import { type RuntimeCallCompletedTelemetry, subscribeTelemetry } from "../src/telemetry/events"; + +async function main(): Promise { + const events: RuntimeCallCompletedTelemetry[] = []; + const unsubscribe = subscribeTelemetry(event => { + if (event.type === "runtime.call.completed") events.push(event); + }); + let callIndex = 0; + const service = new RuntimeService({ + async request(req) { + callIndex += 1; + return okResponse(req.id, { + exitCode: callIndex === 1 ? 0 : 2, + stdout: "", + stderr: callIndex === 1 ? "" : "intentional failure", + durationMs: 1, + killed: false, + }); + }, + }); + try { + await service.run({ code: "print('ok')", language: "python" }, undefined, "benchmark-preflight"); + await service.run({ code: "raise SystemExit(2)", language: "python" }, undefined, "benchmark-preflight"); + } finally { + unsubscribe(); + } + const [success, failure] = events; + if ( + events.length !== 2 || + !success || + success.sessionId !== "benchmark-preflight" || + success.language !== "python" || + success.outcome !== "ok" || + success.exitCode !== 0 || + success.durationMs < 0 + ) { + throw new Error("runtime telemetry success preflight failed"); + } + if ( + failure?.sessionId !== "benchmark-preflight" || + failure.language !== "python" || + failure.outcome !== "error" || + failure.exitCode !== 2 || + failure.errorType !== "non_zero_exit" || + failure.durationMs < 0 + ) { + throw new Error("runtime telemetry failure preflight failed"); + } + process.stdout.write(`${JSON.stringify({ success, failure })}\n`); +} + +main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); +}); diff --git a/packages/coding-agent/src/capability/skill.ts b/packages/coding-agent/src/capability/skill.ts index 3f50016d96d..d850eec10db 100644 --- a/packages/coding-agent/src/capability/skill.ts +++ b/packages/coding-agent/src/capability/skill.ts @@ -6,15 +6,6 @@ import { defineCapability } from "."; import type { SourceMeta } from "./types"; -/** - * Provider id for the runtime skills bundled with the agent. - * - * Lives here rather than in `discovery/builtin-skills` so consumers can test - * for a bundled skill without importing (and thereby registering) the provider - * — mirroring `BUILTIN_DEFAULTS_PROVIDER_ID` on the rule capability. - */ -export const BUILTIN_SKILLS_PROVIDER_ID = "builtin-skills"; - /** * Parsed frontmatter from a skill file. */ diff --git a/packages/coding-agent/src/cli/doctor-cli.ts b/packages/coding-agent/src/cli/doctor-cli.ts index 50e0ab58680..4c536d70e82 100644 --- a/packages/coding-agent/src/cli/doctor-cli.ts +++ b/packages/coding-agent/src/cli/doctor-cli.ts @@ -283,7 +283,7 @@ function runtimeSection(input: DoctorRuntimeInput): DoctorEntry[] { label: "state", status: "warn", detail: - "disabled (runtime.enabled = false) — run/check/build/insights/profile and the five specialized jvm_* tools do not register", + "disabled (runtime.enabled = false) — run/check/insights/profile and the four specialized jvm_* tools do not register", }, protocol, ]; @@ -649,24 +649,12 @@ export const SESSION_GATED_TOOL_NAMES: readonly string[] = ["ask", "checkpoint", * drift test can enumerate it and compare against the real registry. */ const SETTINGS_GATED_TOOLS: Record string | undefined> = { - // Runtime execution/check/build/analysis, the two launch tools, the five - // specialized Jvm*Tool classes, and RuntimeAdviceTool gate on `runtime.enabled`. + // Runtime execution/check/analysis, the launch tool, and the four specialized + // Jvm*Tool classes gate on `runtime.enabled`. ...Object.fromEntries( - [ - "run", - "check", - "build", - "insights", - "profile", - "runtime_debug", - "serve", - "jvm_disassemble", - "jvm_format", - "jvm_jar", - "jvm_deps", - "jvm_javadoc", - "project_advice", - ].map(name => [name, (s: ToolGateSettings) => (s.runtimeEnabled ? undefined : "runtime.enabled = false")]), + ["run", "check", "insights", "profile", "serve", "jvm_disassemble", "jvm_format", "jvm_jar", "jvm_deps"].map( + name => [name, (s: ToolGateSettings) => (s.runtimeEnabled ? undefined : "runtime.enabled = false")], + ), ), // DebugTool.createIf debug: s => (s.debugEnabled ? undefined : "debug.enabled = false"), diff --git a/packages/coding-agent/src/cli/gallery-fixtures/runtime.ts b/packages/coding-agent/src/cli/gallery-fixtures/runtime.ts index d49e8c3f974..21d7d863328 100644 --- a/packages/coding-agent/src/cli/gallery-fixtures/runtime.ts +++ b/packages/coding-agent/src/cli/gallery-fixtures/runtime.ts @@ -1,10 +1,9 @@ /** - * Gallery fixtures for the runtime tool family (`run`, `check`, `build`, - * `insights`, `profile`, `project_advice`, the six `jvm_*` flows, and the - * hub-backed `runtime_debug` / `serve`). + * Gallery fixtures for `run`, `check`, `insights`, `profile`, the four + * specialized `jvm_*` flows, and the hub-backed `serve` tool. * * The success/error envelopes are built by the two helpers below rather than - * spelled out fourteen times: every one of these tools returns the same + * repeated for every tool: the execution tools return the same * `RuntimeExecResult` shape (or that shape plus a JVM flow's extras), so the * only per-tool data worth hand-writing is the args and the output text — which * is exactly what the gallery is there to show. @@ -21,11 +20,11 @@ function execResult(text: string, over: Record = {}): GalleryRe }; } -/** A settled hub job, as `runtime_debug` / `serve` attach it to `details`. */ +/** A settled hub job, as `serve` attaches it to `details`. */ function jobResult(text: string, over: Record): GalleryResult { return { content: [{ type: "text", text }], - details: { mode: "serve", timedOut: false, startupOutput: text, argv: [], cwd: "/repo", ...over }, + details: { timedOut: false, startupOutput: text, argv: [], cwd: "/repo", ...over }, }; } @@ -45,14 +44,6 @@ export const runtimeFixtures: Record = { errorResult: execResult("src/api/routes.ts:88:12 — cannot find symbol `Router`\n(exit code 2)", { exitCode: 2 }), }, - build: { - label: "Build", - streamingArgs: { targets: [":jvm"] }, - args: { targets: [":jvm", ":native"] }, - result: execResult("Built :jvm in 4.2s\nBuilt :native in 31.8s", { durationMs: 36_000 }), - errorResult: execResult("Target :native failed: linker exited with 1\n(exit code 1)", { exitCode: 1 }), - }, - insights: { label: "Insights", args: { path: "src/worker.ts", insightPath: "hooks/alloc-trace.js" }, @@ -69,20 +60,6 @@ export const runtimeFixtures: Record = { errorResult: execResult("--- stderr ---\nprofiler could not attach\n(exit code 1)", { exitCode: 1 }), }, - project_advice: { - label: "Project Advice", - args: {}, - result: execResult( - [ - "This project builds with the project manifest in this directory.", - " build: build :jvm", - " test: test --coverage", - " serve: serve public/", - ].join("\n"), - ), - errorResult: execResult("no project manifest found in this directory\n(exit code 1)", { exitCode: 1 }), - }, - jvm_disassemble: { label: "JVM Disassemble", args: { language: "java", code: "public class Main { static int add(int a, int b) { return a + b; } }" }, @@ -149,46 +126,6 @@ export const runtimeFixtures: Record = { }), }, - jvm_javadoc: { - label: "JVM Javadoc", - args: { code: "/** Entry point. */\npublic class Main {}", output: "docs/api" }, - result: execResult("Generated 12 entries into docs/api", { - action: "javadoc", - phase: "javadoc", - output: "/repo/docs/api", - entryCount: 12, - topLevel: ["index.html", "Main.html"], - }), - errorResult: execResult("docs/api already exists; pass overwrite: true to replace it\n(exit code 1)", { - exitCode: 1, - action: "javadoc", - phase: "javadoc", - }), - }, - - runtime_debug: { - label: "Runtime Debug", - args: { path: "src/worker.ts", protocol: "cdp" }, - result: jobResult( - [ - "CDP debugger listening at ws://127.0.0.1:4242/session/1", - "Open it in Chrome DevTools. The program is suspended until a client attaches.", - "Job: runtime-debug-cdp-1a2b3c4d (use hub logs / hub stop).", - ].join("\n"), - { - mode: "debug", - jobName: "runtime-debug-cdp-1a2b3c4d", - endpoint: "ws://127.0.0.1:4242/session/1", - state: "running", - }, - ), - errorResult: jobResult("The CDP debugger printed no endpoint within 15s.", { - mode: "debug", - jobName: "runtime-debug-cdp-7e6f5a4b", - timedOut: true, - }), - }, - serve: { label: "Serve", args: { directory: "public", port: 8080 }, diff --git a/packages/coding-agent/src/commands/runtime.ts b/packages/coding-agent/src/commands/runtime.ts index e2eb1520718..c6ac157f408 100644 --- a/packages/coding-agent/src/commands/runtime.ts +++ b/packages/coding-agent/src/commands/runtime.ts @@ -1,5 +1,5 @@ /** - * Inspect the managed runtime powering the innate run/check/build/insights/profile tools. + * Inspect the managed runtime powering the innate run/check/insights/profile tools. */ import { getProjectDir } from "@oh-my-pi/pi-utils"; import { Args, Command, Flags } from "@oh-my-pi/pi-utils/cli"; diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 8d8855a49e3..fc68dcdbf71 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -477,7 +477,7 @@ export const SETTINGS_SCHEMA = { tab: "tools", group: "Runtime", label: "Enable runtime capabilities", - description: "Innate run/check/build/insights/profile tools executed on the managed runtime.", + description: "Innate run/check/insights/profile and JVM tools executed on the managed runtime.", }, }, "runtime.adapter": { @@ -4848,17 +4848,6 @@ export const SETTINGS_SCHEMA = { }, }, - "skills.enableBundled": { - type: "boolean", - default: true, - ui: { - tab: "tools", - group: "Runtime", - label: "Bundled runtime skills", - description: "Ship the built-in runtime skills (run/check/build, insights, profiling, JVM, debug & serve)", - }, - }, - "skills.enableCodexUser": { type: "boolean", default: true }, "skills.enableClaudeUser": { type: "boolean", default: true }, @@ -5883,7 +5872,6 @@ export interface BranchSummarySettings { export interface SkillsSettings { enabled?: boolean; enableSkillCommands?: boolean; - enableBundled?: boolean; enableCodexUser?: boolean; enableClaudeUser?: boolean; enableClaudeProject?: boolean; diff --git a/packages/coding-agent/src/discovery/builtin-skill-sources/index.ts b/packages/coding-agent/src/discovery/builtin-skill-sources/index.ts deleted file mode 100644 index 353e4d8f48d..00000000000 --- a/packages/coding-agent/src/discovery/builtin-skill-sources/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Bundled runtime skills shipped with the agent. - * - * Each `SKILL.md` body is embedded via `with { type: "text" }` so it survives - * `bun build --compile` (the compiled binary ships no loose skill files; only - * the embedded text). Mirrors `./builtin-rules` for the rule capability. - * - * These carry the strategy the per-tool descriptions cannot: when `cputracing` - * beats `cpusampling`, what an insight script's hooks can hook, why one-shot - * runs emit no `close`, how the JVM main class is derived, and that a - * `runtime_debug` / `serve` job's lifecycle belongs to `hub`. - * - * Registered by the lowest-priority `builtin-skills` provider, so a - * user/project skill of the same name overrides the bundled copy. - */ -import insights from "./insights.md" with { type: "text" }; -import jvm from "./jvm.md" with { type: "text" }; -import profiling from "./profiling.md" with { type: "text" }; -import runtime from "./runtime.md" with { type: "text" }; -import statefulDebugger from "./stateful-debugger.md" with { type: "text" }; - -/** A bundled skill's directory name (matching its frontmatter `name`) and raw `SKILL.md` text. */ -export interface BuiltinSkillSource { - name: string; - content: string; -} - -/** All bundled skills, ordered by name. */ -export const BUILTIN_SKILL_SOURCES: readonly BuiltinSkillSource[] = [ - { name: "insights", content: insights }, - { name: "jvm", content: jvm }, - { name: "profiling", content: profiling }, - { name: "runtime", content: runtime }, - { name: "stateful-debugger", content: statefulDebugger }, -]; diff --git a/packages/coding-agent/src/discovery/builtin-skill-sources/insights.md b/packages/coding-agent/src/discovery/builtin-skill-sources/insights.md deleted file mode 100644 index 34e9733a366..00000000000 --- a/packages/coding-agent/src/discovery/builtin-skill-sources/insights.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -name: insights -description: Observe a JavaScript, TypeScript or Python program with the `insights` tool — attach an instrumentation script that hooks source loads and function enter/return without editing the program under study. ---- - -# Insights instrumentation - -`insights` runs a program on the managed runtime with an **instrumentation -script** attached. The script observes the guest from the outside: it hooks -source loads and function entry/return, and its output is emitted alongside the -program's own. The program itself is never modified — which is the whole point. -Reach for it when you want to know what a program *did* without editing it, when -adding `print` statements would perturb the thing you are measuring, or when the -code you need to observe is in a dependency you should not touch. - -## Call shape - -Supply **one guest source** and **one instrumentation source**; the two choices -are independent and may be mixed. - -Guest: `code` (inline) or `path` (existing file). -Instrumentation: `insight` (inline JavaScript) or `insightPath` (a file). - -```json -{ "language": "js", "path": "sample.js", "insightPath": "trace.insight.js" } -``` - -```json -{ - "language": "js", - "code": "print(6 * 7)", - "insight": "insight.on('source', e => { if (e.characters) print(e.name); });" -} -``` - -Same optional controls as `run`: `language`, `args`, `stdin`, `cwd`, -`timeoutMs`. The instrumentation script is **always JavaScript**, whatever the -guest language — instrumenting Python still means writing JS hooks. - -The same inline-versus-path rule as `run` applies to both slots: an inline guest -runs from a scratch file and cannot resolve the project's imports, and an -instrumentation script long enough to be worth keeping belongs in a file passed -as `insightPath`. - -## Events - -- `source` — a source unit was loaded. Inspect `name` and `characters`. - Fires for the runtime's own internals too, so filter before printing. -- `enter` — a root (function/program body) was entered. With `{ roots: true }` - the second callback argument exposes the frame, so you can read argument - values at the moment of the call. -- `return` — a root returned. This is where end-of-program summaries go. - -**One-shot runs do not emit a `close` event.** There is no "program is exiting" -hook to flush from, so a summary must be emitted from the `return` of the -top-level root. That root is exposed as `:program` or `:module:eval` depending -on how the guest was loaded — match **both**, and guard with a `reported` flag -so a re-entrant root cannot print the report twice. - -## Read argument values at each call - -```js -insight.on( - "enter", - function (_context, frame) { - print(`fib(${frame.n})`); - }, - { roots: true, rootNameFilter: name => name === "fib" }, -); -``` - -`rootNameFilter` is the cheap way to scope instrumentation: without it, `enter` -fires for every root in the program and in whatever it imports. - -## Count hot roots and report once - -```js -const calls = new Map(); -let reported = false; - -insight.on( - "enter", - function (context) { - calls.set(context.name, (calls.get(context.name) || 0) + 1); - }, - { roots: true }, -); - -insight.on( - "return", - function (context) { - const top = context.name === ":program" || context.name === ":module:eval"; - if (top && !reported) { - reported = true; - for (const [name, count] of calls) print(`${name}:${count}`); - } - }, - { roots: true }, -); -``` - -## When to prefer this over `profile` - -Insights gives **exact, filtered, semantic** counts — "how many times was `fib` -called with n < 2", "which modules got loaded". `profile` gives timings and -whole-program call tables. If the question is "how much work does this -algorithm do", instrument and count; timings and JIT percentages move run to -run and make poor assertions. If the question is "where is the time going", -use `skill://profiling`. diff --git a/packages/coding-agent/src/discovery/builtin-skill-sources/jvm.md b/packages/coding-agent/src/discovery/builtin-skill-sources/jvm.md deleted file mode 100644 index c570aba9cf8..00000000000 --- a/packages/coding-agent/src/discovery/builtin-skill-sources/jvm.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -name: jvm -description: Java and Kotlin execution through `run` plus embedded JVM tooling for disassembly, formatting, jars, dependencies, and Javadoc — including main-class derivation and output guards. ---- - -# The embedded JVM toolchain - -`run` compiles and executes Java and Kotlin on the embedded JVM. The five -specialized `jvm_*` tools disassemble, format, package, inspect dependencies, -and generate Javadoc. Compilation happens in a **scratch directory** — your -project is not a build directory and is not touched, except by the two tools -that explicitly write an artifact (`jvm_jar` with `action: "create"`, and -`jvm_javadoc`). - -Execution accepts inline `code` or a standalone `.java` / `.kt` `path`. -Specialized compilation starts from inline `code`; only the two read-only -inspection modes take an existing artifact (`jvm_jar` with `action: "inspect"`, -and `jvm_deps` with `path`). There is no whole-project execution mode; use -`check` / `build` for multi-file projects (see `skill://runtime`). - -## The tools - -| Tool | Use it for | -| --- | --- | -| `run` with Java or Kotlin `language` | Compile and run inline source or a standalone file; get stdout. | -| `jvm_disassemble` | `javap -c` bytecode — see constant folding, string concat, boxing, lambda desugaring. | -| `jvm_format` | Google Java Format / ktfmt. Returns the formatted text; **does not** write it back. | -| `jvm_jar` | `action: "create"` builds a jar from source; `action: "inspect"` lists an existing one. | -| `jvm_deps` | `jdeps` — the packages and modules a class, jar or class directory actually depends on. | -| `jvm_javadoc` | Generate Javadoc HTML from Java source into a directory. Java only. | - -```json -{ - "language": "java", - "code": "public class Hello { public static void main(String[] a) { System.out.println(6 * 7); } }" -} -``` - -```json -{ "language": "kotlin", "code": "fun main() { println(6 * 7) }" } -``` - -## Main-class derivation — the usual failure - -`run`, `jvm_disassemble`, `jvm_deps` and `jvm_jar` all pick an entrypoint the -same way, and getting it wrong is the most common reason a call fails with a -correct-looking program: - -1. An explicit `mainClass` always wins (it must be a plain class name — - letters, digits, `_`, `$`, `.`). -2. Kotlin is **always `MainKt`**. The guest is written as `Main.kt`, so a - top-level `fun main()` compiles to `MainKt` regardless of what the code - looks like. Declaring `class Foo` in Kotlin does not change this. -3. Java: the `public class X`, else the **first** `class X` in the source, else - `Main`. - -The consequences worth internalizing: - -- Java source is written to `.java`, so the derived name **must** - match the declared public class or `javac` rejects it. Declare exactly one - public class and let derivation find it. -- If several classes are declared and none is public, the *first* one wins — - which is often not the one holding `main`. Pass `mainClass` explicitly. -- Do not put a `package` declaration in inline source. The file is compiled flat - in the scratch directory, so the derived bare class name will not match the - package-qualified one the JVM then looks for. -- Kotlin: the standard library is on the classpath automatically. You do not - need to add it. - -## Java 17 is the floor - -Java is compiled with `--release 17`, and the host's `JAVA_HOME` / `JDK_HOME` -are stripped from the toolchain's environment. Both halves exist for the same -reason: the compiler is the embedded one, but the `java` that runs the result -may be an older host JVM (CI images commonly pin 17), and newer bytecode on an -older JVM dies with `UnsupportedClassVersionError`. - -So: write Java `run` source against Java 17 APIs. Records, sealed types, switch -expressions and text blocks are fine. Anything added after 17 is not, and -the failure will surface as a compile error rather than as a version complaint. - -A compile error comes back exactly as the compiler reported it and the program -is not run — read the diagnostic, do not re-run hoping for a different result. - -## Writing artifacts - -`jvm_jar` (create) and `jvm_javadoc` are the only tools that write into your -project, and both are guarded: - -- `output` must be a path **inside** the working directory. `.`, `..` and - anything escaping the cwd are refused outright. -- An existing `output` is refused unless you pass `overwrite: true`. For - `jvm_jar`, an existing *directory* at `output` is always refused. -- `jvm_javadoc` is the only tool that deletes, and `overwrite: true` only ever - replaces something that already looks like a docs output (empty, or carrying - `index.html` plus javadoc's own scaffolding). A source directory or a static - site is safe — but always name a dedicated docs directory anyway - (`javadoc-out` is the default) and never point it at source. - -`jvm_format` deliberately does **not** write. It returns the formatted source; -apply it with `edit` or `write` if you want it persisted. - -To browse generated docs, `serve` the output directory and stop the job through -`hub` — see `skill://stateful-debugger`. diff --git a/packages/coding-agent/src/discovery/builtin-skill-sources/profiling.md b/packages/coding-agent/src/discovery/builtin-skill-sources/profiling.md deleted file mode 100644 index 24d290a4feb..00000000000 --- a/packages/coding-agent/src/discovery/builtin-skill-sources/profiling.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: profiling -description: Spot-profile JavaScript, TypeScript or Python with the `profile` tool — choosing between cpusampling and cputracing, and reading the report without over-reading its numbers. ---- - -# Spot profiling - -`profile` runs a program on the managed runtime under a profiler and returns the -report as text. Same source rules as `run`: **exactly one** of `code` (inline) -or `path` (existing file), plus optional `language`, `args`, `stdin`, `cwd`, -`timeoutMs`. `mode` is required. - -```json -{ "language": "js", "path": "work.js", "mode": "cpusampling" } -``` - -```json -{ "language": "python", "code": "print(sum(range(1000)))", "mode": "cputracing" } -``` - -## Choosing the mode - -This is the only real decision, and picking wrong wastes the run. - -**`cpusampling`** — statistical. The profiler interrupts on a timer and -attributes wall-clock time to whatever root is executing. Overhead is low and -roughly constant, so the program's own behaviour is largely preserved. - -- Use it to answer *"where is the time going?"* -- Prefer it for anything long enough to matter — I/O-shaped work, programs that - run for seconds, anything you would otherwise time with a stopwatch. -- Short runs give you almost no samples, and a hot function that finishes - between two samples is invisible. Below roughly a second of work, sampling - tells you nothing you can trust. - -**`cputracing`** — exact. Every root entry and exit is recorded, so invocation -counts are precise and split by runtime state (interpreted vs compiled). - -- Use it to answer *"how much work does this actually do?"* — comparing two - algorithms, confirming a memoization landed, proving a call is not made. -- The instrumentation is heavy and it distorts what it measures: tracing - overhead dominates cheap functions, and it inhibits the inlining that would - otherwise happen. Treat tracing *timings* as unusable; only the counts are - meaningful. -- On a long or call-heavy run the trace is enormous and slow. Shrink the input - first, then trace. - -The usual sequence is: sample the real workload to find the hot region, then -trace a reduced input to understand why it is hot. - -## Reading the report - -The report is a per-root table: invocation counts, time attributed, and the -split across runtime states. Read it for **relationships**, not absolutes. - -- Absolute milliseconds and JIT/compiled percentages vary substantially between - runs on the same input — warm-up, compilation timing and machine noise all - move them. Never assert on them and never report them as a measurement. -- Call-count relationships (`inner` ran 10× per `outer`) and the *ranking* of - roots by attributed time are stable and are what you should quote. -- A root you expected to be hot but which does not appear at all usually means - it was inlined or the run was too short to sample — not that it is free. - -When you need a defensible before/after claim, assert on semantic program output -and on stable call-count relationships. For exact, filtered counts, instrument -instead: see `skill://insights`. diff --git a/packages/coding-agent/src/discovery/builtin-skill-sources/runtime.md b/packages/coding-agent/src/discovery/builtin-skill-sources/runtime.md deleted file mode 100644 index d44fcd492ff..00000000000 --- a/packages/coding-agent/src/discovery/builtin-skill-sources/runtime.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -name: runtime -description: Innate execution on the managed polyglot runtime — running JavaScript, TypeScript, Python, Java, and Kotlin with `run`, selecting execution engines, validating projects with `check`, and choosing between runtime tools and `bash`/`eval`. ---- - -# The managed runtime - -Code execution is innate. A managed polyglot runtime is resolved (or downloaded) -for you and driven through named tools, so path controls, output budgets, -process ownership and approval policy all apply. Reaching for the runtime's -command-line binary through `bash` bypasses every one of those, and by default -the shell interceptor blocks the attempt and names the tool to use instead. -Do not read a command that *does* go through as permission: the interceptor -stands down when its target tool is not registered, and the user can switch the -whole group off — the reason to use the innate tool is the policy, not the -block. - -## `run` — one program, one process - -Provide **exactly one** source form: - -- Inline: `{ language: "ts" | "js" | "python" | "java" | "kotlin", - code: "…" }` — the source is written to a temp file and executed. - `language` defaults to `ts`. -- Existing file: `{ path: "tools/report.py" }` — `language` is inferred from - `.js`/`.ts`/`.py`/`.java`/`.kt` (including common module variants). - -Optional: `engine`, `args`, `stdin`, `cwd` (defaults to the session directory), -`timeoutMs`, and `mainClass` for Java/Kotlin. The result carries the resolved -engine and language, stdout, stderr, exit code, and JVM phase/class metadata -where applicable. - -**Path mode is not a convenience — it changes semantics.** An inline snippet -runs from a scratch file, so relative imports, sibling modules and data files -resolve against the temp directory and break. A `path` run executes the file -where it lives, so `import "./util.ts"`, `open("fixtures/data.json")` and the -project's own module resolution all work. If a snippet needs anything from the -project, `write` it into the project and run it by path instead of inlining it. - -Every call is a fresh process. Nothing carries over between calls — no -variables, no imports, no open handles. - -Python is a CPython-compatible engine (3.12), not the host's `python3`. Host -site-packages are not on the path; treat it as a clean interpreter. - -Engine routing: - -| Language | Default | Engine choice | -| --- | --- | --- | -| JavaScript / TypeScript | Bun | Bun or the embedded engine | -| Python | Embedded | Embedded only | -| Java / Kotlin | Embedded | Embedded only | -Invalid language/engine pairs fail before execution. Java and Kotlin compile -and run in a scratch workdir; `mainClass` overrides entrypoint derivation. -Use `check` / `build` rather than `run` for multi-file JVM projects. - -## `run` vs `bash` vs `eval` - -- `run` — you have a self-contained JavaScript, TypeScript, Python, Java, or - Kotlin program and want its output. This is the default for direct execution. -- `bash` — you have a *shell* task: invoking installed CLIs, pipelines, git, - package managers, file plumbing. -- `eval` — you want a **persistent kernel**: incremental exploration where state - survives across calls (imports → define → probe → use). Use it when the next - step depends on objects the previous step built. `run` cannot do that; `eval` - is the wrong tool for a self-contained script you want to run once. - -## Project-level tools - -- `check` — resolve dependencies and compile supported source sets through the - managed project build, without requesting deliverable artifacts. This is a - fast build-integrity gate, not a replacement for project-specific static - analysis: the runtime strips TypeScript types rather than running `tsc`, so - invoke the project's declared typecheck/check command when TypeScript type - correctness is the contract. Optional `cwd`, `timeoutMs`. -- `build` — assemble artifacts. `targets` takes `:`-prefixed build targets with - interleaved per-target options, passed through verbatim (e.g. - `[":deps", "--fresh", ":compile"]`); omit it for the default build. Use - `check` when validation is the goal and `build` when artifacts are. - -Both read the project configuration in `cwd`, so run them from the project root -rather than a subdirectory. - -Unsure what a project supports? Call `project_advice` first — it reads the -working directory in place and reports the commands, declared name/version and -dependencies the project itself declares. It is read-only and executes nothing. - -## Availability - -The runtime is resolved from, in order: an explicit configured path, the -`AURA_RUNTIME_BIN` environment override, a managed copy downloaded into the -agent's own directory, then the `PATH`. When none is present and auto-download -is enabled, the first call provisions it; when it is disabled, the tool returns -installation guidance instead of failing the task. Nothing is installed into the -project. - -If the runtime tools are absent from this session entirely, runtime support is -switched off in settings — say so rather than emulating them through `bash`. - -## Going further - -- `skill://insights` — instrument a program without editing it. -- `skill://profiling` — find where the time goes. -- `skill://jvm` — Java and Kotlin on the embedded JVM. -- `skill://stateful-debugger` — publish a CDP/DAP endpoint, or serve a directory. diff --git a/packages/coding-agent/src/discovery/builtin-skill-sources/stateful-debugger.md b/packages/coding-agent/src/discovery/builtin-skill-sources/stateful-debugger.md deleted file mode 100644 index 90cabbba778..00000000000 --- a/packages/coding-agent/src/discovery/builtin-skill-sources/stateful-debugger.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: stateful-debugger -description: Publish a CDP or DAP debug endpoint with `runtime_debug`, or a static HTTP server with `serve` — long-lived runtime jobs whose lifecycle belongs to `hub`, not to a stop tool. ---- - -# Long-lived runtime jobs: `runtime_debug` and `serve` - -Both of these tools start a process that outlives the call. Neither returns a -finished result; each returns an **endpoint plus a hub job name**, and the job -keeps running until something stops it. Everything below about the lifecycle -applies identically to both. - -## `runtime_debug` — publish a debug endpoint - -```json -{ "path": "src/problem.ts", "protocol": "cdp" } -``` - -`path` is **required and there is no inline-code mode** — the file has to -outlive the call that started it, so write the program to disk first and pass -its path. `language` is inferred from the extension. Optional: `args`, `cwd`, -`timeoutMs` (guest execution timeout), `waitSeconds` (how long startup output is -watched for the endpoint; default 15). - -`protocol` picks the wire format: - -- `cdp` (default) — Chrome DevTools Protocol. Returns a `ws://` URL. Open it in - Chrome/Chromium DevTools. -- `dap` — Debug Adapter Protocol. Returns a bare `host:port`. Attach VS Code or - another DAP client. - -**The program starts suspended and stays suspended until a client attaches.** It -will produce no output and make no progress on its own. That is the feature — -but it means "nothing happened" is the expected state, not a failure, and it -means this tool is only useful when a *human* is going to attach. If nobody is -attaching, you want `run`, `insights` or `debug`. - -`runtime_debug` is **not** the `debug` tool. `debug` is the interactive stepping -debugger this agent drives itself — breakpoints, stepping, variable inspection, -all from tool calls. `runtime_debug` publishes an endpoint for an *external* -debugger and gives this agent no stepping control at all. Reach for `debug` to -debug something yourself; reach for `runtime_debug` to hand a session to the -user. - -## `serve` — static files over HTTP - -```json -{ "directory": "javadoc-out" } -``` - -`directory` is required and resolves against `cwd` (default: the session -directory). Optional `port` (default 8080), `host` (default `127.0.0.1`), and -`waitSeconds` (default 15). Use it to preview a built site, generated API docs, -or any directory of static assets. - -## Lifecycle is hub's — there is no separate stop tool - -The result of either tool carries a hub job name. That name is the handle for -everything afterwards: - -- `hub {op: "logs", name}` — read the job's output. `follow: true` waits for - more; reuse the returned cursor. -- `hub {op: "stop", name}` — graceful tree termination. **This is the only - supported way to stop the job.** Never hunt the PID and kill it through - `bash`. - -Two consequences worth planning around: - -- **Endpoint scraping is best-effort.** The tool watches startup output for - `waitSeconds` and reports the endpoint if it appears. If it does not, the job - is *still returned* along with whatever startup output there was — the process - may simply still be coming up. Poll `hub logs` before concluding anything - failed, and raise `waitSeconds` for a slow-starting program rather than - retrying the launch (a retry leaves the first job running and, for `serve`, - the port already taken). -- **A session without `hub` cannot stop what it started.** When the `hub` tool - is not available, the result says so explicitly. Believe it: nothing in the - session can terminate the job. Report that to the user along with the endpoint - so they can stop it themselves — do not start a second one. - -Leaving a server running holds its port. Stop the job as soon as the user is -done with it, and stop it before starting another `serve` on the same port. diff --git a/packages/coding-agent/src/discovery/builtin-skills.ts b/packages/coding-agent/src/discovery/builtin-skills.ts deleted file mode 100644 index fe15499da7c..00000000000 --- a/packages/coding-agent/src/discovery/builtin-skills.ts +++ /dev/null @@ -1,291 +0,0 @@ -/** - * Builtin Skills Provider - * - * Ships the runtime skill set (`skill://runtime`, `insights`, `profiling`, - * `jvm`, `stateful-debugger`) embedded into the binary, so every session - * discovers them without the user authoring anything. - * - * Why this materializes to disk instead of serving embedded text the way the - * `builtin-defaults` rule provider does: a `Rule` carries its body in memory - * and `rule://` serves that, but a `Skill` is a *path*. Every consumer — - * `buildSkillPromptMessage`, the `skill://` protocol and its sub-path reads — - * re-reads `Skill.filePath` off disk. So the provider writes its embedded - * sources into an agent-owned directory (`/builtin-skills// - * SKILL.md`) and then scans that directory like any other skill root. The - * embedded text stays the authority: a file that drifts is rewritten on the - * next load. - * - * Because that directory is also somewhere a user can drop a skill of their - * own, the provider records what it wrote in a `.bundled.json` manifest and - * deletes only names that manifest claims. Nothing else in there is ever a - * deletion candidate. - * - * Registered at the lowest skill priority so an authored skill of the same name - * from any other provider wins the capability dedup. Users disable the set with - * `skills.enableBundled` (or the whole runtime surface with `runtime.enabled`), - * and a single skill via `skills.ignoredSkills` / a `skill:` entry in - * `disabledExtensions`. - */ -import * as fs from "node:fs/promises"; -import * as path from "node:path"; -import { getAgentDir, isEnoent, logger } from "@oh-my-pi/pi-utils"; -import { registerProvider } from "../capability"; -import { BUILTIN_SKILLS_PROVIDER_ID, type Skill, skillCapability } from "../capability/skill"; -import type { LoadContext, LoadResult } from "../capability/types"; -import { isSettingsInitialized, settings } from "../config/settings"; -import { BUILTIN_SKILL_SOURCES, type BuiltinSkillSource } from "./builtin-skill-sources"; -import { scanSkillsFromDir } from "./helpers"; - -export { BUILTIN_SKILL_SOURCES, type BuiltinSkillSource } from "./builtin-skill-sources"; - -const DISPLAY_NAME = "Bundled Runtime Skills"; -// Lowest skill priority: below the managed auto-learn provider (5) and every -// authored source, so any same-named skill anywhere overrides a bundled one. -const PRIORITY = 3; - -/** Where the bundled sources are materialized (`/builtin-skills`). */ -export function getBuiltinSkillsDir(agentDir: string = getAgentDir()): string { - return path.join(agentDir, "builtin-skills"); -} - -/** - * Whether the bundled set should be materialized and offered at all. - * - * Reads the settings singleton directly — discovery providers get no settings - * in their `LoadContext`, and `claude.ts` establishes the same fallback: assume - * enabled when settings are not initialized (discovery unit tests run without - * `Settings.init()`). The bundled skills document the innate runtime tools, so - * `runtime.enabled = false` (which unregisters those tools) also retires them. - */ -function bundledSkillsEnabled(): boolean { - if (!isSettingsInitialized()) return true; - return settings.get("runtime.enabled") !== false && settings.get("skills.enableBundled") !== false; -} - -/** - * Record of exactly which skill directories THIS provider materialized. It is - * the sole authority for what may be deleted: a directory the manifest never - * named was authored by someone else and is never touched. Dot-prefixed so - * `scanSkillsFromDir` (which only descends into non-dotted directories) cannot - * mistake it for a skill. - */ -const MANIFEST_FILE = ".bundled.json"; - -interface BundledManifest { - /** Skill directory names this provider wrote, as of the last successful pass. */ - names: string[]; -} - -async function readIfPresent(filePath: string): Promise { - try { - return await fs.readFile(filePath, "utf8"); - } catch (error) { - if (isEnoent(error)) return undefined; - throw error; - } -} - -/** Names this provider previously materialized; empty when there is no readable manifest. */ -async function readManifest(dir: string): Promise { - const raw = await readIfPresent(path.join(dir, MANIFEST_FILE)); - if (raw === undefined) return []; - try { - const parsed = JSON.parse(raw) as BundledManifest; - if (!Array.isArray(parsed.names)) return []; - // A hand-edited or partially-corrupted manifest must never aim the prune - // outside this directory: only accept bare directory names (no separators, - // no `..`, no leading dot), so `path.join(dir, name)` cannot escape. - return parsed.names.filter( - (name): name is string => - typeof name === "string" && - name.length > 0 && - !name.startsWith(".") && - !name.includes("/") && - !name.includes("\\") && - !name.includes(".."), - ); - } catch { - // A corrupt manifest means "we no longer know what we own" — the safe - // reading is that we own nothing, so nothing gets pruned this pass. - return []; - } -} - -/** - * Unique staging name. `pid` alone collides between two discovery passes racing - * inside ONE process (parallel providers, concurrent subagent sessions), which - * would let one pass rename a half-written file into place under the other. - * Matches the entropy the fork already uses in `runtime/provision.ts` and - * `utils/markit-cache.ts`. - */ -function stagingPath(target: string): string { - return `${target}.${process.pid}.${Date.now()}.${crypto.randomUUID()}.tmp`; -} - -/** - * Write one bundled skill if the on-disk copy is missing or has drifted. - * Written to a unique temp sibling and renamed, so a concurrent reader never - * scans a half-written `SKILL.md` and two concurrent writers cannot interleave. - */ -async function writeSkillSource(dir: string, source: BuiltinSkillSource, warnings: string[]): Promise { - const skillDir = path.join(dir, source.name); - const target = path.join(skillDir, "SKILL.md"); - try { - if ((await readIfPresent(target)) === source.content) return; - await fs.mkdir(skillDir, { recursive: true }); - const staging = stagingPath(target); - try { - await fs.writeFile(staging, source.content, "utf8"); - await fs.rename(staging, target); - } finally { - // A failed rename would otherwise leave the staging file behind. It is - // invisible to the scanner (only `/SKILL.md` is read), but it would - // accumulate and it would defeat the lone-`SKILL.md` shape check below. - await fs.rm(staging, { force: true }).catch(() => {}); - } - } catch (error) { - warnings.push(`Failed to materialize bundled skill "${source.name}" at ${target} (${String(error)})`); - } -} - -/** - * Reclaim directories for bundled skills an EARLIER version shipped and this one - * no longer does — otherwise a retired skill surfaces forever. - * - * Deletion is gated on the manifest, not on shape: `retired` is exactly - * `previous manifest − current source set`. This directory is also a place a - * user can legitimately drop a skill of their own (the provider scans whatever - * it finds), and a bare user-authored `SKILL.md` is shape-identical to one we - * wrote — so a name the manifest never claimed must never be a deletion - * candidate. Every prune is reported as a warning rather than done silently, - * and a retired directory the user has since added files to is kept. - */ -/** - * Returns the names that were NOT reclaimed because the removal errored — the - * caller must keep those in the manifest so ownership isn't dropped on a name - * we still have a live directory for. A name intentionally kept (user files - * present) or already gone is fully retired and is not returned. - */ -async function pruneRetiredSkills(dir: string, retired: readonly string[], warnings: string[]): Promise { - const retainedOnError: string[] = []; - await Promise.all( - retired.map(async name => { - const target = path.join(dir, name); - try { - const children = await fs.readdir(target).catch(error => { - if (isEnoent(error)) return undefined; - throw error; - }); - if (children === undefined) return; - if (children.length !== 1 || children[0] !== "SKILL.md") { - warnings.push( - `Kept retired bundled skill "${name}" at ${target}: it holds files this provider did not write`, - ); - return; - } - await fs.rm(target, { recursive: true, force: true }); - warnings.push(`Pruned retired bundled skill "${name}" from ${target}`); - } catch (error) { - // The directory is still on disk. Keep owning the name so a later - // pass retries the prune, rather than orphaning it forever. - retainedOnError.push(name); - warnings.push(`Failed to prune retired bundled skill at ${target} (${String(error)})`); - } - }), - ); - return retainedOnError; -} - -async function writeManifest(dir: string, names: readonly string[], warnings: string[]): Promise { - const target = path.join(dir, MANIFEST_FILE); - const body = `${JSON.stringify({ names: [...names] }, null, 2)}\n`; - try { - if ((await readIfPresent(target)) === body) return; - const staging = stagingPath(target); - try { - await fs.writeFile(staging, body, "utf8"); - await fs.rename(staging, target); - } finally { - await fs.rm(staging, { force: true }).catch(() => {}); - } - } catch (error) { - warnings.push(`Failed to record the bundled skills manifest at ${target} (${String(error)})`); - } -} - -/** - * Bring `/builtin-skills` in line with the embedded sources. - * Returns warnings rather than throwing: a read-only or otherwise unwritable - * agent directory must degrade to "no bundled skills", never fail discovery. - */ -export async function materializeBuiltinSkills(dir: string): Promise { - const warnings: string[] = []; - const names = BUILTIN_SKILL_SOURCES.map(source => source.name); - try { - await fs.mkdir(dir, { recursive: true }); - const previous = await readManifest(dir); - const current = new Set(names); - await Promise.all(BUILTIN_SKILL_SOURCES.map(source => writeSkillSource(dir, source, warnings))); - const retainedOnError = await pruneRetiredSkills( - dir, - previous.filter(name => !current.has(name)), - warnings, - ); - // Keep owning any retired name whose removal failed, so the next pass - // retries it instead of leaving an orphaned directory that surfaces as a - // skill forever. - await writeManifest(dir, [...names, ...retainedOnError], warnings); - } catch (error) { - warnings.push(`Failed to prepare the bundled skills directory ${dir} (${String(error)})`); - } - return warnings; -} - -/** - * Remove what this provider materialized once it is switched off, so a disabled - * bundled set leaves no tree behind. Manifest-gated like the retirement prune: - * only names we recorded are removed, and only when they still look like ours. - * The directory itself goes only if it ends up empty — a user skill parked in - * there keeps it (and its manifest) alive. - */ -async function unmaterializeBuiltinSkills(dir: string): Promise { - const previous = await readManifest(dir).catch(() => []); - if (previous.length === 0) return; - // The provider returns no items in this branch, so its warnings would go - // nowhere — log them instead, keeping every deletion accounted for. - const warnings: string[] = []; - const retainedOnError = await pruneRetiredSkills(dir, previous, warnings); - for (const warning of warnings) logger.debug(`builtin-skills: ${warning}`); - if (retainedOnError.length > 0) { - // A removal failed; keep the manifest so a later suppressed pass retries - // rather than orphaning the directory. - await writeManifest(dir, retainedOnError, warnings); - return; - } - await fs.rm(path.join(dir, MANIFEST_FILE), { force: true }).catch(() => {}); - await fs.rmdir(dir).catch(() => {}); -} - -async function loadBuiltinSkills(ctx: LoadContext): Promise> { - const dir = getBuiltinSkillsDir(); - if (!bundledSkillsEnabled()) { - await unmaterializeBuiltinSkills(dir); - return { items: [] }; - } - const warnings = await materializeBuiltinSkills(dir); - const scan = await scanSkillsFromDir(ctx, { - dir, - providerId: BUILTIN_SKILLS_PROVIDER_ID, - level: "user", - requireDescription: true, - }); - return { items: scan.items, warnings: [...warnings, ...(scan.warnings ?? [])] }; -} - -registerProvider(skillCapability.id, { - id: BUILTIN_SKILLS_PROVIDER_ID, - displayName: DISPLAY_NAME, - description: "Runtime skills shipped with the agent (disable via skills.enableBundled)", - priority: PRIORITY, - load: loadBuiltinSkills, -}); diff --git a/packages/coding-agent/src/discovery/claude-plugins.ts b/packages/coding-agent/src/discovery/claude-plugins.ts index fc8145031d5..3065d096565 100644 --- a/packages/coding-agent/src/discovery/claude-plugins.ts +++ b/packages/coding-agent/src/discovery/claude-plugins.ts @@ -30,6 +30,22 @@ const PROVIDER_ID = "claude-plugins"; const DISPLAY_NAME = "Claude Code Marketplace"; const PRIORITY = 70; // Below claude.ts (80) so user .claude/ overrides win +const INHERENT_SUPERPOWERS_SKILLS: Readonly> = { + "using-superpowers": true, + brainstorming: true, + "writing-plans": true, + "executing-plans": true, + "test-driven-development": true, + "systematic-debugging": true, + "verification-before-completion": true, + "dispatching-parallel-agents": true, + "subagent-driven-development": true, + "using-git-worktrees": true, + "requesting-code-review": true, + "receiving-code-review": true, + "finishing-a-development-branch": true, +}; + interface ClaudePluginManifest { skills?: string | string[]; "slash-commands"?: string | string[]; @@ -209,10 +225,10 @@ async function loadSkills(ctx: LoadContext): Promise> { }), ), ); - return { scanResults, resolveWarnings }; + return { root, scanResults, resolveWarnings }; }), ); - for (const { scanResults, resolveWarnings } of results) { + for (const { root, scanResults, resolveWarnings } of results) { warnings.push(...resolveWarnings); // Intentionally do NOT prefix skill names with `root.plugin`. // The `plugin:name` format breaks skill:// URL parsing (colons are @@ -220,7 +236,11 @@ async function loadSkills(ctx: LoadContext): Promise> { // Dedup-by-key in the capability layer already handles name collisions // across providers using priority ordering. for (const result of scanResults) { - items.push(...result.items); + const discovered = + root.plugin === "superpowers" + ? result.items.filter(skill => INHERENT_SUPERPOWERS_SKILLS[skill.name] !== true) + : result.items; + items.push(...discovered); if (result.warnings) warnings.push(...result.warnings); } } diff --git a/packages/coding-agent/src/discovery/index.ts b/packages/coding-agent/src/discovery/index.ts index 48e24cffdea..c5ed5fd00c7 100644 --- a/packages/coding-agent/src/discovery/index.ts +++ b/packages/coding-agent/src/discovery/index.ts @@ -23,7 +23,6 @@ import "../capability/tool"; import "./agents-md"; import "./builtin"; import "./builtin-defaults"; -import "./builtin-skills"; import "./claude"; import "./claude-plugins"; import "./cline"; diff --git a/packages/coding-agent/src/extensibility/skills.ts b/packages/coding-agent/src/extensibility/skills.ts index 2c97a695ca8..9669c37ea74 100644 --- a/packages/coding-agent/src/extensibility/skills.ts +++ b/packages/coding-agent/src/extensibility/skills.ts @@ -6,7 +6,7 @@ import { MANAGED_SKILLS_PROVIDER_ID, sanitizeManagedDescription, } from "../autolearn/managed-skills"; -import { BUILTIN_SKILLS_PROVIDER_ID, skillCapability } from "../capability/skill"; +import { skillCapability } from "../capability/skill"; import type { SourceMeta } from "../capability/types"; import type { SkillsSettings } from "../config/settings"; import { type Skill as CapabilitySkill, loadCapability } from "../discovery"; @@ -131,7 +131,6 @@ export async function loadSkills(options: LoadSkillsOptions = {}): Promise` before proceeding. +Skills are optional domain knowledge and workflows. If one matches your task, you MUST read `skill://` before proceeding. {{#each skills}} - {{name}}: {{description}} @@ -129,6 +138,24 @@ Specialized tools MUST replace shell equivalents: {{#has tools "bash"}}- `{{toolRefs.bash}}`: real binaries and short fact pipelines only; shadowed specialized commands are blocked.{{/has}} {{#has tools "bash"}}- Bash litmus: one external CLI or short pipeline producing a count, frequency, set difference, or checksum. Use specialized tools to move, page, or trim bytes.{{/has}} +{{#ifAny (includes tools "run") (includes tools "check") (includes tools "insights") (includes tools "profile") (includes tools "serve") (includes tools "jvm_disassemble") (includes tools "jvm_format") (includes tools "jvm_jar") (includes tools "jvm_deps")}} +# Runtime execution +{{#has tools "run"}}- Direct program execution → `{{toolRefs.run}}`.{{/has}} +{{#has tools "eval"}}- Persistent exploration across calls → `{{toolRefs.eval}}`.{{/has}} +{{#has tools "bash"}}- Shell commands and installed CLIs → `{{toolRefs.bash}}`.{{/has}} +{{#has tools "check"}}- Validation without artifacts → `{{toolRefs.check}}`.{{/has}} +{{#has tools "insights"}}- Source-load and function observations → `{{toolRefs.insights}}`.{{/has}} +{{#has tools "profile"}}- CPU profiling → `{{toolRefs.profile}}`.{{/has}} +{{#has tools "serve"}}- Static HTTP previews → `{{toolRefs.serve}}`.{{/has}} +{{#has tools "jvm_disassemble"}}- JVM bytecode disassembly → `{{toolRefs.jvm_disassemble}}`.{{/has}} +{{#has tools "jvm_format"}}- Java/Kotlin source formatting → `{{toolRefs.jvm_format}}`.{{/has}} +{{#has tools "jvm_jar"}}- JAR creation or inspection → `{{toolRefs.jvm_jar}}`.{{/has}} +{{#has tools "jvm_deps"}}- JVM dependency analysis → `{{toolRefs.jvm_deps}}`.{{/has}} +{{#has tools "run"}}- Standalone Java/Kotlin → `{{toolRefs.run}}` or the matching `jvm_*`; use project build commands only for declared builds.{{/has}} +- A successful runtime result is execution evidence; do not repeat equivalent commands solely to confirm it. +{{#has tools "bash"}}- NEVER invoke the runtime binary through `{{toolRefs.bash}}`.{{/has}} +{{/ifAny}} + {{#if autoQaEnabled}} `{{toolRefs.write}} xd://report_issue` powers automated QA. If ANY tool returns output inconsistent with its described behavior given your parameters, write `: ` as plain text to `xd://report_issue`. Don't hesitate — false positives are fine. diff --git a/packages/coding-agent/src/prompts/tools/jvm-deps.md b/packages/coding-agent/src/prompts/tools/jvm-deps.md index 036786d2fd6..4b724bf4789 100644 --- a/packages/coding-agent/src/prompts/tools/jvm-deps.md +++ b/packages/coding-agent/src/prompts/tools/jvm-deps.md @@ -1,4 +1,4 @@ -Analyze JVM package/module dependencies with `jdeps`. Provide `path` to an -existing `.class`, `.jar`, or class directory for read-only analysis, or provide -`language` + `code` to compile in scratch space and then analyze. No project -files are modified. +Analyze JVM dependencies with `jdeps`. `path` accepts Java/Kotlin source, a +class, JAR, or class directory; alternatively provide `language` + `code`. +Sources compile in scratch space. `output` optionally writes a cwd-relative +report; replacing it requires `overwrite: true`. diff --git a/packages/coding-agent/src/prompts/tools/jvm-jar.md b/packages/coding-agent/src/prompts/tools/jvm-jar.md index 468a028b232..02dc17d11e8 100644 --- a/packages/coding-agent/src/prompts/tools/jvm-jar.md +++ b/packages/coding-agent/src/prompts/tools/jvm-jar.md @@ -1,9 +1,6 @@ -Create a JAR from Java/Kotlin source or inspect an existing JAR. +Create or inspect a JAR. -- `create` requires `language` + `code` + `output`; compile in scratch space, - write to in-project `output`, and derive the manifest main class unless - `mainClass` is supplied. Reject `.`, `..`, - paths outside the cwd, existing directories, and existing files unless - `overwrite: true`. -- `inspect` requires `jar`; list entries from an existing in-project JAR; - read-only. +- `create`: compile `language` + `code` in scratch space and write `output`. + The main class is derived unless supplied. Output must stay inside the cwd; + existing files require `overwrite: true`, and directories are refused. +- `inspect`: list an existing in-project `jar`; read-only. diff --git a/packages/coding-agent/src/prompts/tools/jvm-javadoc.md b/packages/coding-agent/src/prompts/tools/jvm-javadoc.md deleted file mode 100644 index 84f37837f5d..00000000000 --- a/packages/coding-agent/src/prompts/tools/jvm-javadoc.md +++ /dev/null @@ -1,8 +0,0 @@ -Generate Javadoc HTML from Java source into a dedicated in-project directory -(`javadoc-out` default). Reject `.`, `..`, paths outside the cwd, and existing -output unless `overwrite: true`. - -Overwrite only empty directories or recognized Javadoc output (`index.html` -plus Javadoc scaffolding); never other directories. Use a dedicated docs -directory, never source or static-site directories. Open `/index.html` -to browse the result. diff --git a/packages/coding-agent/src/prompts/tools/runtime-advice.md b/packages/coding-agent/src/prompts/tools/runtime-advice.md deleted file mode 100644 index 8efbb017577..00000000000 --- a/packages/coding-agent/src/prompts/tools/runtime-advice.md +++ /dev/null @@ -1,9 +0,0 @@ -Read project configuration and manifests to return the runtime's -build/run/test/install guidance. Read-only: executes, builds, and writes nothing. -Use before guessing when project manifests define the answer. - -Returned CLI verbs describe the runtime; invoke only verbs exposed by innate -tools (`run`, `check`, `build`, `jvm_*`). Treat the rest as informational. - -Guidance availability and wording depend on the installed runtime build; -failures return the runtime error verbatim. diff --git a/packages/coding-agent/src/prompts/tools/runtime-build.md b/packages/coding-agent/src/prompts/tools/runtime-build.md deleted file mode 100644 index 57f82931a83..00000000000 --- a/packages/coding-agent/src/prompts/tools/runtime-build.md +++ /dev/null @@ -1,6 +0,0 @@ -Build project artifacts on the managed runtime; use `check` for validation-only -runs. - -`targets` accepts `:`-prefixed build targets with interleaved per-target options, -passed verbatim (e.g. `[":deps", "--fresh", ":compile"]`). Omit `targets` for -the default build. diff --git a/packages/coding-agent/src/prompts/tools/runtime-check.md b/packages/coding-agent/src/prompts/tools/runtime-check.md index 0939d75cdce..492353e9a5d 100644 --- a/packages/coding-agent/src/prompts/tools/runtime-check.md +++ b/packages/coding-agent/src/prompts/tools/runtime-check.md @@ -1,7 +1,4 @@ Resolve dependencies and compile supported source sets without producing -artifacts. Use as a fast build-integrity check after edits; use `build` when -artifacts are required. - -This is not project-specific static analysis or a TypeScript typecheck: the -runtime strips TypeScript types instead of running `tsc`. Run the project's -declared typecheck/check command when TypeScript correctness is the contract. +artifacts. This is a fast integrity probe, not project static analysis or a +TypeScript typecheck: TypeScript types are stripped. Run the project's declared +typecheck/check command when TypeScript correctness is the contract. diff --git a/packages/coding-agent/src/prompts/tools/runtime-debug.md b/packages/coding-agent/src/prompts/tools/runtime-debug.md deleted file mode 100644 index 602137b9a3d..00000000000 --- a/packages/coding-agent/src/prompts/tools/runtime-debug.md +++ /dev/null @@ -1,12 +0,0 @@ -Publish a JS/TS/Python program's debug endpoint from the managed runtime as a -supervised background job. `path` is required; inline code cannot outlive the -call. `protocol: "cdp"` (default) returns `ws://`; `"dap"` returns `host:port`. - -The program starts suspended until an external debugger attaches. This differs -from `debug`, which the agent drives interactively. - -Returns the endpoint plus a hub job name. Use `hub logs` for output and -`hub stop` to release the job; no separate stop tool exists. No hub? Report that -the job cannot be stopped from this session. A missing endpoint after the wait -still returns the job and startup output; inspect `hub logs` before declaring -failure. diff --git a/packages/coding-agent/src/prompts/tools/runtime-insights.md b/packages/coding-agent/src/prompts/tools/runtime-insights.md index 2f61a1e668d..4451c319ea8 100644 --- a/packages/coding-agent/src/prompts/tools/runtime-insights.md +++ b/packages/coding-agent/src/prompts/tools/runtime-insights.md @@ -1,6 +1,4 @@ -Run a program on the managed runtime with JavaScript instrumentation. Provide -the program via `code` or `path`; provide instrumentation via `insight` or -`insightPath`. - -Hooks source loads and function enter/return; observations accompany program -output. One-shot runs emit no close event. +Observe a JavaScript, TypeScript, or Python program with instrumentation. +`code`/`path` selects the program; `insight`/`insightPath` selects the hooks. +Source loads and function enter/return events accompany program output. +One-shot runs emit no close event. diff --git a/packages/coding-agent/src/prompts/tools/runtime-profile.md b/packages/coding-agent/src/prompts/tools/runtime-profile.md index d77fd68cf78..5bb9e166564 100644 --- a/packages/coding-agent/src/prompts/tools/runtime-profile.md +++ b/packages/coding-agent/src/prompts/tools/runtime-profile.md @@ -1,5 +1,3 @@ -Profile a program on the managed runtime. `mode: "cputracing"` gives exact call -tracing; `"cpusampling"` gives lower-overhead statistical sampling and SHOULD be -used for longer runs. - -Provide the program via `code` or `path`. Returns a text report. +Profile JavaScript, TypeScript, or Python. `cputracing` records exact calls; +`cpusampling` collects lower-overhead statistical samples and SHOULD be used for +longer runs. Returns a text report. diff --git a/packages/coding-agent/src/prompts/tools/runtime-run.md b/packages/coding-agent/src/prompts/tools/runtime-run.md index 8eb92964885..3f1f3ef39fd 100644 --- a/packages/coding-agent/src/prompts/tools/runtime-run.md +++ b/packages/coding-agent/src/prompts/tools/runtime-run.md @@ -1,12 +1,9 @@ -Execute JavaScript, TypeScript, Python, Java, or Kotlin directly on the managed -runtime; use `bash` for shell commands and `eval` for notebook-style execution. +Execute JavaScript, TypeScript, Python, Java, or Kotlin directly. Use `bash` for +shell commands and `eval` for persistent notebook exploration. -Provide exactly one of `code` (temporary source) or `path` (existing file; -preserves project-relative imports and data access). Inline language defaults to -`ts`; path language is inferred unless supplied. JavaScript/TypeScript default -to Bun and may select either available engine; Python/Java/Kotlin require the -embedded engine. +Provide exactly one of `code` or `path`; a path retains project-relative imports +and data access. Inline source defaults to TypeScript, path language is inferred, +and JavaScript/TypeScript default to Bun. -Returns resolved language/engine, stdout, stderr, exit code, and JVM compile/run -phase when applicable. Missing runtime returns installation guidance instead of -failure. +Returns the resolved language/engine, streams, exit status, and JVM phase. +Missing runtime support yields setup guidance. diff --git a/packages/coding-agent/src/prompts/tools/runtime-serve.md b/packages/coding-agent/src/prompts/tools/runtime-serve.md index cbd55e0cfe6..22f26ecde0a 100644 --- a/packages/coding-agent/src/prompts/tools/runtime-serve.md +++ b/packages/coding-agent/src/prompts/tools/runtime-serve.md @@ -1,7 +1,5 @@ -Serve a static directory over HTTP on the managed runtime to preview built -sites, generated docs, or other static assets. +Serve a static directory over HTTP as a supervised background job. -Returns the URL plus a hub job name. Use `hub logs` for output and `hub stop` to -release the job and port; no separate stop tool exists. No hub? Report that the -job cannot be stopped from this session. A missing URL after the wait still -returns the job and startup output; inspect `hub logs` before declaring failure. +Returns the URL and hub job. Use `hub logs` for output and `hub stop` to release +it. If URL discovery times out, inspect the job logs before declaring failure. +Without hub, report that this session cannot stop the job. diff --git a/packages/coding-agent/src/runtime/format.ts b/packages/coding-agent/src/runtime/format.ts index 2ac25372797..407457ca607 100644 --- a/packages/coding-agent/src/runtime/format.ts +++ b/packages/coding-agent/src/runtime/format.ts @@ -1,5 +1,10 @@ import type { RuntimeExecResult } from "./protocol"; +/** Whether an execution result represents a failed, timed out, or cancelled tool call. */ +export function execResultFailed(result: RuntimeExecResult): boolean { + return result.exitCode !== 0 || result.killed; +} + /** * Render an exec result for the model: stdout, stderr, and an exit annotation. * diff --git a/packages/coding-agent/src/runtime/protocol.ts b/packages/coding-agent/src/runtime/protocol.ts index cfe975e3f56..49d41cdd5e9 100644 --- a/packages/coding-agent/src/runtime/protocol.ts +++ b/packages/coding-agent/src/runtime/protocol.ts @@ -20,12 +20,10 @@ export const RUNTIME_PROTOCOL_VERSION = 3 as const; export type RuntimeMethod = | "runtime/run" | "runtime/check" - | "runtime/build" | "runtime/insights" | "runtime/profile" | "runtime/jvm" | "runtime/spawn" - | "runtime/advice" | "runtime/status"; export type RuntimeLanguage = "js" | "ts" | "python" | "java" | "kotlin"; @@ -108,8 +106,8 @@ export interface RuntimeProfileParams extends RuntimeRunParams { mode: "cputracing" | "cpusampling"; } -/** The six JVM flows behind the single `runtime/jvm` method. */ -export type RuntimeJvmAction = "run" | "disassemble" | "format" | "jar" | "deps" | "javadoc"; +/** The five JVM flows behind the single `runtime/jvm` method. */ +export type RuntimeJvmAction = "run" | "disassemble" | "format" | "jar" | "deps"; /** * Parameters for `runtime/jvm`. One method, one action union: every flow is @@ -120,7 +118,7 @@ export type RuntimeJvmAction = "run" | "disassemble" | "format" | "jar" | "deps" */ export interface RuntimeJvmParams { action: RuntimeJvmAction; - /** Source language. Required for every action except `javadoc` (Java-only) and `jar`/`deps` in artifact mode. */ + /** Source language. Required except for `jar`/`deps` in artifact mode. */ language?: JvmLanguage; /** Inline source to compile. */ code?: string; @@ -132,13 +130,13 @@ export interface RuntimeJvmParams { stdin?: string; /** `jar` sub-mode: build a jar from source, or list an existing one. Default `create`. */ mode?: "create" | "inspect"; - /** Destination written by `jar` (create) and `javadoc`, resolved against `cwd`. */ + /** Destination written by `jar` (create) or `deps`, resolved against `cwd`. */ output?: string; /** Required to replace an existing `output`. */ overwrite?: boolean; /** Existing jar to inspect (`jar`, mode `inspect`), resolved against `cwd`. */ jar?: string; - /** Source path for `run`, or existing `.class`/`.jar`/directory for `deps`; resolved against `cwd`. */ + /** Source path for `run`; source, `.class`, `.jar`, or class directory for `deps`; resolved against `cwd`. */ path?: string; /** Base directory for path-bearing fields and the `run` program cwd. */ cwd?: string; @@ -163,16 +161,12 @@ export interface RuntimeJvmResult extends RuntimeExecResult { className?: string; /** `format`: the formatted source, read back from the workdir. */ formatted?: string; - /** `jar`/`javadoc`: absolute path actually written. */ + /** `jar`/`deps`: absolute path actually written. */ output?: string; /** `jar` (inspect): absolute path of the archive that was listed. */ jar?: string; /** `jar`: `jar --list` output for the built or inspected archive. */ listing?: string; - /** `javadoc`: number of entries copied into `output`. */ - entryCount?: number; - /** `javadoc`: first few top-level entries of `output`, for orientation. */ - topLevel?: string[]; } // ── runtime/spawn: launch descriptors for long-running processes ───────────── @@ -183,36 +177,15 @@ export interface RuntimeJvmResult extends RuntimeExecResult { // the endpoint the process prints — and nothing in the runtime layer holds a // process handle. -/** Wire protocol a debug session speaks: Chrome DevTools, or Debug Adapter. */ -export type RuntimeDebugProtocol = "cdp" | "dap"; - -/** Which long-running flow to compose a descriptor for. */ -export type RuntimeSpawnMode = "debug" | "serve"; - +/** Parameters for composing a supervised static-file server launch. */ export interface RuntimeSpawnParams { - mode: RuntimeSpawnMode; - /** - * `debug`: existing program file to run under the debugger. Required — unlike - * `runtime/run` there is no inline-code mode, because the file would have to - * outlive the request that created it and no request-scoped workdir can - * promise that. - */ - path?: string; - /** `debug`: language override; inferred from `path`'s extension otherwise. */ - language?: RuntimeLanguage; - /** `debug`: debug wire protocol. Default `cdp`. */ - protocol?: RuntimeDebugProtocol; - /** `debug`: arguments passed to the program after `--`. */ - args?: string[]; - /** `debug`: guest execution timeout, passed to the runtime as `--timeout ms`. */ - timeoutMs?: number; - /** `serve`: directory of static files to serve, resolved against `cwd`. Required. */ - directory?: string; - /** `serve`: TCP port to bind. */ + /** Directory of static files to serve, resolved against `cwd`. */ + directory: string; + /** TCP port to bind. */ port?: number; - /** `serve`: interface to bind. */ + /** Interface to bind. */ host?: string; - /** Working directory for the launched process, and the base for `path`/`directory`. */ + /** Working directory for the launched process and base for `directory`. */ cwd?: string; } @@ -260,28 +233,12 @@ export interface RuntimeLaunchDescriptor { shimWarning?: string; } -/** - * Parameters for `runtime/advice` — the runtime's own project guidance. There is - * nothing to configure but *where* to look: the guidance is derived entirely from - * what the directory contains (`elide.pkl`, package manifests), so `cwd` is the - * only input and it names a real project directory, never a request workdir. - */ -export interface RuntimeAdviceParams { - /** Project directory to inspect. Defaults to the endpoint process cwd. */ +/** Validation-only project compilation; never emits build artifacts. */ +export interface RuntimeCheckParams { cwd?: string; timeoutMs?: number; } -export interface RuntimeBuildParams { - /** ':'-prefixed build targets with scoped options, passed through verbatim. */ - targets?: string[]; - cwd?: string; - timeoutMs?: number; -} - -/** v1: check = validation build (resolve + compile, no artifacts requested). */ -export type RuntimeCheckParams = RuntimeBuildParams; - export interface RuntimeExecResult { exitCode: number; stdout: string; diff --git a/packages/coding-agent/src/runtime/service.ts b/packages/coding-agent/src/runtime/service.ts index f7069e9268a..fe3433fb829 100644 --- a/packages/coding-agent/src/runtime/service.ts +++ b/packages/coding-agent/src/runtime/service.ts @@ -1,7 +1,5 @@ import { createRequest, - type RuntimeAdviceParams, - type RuntimeBuildParams, type RuntimeCheckParams, type RuntimeExecResult, type RuntimeInsightsParams, @@ -19,6 +17,7 @@ import { type RuntimeStatusResult, unwrapResponse, } from "./protocol"; +import { observeRuntimeCall } from "./telemetry"; export interface RuntimeEndpoint { request(req: RuntimeRpcRequest, signal?: AbortSignal): Promise; @@ -35,45 +34,35 @@ export class RuntimeService { constructor(private readonly endpoint: RuntimeEndpoint) {} - async #call(method: RuntimeMethod, params: unknown, signal?: AbortSignal): Promise { - if (this.#closed) throw new RuntimeRpcError("internal", "Runtime service is closed."); - return unwrapResponse(await this.endpoint.request(createRequest(method, params), signal)); + async #call(method: RuntimeMethod, params: unknown, signal?: AbortSignal, sessionId?: string): Promise { + return observeRuntimeCall(method, params, signal, sessionId, async () => { + if (this.#closed) throw new RuntimeRpcError("internal", "Runtime service is closed."); + return unwrapResponse(await this.endpoint.request(createRequest(method, params), signal)); + }); } - run(params: RuntimeRunParams, signal?: AbortSignal): Promise { - return this.#call("runtime/run", params, signal); + run(params: RuntimeRunParams, signal?: AbortSignal, sessionId?: string): Promise { + return this.#call("runtime/run", params, signal, sessionId); } - check(params: RuntimeCheckParams, signal?: AbortSignal): Promise { - return this.#call("runtime/check", params, signal); + check(params: RuntimeCheckParams, signal?: AbortSignal, sessionId?: string): Promise { + return this.#call("runtime/check", params, signal, sessionId); } - build(params: RuntimeBuildParams, signal?: AbortSignal): Promise { - return this.#call("runtime/build", params, signal); + insights(params: RuntimeInsightsParams, signal?: AbortSignal, sessionId?: string): Promise { + return this.#call("runtime/insights", params, signal, sessionId); } - insights(params: RuntimeInsightsParams, signal?: AbortSignal): Promise { - return this.#call("runtime/insights", params, signal); + profile(params: RuntimeProfileParams, signal?: AbortSignal, sessionId?: string): Promise { + return this.#call("runtime/profile", params, signal, sessionId); } - profile(params: RuntimeProfileParams, signal?: AbortSignal): Promise { - return this.#call("runtime/profile", params, signal); - } - /** One of the six JVM flows; see {@link RuntimeJvmParams.action}. */ - jvm(params: RuntimeJvmParams, signal?: AbortSignal): Promise { - return this.#call("runtime/jvm", params, signal); - } - /** - * Compose the command line for a long-running flow (`debug`, `serve`) without - * starting anything. The caller starts the returned descriptor through the - * `hub` supervisor and owns its lifecycle; nothing here holds a process. - */ - spawn(params: RuntimeSpawnParams, signal?: AbortSignal): Promise { - return this.#call("runtime/spawn", params, signal); + /** Run a specialized JVM analysis, formatting, or artifact flow. */ + jvm(params: RuntimeJvmParams, signal?: AbortSignal, sessionId?: string): Promise { + return this.#call("runtime/jvm", params, signal, sessionId); } /** - * The runtime's own build/run/test/install guidance for a project directory. - * Read-only, and it runs in the real directory — the guidance is derived from - * the manifests it finds there. + * Compose a supervised static-server command without starting it. The caller + * starts the descriptor through `hub` and owns its lifecycle. */ - advice(params: RuntimeAdviceParams, signal?: AbortSignal): Promise { - return this.#call("runtime/advice", params, signal); + spawn(params: RuntimeSpawnParams, signal?: AbortSignal, sessionId?: string): Promise { + return this.#call("runtime/spawn", params, signal, sessionId); } status(): Promise { return this.#call("runtime/status", undefined); diff --git a/packages/coding-agent/src/runtime/telemetry.ts b/packages/coding-agent/src/runtime/telemetry.ts new file mode 100644 index 00000000000..34f98357d03 --- /dev/null +++ b/packages/coding-agent/src/runtime/telemetry.ts @@ -0,0 +1,179 @@ +import { isRecord, logger } from "@oh-my-pi/pi-utils"; +import { trace } from "@opentelemetry/api"; +import { + emitTelemetryEvent, + type RuntimeCallCompletedTelemetry, + type RuntimeCallErrorType, + type RuntimeCallOutcome, +} from "../telemetry/events"; +import { + type RuntimeExecResult, + type RuntimeJvmAction, + type RuntimeLanguage, + type RuntimeMethod, + RuntimeRpcError, + type RuntimeRunParams, + resolveRunTarget, +} from "./protocol"; + +const RUNTIME_LANGUAGES: Readonly> = { + js: true, + ts: true, + python: true, + java: true, + kotlin: true, +}; +const JVM_ACTIONS: Readonly> = { + run: "run", + disassemble: "disassemble", + format: "format", + jar: "jar", + deps: "deps", +}; + +interface RuntimeCallClassification { + action?: RuntimeJvmAction; + language?: RuntimeLanguage; + outcome: RuntimeCallOutcome; + exitCode?: number; + killed?: boolean; + errorType?: RuntimeCallErrorType; +} + +/** Observe one protocol request without changing its result or failure. */ +export async function observeRuntimeCall( + method: RuntimeMethod, + params: unknown, + signal: AbortSignal | undefined, + sessionId: string | undefined, + call: () => Promise, +): Promise { + const startedAt = performance.now(); + let result: T | undefined; + let failure: unknown; + let failed = false; + try { + result = await call(); + return result; + } catch (error) { + failed = true; + failure = error; + throw error; + } finally { + const durationMs = Math.max(0, performance.now() - startedAt); + const classification = classifyRuntimeCall(method, params, result, failed, failure, signal); + const event: RuntimeCallCompletedTelemetry = { + type: "runtime.call.completed", + sessionId, + method, + durationMs, + ...classification, + }; + annotateActiveSpan(event); + emitTelemetryEvent(event); + } +} + +function classifyRuntimeCall( + method: RuntimeMethod, + params: unknown, + result: unknown, + failed: boolean, + failure: unknown, + signal: AbortSignal | undefined, +): RuntimeCallClassification { + const action = runtimeAction(method, params); + const language = runtimeLanguage(method, params, result); + if (failed) { + const errorType = runtimeErrorType(failure, signal); + return { + action, + language, + outcome: errorType === "timeout" ? "timeout" : errorType === "cancelled" ? "cancelled" : "error", + errorType, + }; + } + const exec = runtimeExecResult(result); + if (!exec) return { action, language, outcome: "ok" }; + if (exec.killed) { + return { + action, + language, + outcome: signal?.aborted ? "cancelled" : "timeout", + exitCode: exec.exitCode, + killed: true, + errorType: "killed", + }; + } + if (exec.exitCode !== 0) { + return { + action, + language, + outcome: "error", + exitCode: exec.exitCode, + killed: false, + errorType: "non_zero_exit", + }; + } + return { action, language, outcome: "ok", exitCode: exec.exitCode, killed: false }; +} + +function runtimeAction(method: RuntimeMethod, params: unknown): RuntimeJvmAction | undefined { + if (!isRecord(params)) return undefined; + if (method === "runtime/jvm" && typeof params.action === "string") return JVM_ACTIONS[params.action]; + return undefined; +} + +function runtimeLanguage(method: RuntimeMethod, params: unknown, result: unknown): RuntimeLanguage | undefined { + if (isRecord(result) && isRuntimeLanguage(result.language)) return result.language; + if (method === "runtime/jvm" && isRecord(params)) { + if (isRuntimeLanguage(params.language)) return params.language; + return undefined; + } + if (method === "runtime/run" || method === "runtime/insights" || method === "runtime/profile") { + try { + return resolveRunTarget(params as RuntimeRunParams).language; + } catch { + return isRecord(params) && isRuntimeLanguage(params.language) ? params.language : undefined; + } + } + return undefined; +} + +function runtimeExecResult(value: unknown): RuntimeExecResult | undefined { + if ( + !isRecord(value) || + typeof value.exitCode !== "number" || + typeof value.durationMs !== "number" || + typeof value.killed !== "boolean" + ) { + return undefined; + } + return value as unknown as RuntimeExecResult; +} + +function runtimeErrorType(failure: unknown, signal: AbortSignal | undefined): RuntimeCallErrorType { + if (failure instanceof RuntimeRpcError) return failure.code; + if (signal?.aborted) return "cancelled"; + return "unknown"; +} + +function isRuntimeLanguage(value: unknown): value is RuntimeLanguage { + return typeof value === "string" && RUNTIME_LANGUAGES[value] === true; +} + +function annotateActiveSpan(event: RuntimeCallCompletedTelemetry): void { + const span = trace.getActiveSpan(); + if (!span) return; + try { + span.setAttribute("aura.runtime.method", event.method); + if (event.action !== undefined) span.setAttribute("aura.runtime.action", event.action); + if (event.language !== undefined) span.setAttribute("aura.runtime.language", event.language); + span.setAttribute("aura.runtime.outcome", event.outcome); + span.setAttribute("aura.runtime.duration_ms", event.durationMs); + if (event.exitCode !== undefined) span.setAttribute("aura.runtime.exit_code", event.exitCode); + if (event.killed !== undefined) span.setAttribute("aura.runtime.killed", event.killed); + } catch (error) { + logger.debug("Failed to annotate runtime telemetry span", { error: String(error) }); + } +} diff --git a/packages/coding-agent/src/runtime/transport/local.ts b/packages/coding-agent/src/runtime/transport/local.ts index 8ad745d6dd3..755883f1082 100644 --- a/packages/coding-agent/src/runtime/transport/local.ts +++ b/packages/coding-agent/src/runtime/transport/local.ts @@ -9,9 +9,7 @@ import { type JvmLanguage, okResponse, RUNTIME_PROTOCOL_VERSION, - type RuntimeAdviceParams, - type RuntimeBuildParams, - type RuntimeDebugProtocol, + type RuntimeCheckParams, type RuntimeEndpointRule, type RuntimeExecResult, type RuntimeInsightsParams, @@ -218,18 +216,13 @@ async function realpathExistingPrefix(target: string): Promise { } /** - * Resolve a project-writing destination and bound it to somewhere strictly - * *inside* the working directory. `output: "."` otherwise resolves to the - * project root, and the javadoc replace path is an `rm -rf` of that - * destination — so an unbounded `output` plus `overwrite: true` deletes the - * user's project. `..`, an ancestor, and any absolute path outside the working - * directory are refused for the same reason. + * Resolve a project-writing destination and bound it strictly inside the + * working directory. The project root, its parents, and outside absolute paths + * are never valid file outputs. * - * The check is run twice: once lexically, and once on the *resolved* pair, since - * `path.relative` never follows symlinks — `output: "link/docs"` where `link` - * points outside reads as inside while the recursive remove lands elsewhere. The - * final component is deliberately left unresolved: a leaf symlink is unlinked - * rather than followed, so pointing one at a directory elsewhere is harmless. + * Check both lexical and resolved paths. `path.relative` does not follow + * symlinks, so either a parent or an existing destination symlink could + * otherwise redirect a write outside the project. */ async function resolveOutputDest(baseCwd: string, output: string): Promise { const cwd = path.resolve(baseCwd); @@ -244,52 +237,25 @@ async function resolveOutputDest(baseCwd: string, output: string): Promise null))?.isSymbolicLink()) refuse(); + const realDest = await realpathExistingPrefix(dest); + if (!isStrictlyInside(realCwd, realDest)) refuse(); return dest; } -/** - * Files javadoc emits that a hand-written site would not. `index.html` alone is - * not evidence — a static site at `public/` has one too — so a replace needs an - * `index.html` *and* one of these. - */ -const JAVADOC_SIGNATURE_FILES = ["element-list", "help-doc.html", "member-search-index.js"]; - -/** - * `overwrite: true` on javadoc means "replace the docs I generated last time", - * and the replacement is a recursive remove. So the destination has to actually - * look like a docs tree: absent, an empty directory, or a directory that carries - * both an `index.html` and one of javadoc's own scaffolding files. A directory - * full of source, a static site, or a plain file is someone else's data and is - * refused. - */ -async function assertReplaceableDocsDir(dest: string): Promise { +/** A file output may not replace a directory, even when overwrite was authorized. */ +async function assertNotDirectory(dest: string, label = "jar"): Promise { const stat = await fs.stat(dest).catch(() => null); - if (stat === null) return; - if (stat.isDirectory()) { - const entries = await fs.readdir(dest); - if (entries.length === 0) return; - if (entries.includes("index.html") && entries.some(e => JAVADOC_SIGNATURE_FILES.includes(e))) return; - } + if (stat?.isDirectory() !== true) return; throw new RuntimeRpcError( "invalid-params", - `Refusing to replace ${dest} — it does not look like a previous jvm_javadoc output ` + - `(needs index.html plus one of ${JAVADOC_SIGNATURE_FILES.join(", ")}). ` + - "Choose a fresh directory, or the output directory of a previous run.", - { output: dest }, + `Refusing to write the ${label} to ${dest} — it is an existing directory.`, + { + output: dest, + }, ); } -/** A jar is a file: an existing directory in its place is a caller mistake, not an internal failure. */ -async function assertNotDirectory(dest: string): Promise { - const stat = await fs.stat(dest).catch(() => null); - if (stat?.isDirectory() !== true) return; - throw new RuntimeRpcError("invalid-params", `Refusing to write the jar to ${dest} — it is an existing directory.`, { - output: dest, - }); -} - const JAR_CREATE_REQUIREMENTS = "jvm_jar create requires `language`, `code`, and `output` (cwd-relative path for the built jar)."; @@ -320,14 +286,8 @@ export class LocalRuntimeEndpoint implements RuntimeEndpoint { return okResponse(req.id, await this.execJvm(req.params as RuntimeJvmParams, signal)); case "runtime/spawn": return okResponse(req.id, await this.describeSpawn(req.params as RuntimeSpawnParams)); - case "runtime/advice": - return okResponse(req.id, await this.execAdvice(req.params as RuntimeAdviceParams | undefined, signal)); case "runtime/check": - return okResponse(req.id, await this.execBuild(req.params as RuntimeBuildParams, [], signal)); - case "runtime/build": { - const params = req.params as RuntimeBuildParams; - return okResponse(req.id, await this.execBuild(params, params.targets ?? [], signal)); - } + return okResponse(req.id, await this.execCheck(req.params as RuntimeCheckParams, signal)); default: return errorResponse(req.id, new RuntimeRpcError("invalid-params", `Unknown method ${req.method}`)); } @@ -472,11 +432,7 @@ export class LocalRuntimeEndpoint implements RuntimeEndpoint { ? this.jvmJarInspect(params, signal) : this.jvmJarCreate(params, signal); case "deps": - // Truthiness, not presence: an empty `path` must not resolve to the - // working directory and quietly analyze the whole project. - return params.path ? this.jvmDepsFromPath(params, signal) : this.jvmDepsFromSource(params, signal); - case "javadoc": - return this.jvmJavadoc(params, signal); + return this.jvmDeps(params, signal); default: throw new RuntimeRpcError("invalid-params", `Unknown jvm action ${String(params.action)}.`); } @@ -709,10 +665,45 @@ export class LocalRuntimeEndpoint implements RuntimeEndpoint { }); } + private async jvmDeps(params: RuntimeJvmParams, signal?: AbortSignal): Promise { + const dest = params.output ? await resolveOutputDest(this.jvmBaseCwd(params), params.output) : undefined; + if (dest) { + await refuseExistingOutput(dest, params.overwrite); + await assertNotDirectory(dest, "dependency report"); + } + // Truthiness, not presence: an empty `path` must not resolve to the + // working directory and quietly analyze the whole project. + const result = params.path + ? await this.jvmDepsFromPath(params, signal) + : await this.jvmDepsFromSource(params, signal); + if (!dest || result.exitCode !== 0 || result.killed) return result; + await Bun.write(dest, result.stdout); + return { ...result, output: dest }; + } + private async jvmDepsFromPath(params: RuntimeJvmParams, signal?: AbortSignal): Promise { const target = path.resolve(this.jvmBaseCwd(params), params.path ?? ""); if (!(await pathExists(target))) { - throw new RuntimeRpcError("invalid-params", `No class file or jar found at ${target}.`, { path: target }); + throw new RuntimeRpcError( + "invalid-params", + `No JVM source, class, JAR, or class directory found at ${target}.`, + { + path: target, + }, + ); + } + const extension = path.extname(target).toLowerCase(); + if (extension === ".java" || extension === ".kt") { + const code = await Bun.file(target).text(); + return this.jvmDepsFromSource( + { + ...params, + path: undefined, + language: extension === ".java" ? "java" : "kotlin", + code, + }, + signal, + ); } const { binaryPath } = await this.ensureBinary(); const result = await this.jvmInCwd(params, [binaryPath, "jdeps", "--", target], signal); @@ -722,7 +713,7 @@ export class LocalRuntimeEndpoint implements RuntimeEndpoint { private async jvmDepsFromSource(params: RuntimeJvmParams, signal?: AbortSignal): Promise { const { language, code } = this.requireJvmSource( params, - "jvm_deps requires either `path` (existing .class/.jar) or `language` + `code`.", + "jvm_deps requires `path` (source, .class, .jar, or class directory) or `language` + `code`.", ); return this.withJvmWorkdir(params, signal, async (wd, bin, run) => { const { className, failure } = await this.compileJvm(wd, bin, run, "deps", language, code, params.mainClass); @@ -733,35 +724,6 @@ export class LocalRuntimeEndpoint implements RuntimeEndpoint { }); } - private async jvmJavadoc(params: RuntimeJvmParams, signal?: AbortSignal): Promise { - if (params.code === undefined) { - throw new RuntimeRpcError("invalid-params", "jvm_javadoc requires `code` (Java source to document)."); - } - const code = params.code; - const dest = await resolveOutputDest(this.jvmBaseCwd(params), params.output ?? "javadoc-out"); - await refuseExistingOutput(dest, params.overwrite); - if (params.overwrite === true) await assertReplaceableDocsDir(dest); - return this.withJvmWorkdir(params, signal, async (wd, bin, run) => { - const className = deriveJvmMainClass("java", code, params.mainClass); - await wd.write(`${className}.java`, code); - const result = await run([bin, "javadoc", "--", "-d", "apidocs", `${className}.java`]); - const base = { - ...result, - action: "javadoc" as const, - phase: "javadoc" as const, - language: "java" as const, - className, - }; - if (result.exitCode !== 0 || result.killed) return base; - await fs.rm(dest, { recursive: true, force: true }); - await fs.mkdir(path.dirname(dest), { recursive: true }); - await fs.cp(path.join(wd.dir, "apidocs"), dest, { recursive: true }); - const entries = await fs.readdir(dest, { recursive: true }); - const topLevel = (await fs.readdir(dest)).sort().slice(0, 12); - return { ...base, output: dest, entryCount: entries.length, topLevel }; - }); - } - // ── runtime/spawn ──────────────────────────────────────────────────────── // Composition only: resolve the binary, validate the caller's paths, and // return the command line plus the rules for recognizing the endpoint the @@ -791,35 +753,9 @@ export class LocalRuntimeEndpoint implements RuntimeEndpoint { }; } - // ── runtime/advice ─────────────────────────────────────────────────────── - // Read-only, and deliberately the one exec flow with no workdir: the guidance - // is produced by *detecting* `elide.pkl` and package manifests, so it has to - // run in the real project directory — a temp workdir would always report an - // empty project. Nothing is materialized, nothing is written, and the JVM env - // hygiene does not apply because no JVM toolchain is invoked. - - // `params` is optional all the way down: every field is, so a caller with - // nothing to say may legitimately send no params object at all. - private async execAdvice(params: RuntimeAdviceParams | undefined, signal?: AbortSignal): Promise { - const { binaryPath } = await this.ensureBinary(); - return this.spawn( - [binaryPath, "project", "advice", "--error-format=plain", "--no-color"], - { cwd: params?.cwd, timeoutMs: params?.timeoutMs }, - signal, - ); - } - - private async execBuild( - params: RuntimeBuildParams, - targets: string[], - signal?: AbortSignal, - ): Promise { + private async execCheck(params: RuntimeCheckParams, signal?: AbortSignal): Promise { const { binaryPath } = await this.ensureBinary(); - return this.spawn( - [binaryPath, "build", "--no-color", ...targets], - { cwd: params.cwd, timeoutMs: params.timeoutMs }, - signal, - ); + return this.spawn([binaryPath, "build", "--no-color"], { cwd: params.cwd, timeoutMs: params.timeoutMs }, signal); } /** The endpoint's spawn as a {@link RuntimeSpawn}, for workdir-bound flows. */ @@ -924,18 +860,6 @@ const PATH_SHIM_WARNING = "managed install is used — resolution prefers a binary on PATH over auto-downloading, so a wrapper " + "there always wins."; -/** The debug flows' endpoint banners, per wire protocol. */ -const DEBUG_ENDPOINT_RULES: Record = { - // CDP prints a full inspector URL: `Debugger listening on ws://127.0.0.1:9229//inspect`. - // The URL itself is the endpoint, so the whole match is taken. - cdp: [{ pattern: "ws://\\S+" }], - // DAP prints `[Graal DAP] Starting server and listening on /0.0.0.0:4711` — the - // address carries a leading slash (Java's InetSocketAddress formatting for an - // unresolved host). It is consumed by the rule rather than captured: `/0.0.0.0:4711` - // is not an address any DAP client can attach to. - dap: [{ pattern: "listening on\\s+/?(\\S+)", group: 1 }], -}; - /** `serve` prints a bare `host:port`; the scheme is implied. */ const SERVE_ENDPOINT_RULES: RuntimeEndpointRule[] = [ { pattern: "Serving static files on\\s+(\\S+)", group: 1, prefix: "http://" }, @@ -952,17 +876,6 @@ async function statOrNull(target: string): Promise { return await fs.stat(target).catch(() => null); } -async function requireExistingFile(base: string, value: string | undefined, label: string): Promise { - if (!value) throw new RuntimeRpcError("invalid-params", `${label} is required.`); - const resolved = path.resolve(base, value); - const stat = await statOrNull(resolved); - if (stat === null) throw new RuntimeRpcError("invalid-params", `${label} does not exist: ${resolved}`, { resolved }); - if (!stat.isFile()) { - throw new RuntimeRpcError("invalid-params", `${label} is not a file: ${resolved}`, { resolved }); - } - return resolved; -} - async function requireExistingDirectory(base: string, value: string | undefined, label: string): Promise { if (!value) throw new RuntimeRpcError("invalid-params", `${label} is required.`); const resolved = path.resolve(base, value); @@ -979,38 +892,13 @@ async function requireExistingDirectory(base: string, value: string | undefined, * independent of binary resolution so it can run first — see `describeSpawn`. */ async function composeSpawn(params: RuntimeSpawnParams, cwd: string): Promise { - switch (params.mode) { - case "debug": - return debugArgv(params, cwd); - case "serve": - return serveArgv(params, cwd); - default: - throw new RuntimeRpcError("invalid-params", `Unknown spawn mode ${String(params.mode)}.`); - } -} - -/** - * `run --debugger=`. The program runs suspended until a - * client attaches, which is exactly why this is a supervised job rather than a - * request that waits for an exit code. - */ -async function debugArgv(params: RuntimeSpawnParams, cwd: string): Promise { - const file = await requireExistingFile(cwd, params.path, "debug `path` (the program to debug)"); - const protocol: RuntimeDebugProtocol = params.protocol ?? "cdp"; - if (DEBUG_ENDPOINT_RULES[protocol] === undefined) { - throw new RuntimeRpcError("invalid-params", `Unknown debug protocol ${String(protocol)}; expected cdp or dap.`); - } - const language: RuntimeLanguage = params.language ?? inferLanguage(file); - const args = ["run", `--debugger=${protocol}`, "--error-format=plain", "--no-color"]; - if (params.timeoutMs !== undefined) { - if (!Number.isFinite(params.timeoutMs) || params.timeoutMs <= 0) { - throw new RuntimeRpcError("invalid-params", "timeoutMs must be a positive number of milliseconds."); - } - args.push("--timeout", `${Math.round(params.timeoutMs)}ms`); + if ("mode" in params) { + throw new RuntimeRpcError( + "invalid-params", + "runtime/spawn no longer accepts a mode; only static serving remains.", + ); } - args.push("-l", language, file); - if (params.args?.length) args.push("--", ...params.args); - return { args, endpointPattern: DEBUG_ENDPOINT_RULES[protocol] }; + return serveArgv(params, cwd); } /** `serve --no-tui [--port p] [--host h]`. `--no-tui` keeps the output scrapable. */ diff --git a/packages/coding-agent/src/runtime/transport/selected.ts b/packages/coding-agent/src/runtime/transport/selected.ts index 2273c52a99e..f0a4c0053e4 100644 --- a/packages/coding-agent/src/runtime/transport/selected.ts +++ b/packages/coding-agent/src/runtime/transport/selected.ts @@ -88,7 +88,10 @@ export class SelectedRuntimeEndpoint implements RuntimeEndpoint { }, } : request; - const response = await this.#requestElide(routedRequest, signal); + const response = + this.#adapter === "auto" && target.language === "python" + ? await this.#process.request(routedRequest, signal) + : await this.#requestElide(routedRequest, signal); if ("error" in response) return response; return okResponse(request.id, { ...(response.result as Record), diff --git a/packages/coding-agent/src/telemetry/events.ts b/packages/coding-agent/src/telemetry/events.ts index 244b51813c7..92054819983 100644 --- a/packages/coding-agent/src/telemetry/events.ts +++ b/packages/coding-agent/src/telemetry/events.ts @@ -9,6 +9,7 @@ import type { AgentRunCoverage, AgentRunSummary, ChatUsageEvent } from "@oh-my-pi/pi-agent-core"; import type { UsageHistoryEntry } from "@oh-my-pi/pi-ai"; import { logger } from "@oh-my-pi/pi-utils"; +import type { RuntimeErrorCode, RuntimeJvmAction, RuntimeLanguage, RuntimeMethod } from "../runtime/protocol"; export type SessionMode = "tui" | "acp" | "rpc" | "print" | "sdk"; export type CompactionTrigger = "threshold" | "overflow" | "idle" | "incomplete" | "manual"; @@ -23,6 +24,22 @@ export type CompactionStrategy = "context-full" | "handoff" | "shake" | "snapcom export type CompactionOutcome = "ok" | "aborted" | "error" | "will-retry" | "skipped"; export type ErrorPhase = "chat" | "tool" | "compaction" | "session"; +export type RuntimeCallOutcome = "ok" | "error" | "timeout" | "cancelled"; +export type RuntimeCallErrorType = RuntimeErrorCode | "non_zero_exit" | "killed" | "unknown"; + +export interface RuntimeCallCompletedTelemetry { + type: "runtime.call.completed"; + sessionId: string | undefined; + method: RuntimeMethod; + action?: RuntimeJvmAction; + language?: RuntimeLanguage; + outcome: RuntimeCallOutcome; + durationMs: number; + exitCode?: number; + killed?: boolean; + errorType?: RuntimeCallErrorType; +} + export interface SessionStartedTelemetry { type: "session.started"; sessionId: string; @@ -101,6 +118,7 @@ export type TelemetryEvent = | ErrorReportedTelemetry | CompactionCompletedTelemetry | CompactionSavingsTelemetry + | RuntimeCallCompletedTelemetry | UsageLimitSnapshotTelemetry; export type TelemetrySubscriber = (event: TelemetryEvent) => void; diff --git a/packages/coding-agent/src/telemetry/metrics.ts b/packages/coding-agent/src/telemetry/metrics.ts index 38b7b8e6a83..4c188809287 100644 --- a/packages/coding-agent/src/telemetry/metrics.ts +++ b/packages/coding-agent/src/telemetry/metrics.ts @@ -13,6 +13,7 @@ import type { Attributes, Counter, Gauge, Histogram, Meter } from "@opentelemetr import type { CompactionCompletedTelemetry, CompactionSavingsTelemetry, + RuntimeCallCompletedTelemetry, SessionEndedTelemetry, UsageLimitSnapshotTelemetry, } from "./events"; @@ -57,6 +58,8 @@ export class AuraMetricRecorder { readonly #toolCalls: Counter; readonly #toolDurationMs: Histogram; readonly #errors: Counter; + readonly #runtimeCalls: Counter; + readonly #runtimeDurationMs: Histogram; readonly #sessionDuration: Histogram; readonly #sessionTurns: Histogram; readonly #compactions: Counter; @@ -110,6 +113,14 @@ export class AuraMetricRecorder { description: "Errors observed in chat and tool execution.", unit: "{error}", }); + this.#runtimeCalls = meter.createCounter("aura.runtime.calls", { + description: "Completed managed runtime calls.", + unit: "{call}", + }); + this.#runtimeDurationMs = meter.createHistogram("aura.runtime.duration", { + description: "Managed runtime call wall-clock latency.", + unit: "ms", + }); this.#sessionDuration = meter.createHistogram("aura.session.duration", { description: "Wall-clock session duration.", unit: "s", @@ -207,6 +218,17 @@ export class AuraMetricRecorder { } } + recordRuntimeCall(event: RuntimeCallCompletedTelemetry): void { + const attrs = metricAttributes({ + "aura.runtime.method": event.method, + "aura.runtime.action": event.action, + "aura.runtime.language": event.language, + "aura.runtime.outcome": event.outcome, + }); + this.#runtimeCalls.add(1, attrs); + this.#runtimeDurationMs.record(event.durationMs, attrs); + } + recordSessionEnd(event: SessionEndedTelemetry): void { const attrs = metricAttributes({ "aura.session.mode": event.mode, diff --git a/packages/coding-agent/src/telemetry/sink-otlp.ts b/packages/coding-agent/src/telemetry/sink-otlp.ts index b1044582544..260f8313221 100644 --- a/packages/coding-agent/src/telemetry/sink-otlp.ts +++ b/packages/coding-agent/src/telemetry/sink-otlp.ts @@ -68,6 +68,25 @@ function handle(deps: OtlpSinkDeps, event: TelemetryEvent): void { case "chat.usage": deps.recorder?.recordChatUsage(event.event); break; + case "runtime.call.completed": + deps.recorder?.recordRuntimeCall(event); + deps.emitLog( + event.outcome === "ok" ? "info" : "warn", + "runtime call completed", + otelAttributes({ + "session.id": event.sessionId, + "aura.runtime.method": event.method, + "aura.runtime.action": event.action, + "aura.runtime.language": event.language, + "aura.runtime.outcome": event.outcome, + "aura.runtime.duration_ms": event.durationMs, + "aura.runtime.exit_code": event.exitCode, + "aura.runtime.killed": event.killed, + "error.type": event.errorType, + }), + "aura.runtime.call.completed", + ); + break; case "error.reported": deps.recorder?.recordError(event.phase, event.errorType); break; diff --git a/packages/coding-agent/src/tools/builtin-names.ts b/packages/coding-agent/src/tools/builtin-names.ts index 5fa2b87296b..520943bb78b 100644 --- a/packages/coding-agent/src/tools/builtin-names.ts +++ b/packages/coding-agent/src/tools/builtin-names.ts @@ -30,18 +30,13 @@ export const BUILTIN_TOOL_NAMES = [ "manage_skill", "run", "check", - "build", "insights", "profile", - // `runtime_debug`, not `debug`: `debug` is the interactive stepping debugger. - "runtime_debug", "serve", "jvm_disassemble", "jvm_format", "jvm_jar", "jvm_deps", - "jvm_javadoc", - "project_advice", ] as const; export type BuiltinToolName = (typeof BUILTIN_TOOL_NAMES)[number]; diff --git a/packages/coding-agent/src/tools/essential-tools.ts b/packages/coding-agent/src/tools/essential-tools.ts index d032e9c5f17..13bc115fbd6 100644 --- a/packages/coding-agent/src/tools/essential-tools.ts +++ b/packages/coding-agent/src/tools/essential-tools.ts @@ -34,7 +34,6 @@ export const ESSENTIAL_BUILTIN_TOOL_NAMES: Record = { manage_skill: true, run: true, check: true, - build: true, }; /** diff --git a/packages/coding-agent/src/tools/index.ts b/packages/coding-agent/src/tools/index.ts index d9d315f5b5d..61e2ce824b8 100644 --- a/packages/coding-agent/src/tools/index.ts +++ b/packages/coding-agent/src/tools/index.ts @@ -57,7 +57,6 @@ import { JvmDepsTool } from "./jvm-deps"; import { JvmDisassembleTool } from "./jvm-disassemble"; import { JvmFormatTool } from "./jvm-format"; import { JvmJarTool } from "./jvm-jar"; -import { JvmJavadocTool } from "./jvm-javadoc"; import { LearnTool } from "./learn"; import { ManageSkillTool } from "./manage-skill"; import { MemoryEditTool } from "./memory-edit"; @@ -67,10 +66,7 @@ import { MemoryRetainTool } from "./memory-retain"; import { wrapToolWithMetaNotice } from "./output-meta"; import { ReadTool } from "./read"; import type { PlanProposalHandler } from "./resolve"; -import { RuntimeAdviceTool } from "./runtime-advice"; -import { RuntimeBuildTool } from "./runtime-build"; import { RuntimeCheckTool } from "./runtime-check"; -import { RuntimeDebugTool } from "./runtime-debug"; import { RuntimeInsightsTool } from "./runtime-insights"; import { RuntimeProfileTool } from "./runtime-profile"; import { RuntimeRunTool } from "./runtime-run"; @@ -257,7 +253,7 @@ export interface ToolSession { getHindsightSessionState?: () => HindsightSessionState | undefined; /** Get Mnemopi runtime state for this agent session. */ getMnemopiSessionState?: () => MnemopiSessionState | undefined; - /** Aura runtime capability service (run/check/build/insights/profile); undefined when runtime.enabled is off. */ + /** Aura runtime capability service; undefined when runtime.enabled is off. */ getRuntimeService?: () => RuntimeService | undefined; /** Agent identity used for IRC routing. Returns the registry id (e.g. "Main", "AuthLoader"). */ getAgentId?: () => string | null; @@ -453,17 +449,13 @@ export const BUILTIN_TOOLS: Record = { manage_skill: ManageSkillTool.createIf, run: RuntimeRunTool.createIf, check: RuntimeCheckTool.createIf, - build: RuntimeBuildTool.createIf, insights: RuntimeInsightsTool.createIf, profile: RuntimeProfileTool.createIf, - runtime_debug: RuntimeDebugTool.createIf, serve: RuntimeServeTool.createIf, jvm_disassemble: JvmDisassembleTool.createIf, jvm_format: JvmFormatTool.createIf, jvm_jar: JvmJarTool.createIf, jvm_deps: JvmDepsTool.createIf, - jvm_javadoc: JvmJavadocTool.createIf, - project_advice: RuntimeAdviceTool.createIf, }; export const HIDDEN_TOOLS: Record = { diff --git a/packages/coding-agent/src/tools/jvm-deps.ts b/packages/coding-agent/src/tools/jvm-deps.ts index f6a64a453de..c460493408f 100644 --- a/packages/coding-agent/src/tools/jvm-deps.ts +++ b/packages/coding-agent/src/tools/jvm-deps.ts @@ -1,17 +1,19 @@ import type { AgentTool, AgentToolResult } from "@oh-my-pi/pi-agent-core"; import { type } from "arktype"; import jvmDepsDescription from "../prompts/tools/jvm-deps.md" with { type: "text" }; -import { formatExecResult } from "../runtime/format"; +import { execResultFailed, formatExecResult } from "../runtime/format"; import type { RuntimeJvmResult } from "../runtime/protocol"; import type { ToolSession } from "."; import { jvmLanguage, requireRuntimeService } from "./jvm-common"; const jvmDepsSchema = type({ - "language?": jvmLanguage.describe("source language (with code)"), - "code?": type("string").describe("source to compile and analyze"), - "mainClass?": type("string").describe("entrypoint class (default: the public class, or MainKt for Kotlin)"), - "path?": type("string").describe("existing .class, .jar, or class directory to analyze (relative to the cwd)"), - "timeoutMs?": type("number").describe("kill the analysis after this many milliseconds"), + "language?": jvmLanguage.describe("source language"), + "code?": type("string").describe("inline source"), + "mainClass?": type("string").describe("target class override"), + "path?": type("string").describe("source, class, JAR, or class directory"), + "output?": type("string").describe("cwd-relative report output"), + "overwrite?": type("boolean").describe("replace output"), + "timeoutMs?": type("number").describe("timeout (ms)"), }); export type JvmDepsToolParams = typeof jvmDepsSchema.infer; @@ -41,7 +43,15 @@ export class JvmDepsTool implements AgentTool { - readonly name = "jvm_javadoc"; - readonly approval = "exec" as const; - readonly label = "JVM Javadoc"; - readonly description = jvmJavadocDescription; - readonly parameters = jvmJavadocSchema; - readonly strict = true; - readonly loadMode = "discoverable" as const; - readonly summary = "Generate Javadoc HTML API docs from Java source"; - - constructor(private readonly session: ToolSession) {} - - static createIf(session: ToolSession): JvmJavadocTool | null { - if (!session.settings.get("runtime.enabled")) return null; - return new JvmJavadocTool(session); - } - - async execute( - _toolCallId: string, - params: JvmJavadocToolParams, - signal?: AbortSignal, - ): Promise> { - const result = await requireRuntimeService(this.session).jvm( - { action: "javadoc", ...params, cwd: this.session.cwd }, - signal, - ); - if (result.exitCode !== 0 || result.killed) { - return { content: [{ type: "text", text: formatExecResult(result) }], details: result }; - } - const text = - `Generated API docs for ${result.className} → ${result.output} (${result.entryCount} entries).\n` + - `Top-level: ${(result.topLevel ?? []).join(", ")}\n` + - `Tip: open ${result.output}/index.html to browse them.`; - return { content: [{ type: "text", text }], details: result }; - } -} diff --git a/packages/coding-agent/src/tools/renderers.ts b/packages/coding-agent/src/tools/renderers.ts index 4223fd19340..9314a15ed78 100644 --- a/packages/coding-agent/src/tools/renderers.ts +++ b/packages/coding-agent/src/tools/renderers.ts @@ -79,8 +79,8 @@ export type ToolRenderer = { }; export const toolRenderers: Record = { - // Runtime tool family (`run`/`check`/`build`/`insights`/`profile`/`jvm_*`/ - // `runtime_debug`/`serve`/`project_advice`) — one factory, per-tool specs. + // Runtime tool family (`run`/`check`/`insights`/`profile`/`jvm_*`/`serve`) — + // one factory, per-tool specs. ...runtimeToolRenderers, ask: askToolRenderer as ToolRenderer, ast_grep: astGrepToolRenderer as ToolRenderer, diff --git a/packages/coding-agent/src/tools/runtime-advice.ts b/packages/coding-agent/src/tools/runtime-advice.ts deleted file mode 100644 index 5f1b972cf90..00000000000 --- a/packages/coding-agent/src/tools/runtime-advice.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { AgentTool, AgentToolResult } from "@oh-my-pi/pi-agent-core"; -import { type } from "arktype"; -import runtimeAdviceDescription from "../prompts/tools/runtime-advice.md" with { type: "text" }; -import { formatExecResult } from "../runtime/format"; -import type { RuntimeExecResult } from "../runtime/protocol"; -import type { ToolSession } from "."; - -const runtimeAdviceSchema = type({ - "cwd?": type("string").describe("project directory to inspect (defaults to the session cwd)"), - "timeoutMs?": type("number").describe("give up after this many milliseconds"), -}); - -export type RuntimeAdviceToolParams = typeof runtimeAdviceSchema.infer; - -/** - * The runtime's own project guidance. `approval` is `"read"`, not the `"exec"` - * the other runtime tools use: the argv is fixed (`project advice`), the caller - * supplies no code, no arguments and no output path, and the flow only inspects - * manifests in a directory the session can already read. Charging an exec - * approval for a read is what trains users to wave approvals through. - */ -export class RuntimeAdviceTool implements AgentTool { - readonly name = "project_advice"; - readonly approval = "read" as const; - readonly label = "Project Advice"; - readonly description = runtimeAdviceDescription; - readonly parameters = runtimeAdviceSchema; - readonly strict = true; - readonly loadMode = "discoverable" as const; - readonly summary = "Get the runtime's build/run/test/install guidance for this project"; - - constructor(private readonly session: ToolSession) {} - - static createIf(session: ToolSession): RuntimeAdviceTool | null { - if (!session.settings.get("runtime.enabled")) return null; - return new RuntimeAdviceTool(session); - } - - async execute( - _toolCallId: string, - params: RuntimeAdviceToolParams, - signal?: AbortSignal, - ): Promise> { - const service = this.session.getRuntimeService?.(); - if (!service) - throw new Error( - "The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).", - ); - const result = await service.advice({ ...params, cwd: params.cwd ?? this.session.cwd }, signal); - return { - content: [{ type: "text", text: formatExecResult(result) }], - details: result, - }; - } -} diff --git a/packages/coding-agent/src/tools/runtime-build.ts b/packages/coding-agent/src/tools/runtime-build.ts deleted file mode 100644 index b90dae2344f..00000000000 --- a/packages/coding-agent/src/tools/runtime-build.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { AgentTool, AgentToolResult } from "@oh-my-pi/pi-agent-core"; -import { type } from "arktype"; -import runtimeBuildDescription from "../prompts/tools/runtime-build.md" with { type: "text" }; -import { formatExecResult } from "../runtime/format"; -import type { RuntimeExecResult } from "../runtime/protocol"; -import type { ToolSession } from "."; - -const runtimeBuildSchema = type({ - "targets?": type("string[]").describe( - "':'-prefixed build targets with interleaved options, passed through verbatim", - ), - "cwd?": type("string").describe("project directory (defaults to the session cwd)"), - "timeoutMs?": type("number").describe("kill the build after this many milliseconds"), -}); - -export type RuntimeBuildToolParams = typeof runtimeBuildSchema.infer; - -export class RuntimeBuildTool implements AgentTool { - readonly name = "build"; - readonly approval = "exec" as const; - readonly label = "Build"; - readonly description = runtimeBuildDescription; - readonly parameters = runtimeBuildSchema; - readonly strict = true; - readonly loadMode = "essential" as const; - readonly summary = "Assemble project artifacts on the managed runtime"; - - constructor(private readonly session: ToolSession) {} - - static createIf(session: ToolSession): RuntimeBuildTool | null { - if (!session.settings.get("runtime.enabled")) return null; - return new RuntimeBuildTool(session); - } - - async execute( - _toolCallId: string, - params: RuntimeBuildToolParams, - signal?: AbortSignal, - ): Promise> { - const service = this.session.getRuntimeService?.(); - if (!service) - throw new Error( - "The runtime service is unavailable on this session (runtime.enabled may be false, or this host does not provide it).", - ); - const result = await service.build({ ...params, cwd: params.cwd ?? this.session.cwd }, signal); - return { - content: [{ type: "text", text: formatExecResult(result) }], - details: result, - }; - } -} diff --git a/packages/coding-agent/src/tools/runtime-check.ts b/packages/coding-agent/src/tools/runtime-check.ts index 6b803314efa..885b35afdb0 100644 --- a/packages/coding-agent/src/tools/runtime-check.ts +++ b/packages/coding-agent/src/tools/runtime-check.ts @@ -1,13 +1,13 @@ import type { AgentTool, AgentToolResult } from "@oh-my-pi/pi-agent-core"; import { type } from "arktype"; import runtimeCheckDescription from "../prompts/tools/runtime-check.md" with { type: "text" }; -import { formatExecResult } from "../runtime/format"; +import { execResultFailed, formatExecResult } from "../runtime/format"; import type { RuntimeExecResult } from "../runtime/protocol"; import type { ToolSession } from "."; const runtimeCheckSchema = type({ - "cwd?": type("string").describe("project directory (defaults to the session cwd)"), - "timeoutMs?": type("number").describe("kill the validation after this many milliseconds"), + "cwd?": type("string").describe("project directory (session cwd)"), + "timeoutMs?": type("number").describe("timeout (ms)"), }); export type RuntimeCheckToolParams = typeof runtimeCheckSchema.infer; @@ -39,10 +39,15 @@ export class RuntimeCheckTool implements AgentTool = { - cdp: "Open it in Chrome DevTools.", - dap: "Attach a DAP client (for example VS Code).", -}; - -/** - * Publish a debug endpoint for a guest program and hand back the hub job that - * owns it. - * - * Named `runtime_debug` rather than `debug`: `debug` is already the built-in - * interactive stepping debugger this agent drives itself (`DebugTool`), and the - * two are genuinely different tools — that one steps through code on the agent's - * behalf, this one starts a server for an *external* debugger to attach to. - */ -export class RuntimeDebugTool implements AgentTool { - readonly name = "runtime_debug"; - readonly approval = "exec" as const; - readonly label = "Runtime Debug"; - readonly description = runtimeDebugDescription; - readonly parameters = runtimeDebugSchema; - readonly strict = true; - readonly loadMode = "discoverable" as const; - readonly summary = "Start a CDP/DAP debug endpoint for a program, as a hub job"; - - constructor( - private readonly session: ToolSession, - private readonly launch?: LaunchExecutor, - ) {} - - static createIf(session: ToolSession): RuntimeDebugTool | null { - if (!session.settings.get("runtime.enabled")) return null; - return new RuntimeDebugTool(session); - } - - async execute( - _toolCallId: string, - params: RuntimeDebugToolParams, - signal?: AbortSignal, - ): Promise> { - const protocol: RuntimeDebugProtocol = params.protocol ?? "cdp"; - const descriptor = await requireRuntimeService(this.session).spawn( - { - mode: "debug", - path: params.path, - protocol, - language: params.language, - args: params.args, - timeoutMs: params.timeoutMs, - cwd: params.cwd ?? this.session.cwd, - }, - signal, - ); - const waitSeconds = resolveWaitSeconds(params.waitSeconds); - const job = await startRuntimeJob(this.session, descriptor, { - namePrefix: `runtime-debug-${protocol}`, - mode: "debug", - waitSeconds, - signal, - launch: this.launch, - }); - const hubAvailable = hubToolAvailable(this.session); - if (job.failed) return runtimeJobResult(failedLaunchBody(job, hubAvailable), job, descriptor); - const endpoint = job.details.endpoint; - const body = - endpoint === undefined - ? noEndpointReport(`The ${protocol.toUpperCase()} debugger`, job.details, waitSeconds, hubAvailable) - : [ - `${protocol.toUpperCase()} debugger listening at ${endpoint}`, - `${ATTACH_HINT[protocol]} The program is suspended until a client attaches.`, - jobHandleLine(job.details, hubAvailable), - ].join("\n"); - return runtimeJobResult(body, job, descriptor); - } -} diff --git a/packages/coding-agent/src/tools/runtime-insights.ts b/packages/coding-agent/src/tools/runtime-insights.ts index c97ff95e478..2f09ce67dc1 100644 --- a/packages/coding-agent/src/tools/runtime-insights.ts +++ b/packages/coding-agent/src/tools/runtime-insights.ts @@ -1,20 +1,20 @@ import type { AgentTool, AgentToolResult } from "@oh-my-pi/pi-agent-core"; import { type } from "arktype"; import runtimeInsightsDescription from "../prompts/tools/runtime-insights.md" with { type: "text" }; -import { formatExecResult } from "../runtime/format"; +import { execResultFailed, formatExecResult } from "../runtime/format"; import type { RuntimeExecResult } from "../runtime/protocol"; import type { ToolSession } from "."; const runtimeInsightsSchema = type({ - "code?": type("string").describe("inline program source (mutually exclusive with path)"), + "code?": type("string").describe("inline program (exclusive with path)"), "path?": type("string").describe("existing program file"), - "insight?": type("string").describe("inline insight instrumentation script (JavaScript)"), - "insightPath?": type("string").describe("existing insight script path"), - "language?": type("'js' | 'ts' | 'python'").describe("program language (default ts for inline code)"), - "args?": type("string[]").describe("arguments passed to the program"), - "stdin?": type("string").describe("data piped to stdin"), - "timeoutMs?": type("number").describe("kill the run after this many milliseconds"), - "cwd?": type("string").describe("working directory (defaults to the session cwd)"), + "insight?": type("string").describe("inline JS instrumentation"), + "insightPath?": type("string").describe("instrumentation file"), + "language?": type("'js' | 'ts' | 'python'").describe("inline language (default ts)"), + "args?": type("string[]").describe("program arguments"), + "stdin?": type("string").describe("program stdin"), + "timeoutMs?": type("number").describe("timeout (ms)"), + "cwd?": type("string").describe("working directory (session cwd)"), }); export type RuntimeInsightsToolParams = typeof runtimeInsightsSchema.infer; @@ -46,10 +46,15 @@ export class RuntimeInsightsTool implements AgentTool Frame; /** - * Append a `passed` / `failed` verdict to the result line. For the two tools - * whose whole purpose is the verdict (`check`, `build`), an exit code alone - * makes the reader do the translation. + * Append a `passed` / `failed` verdict to the result line. For `check`, + * whose whole purpose is the verdict, an exit code alone makes the reader + * do the translation. */ passFail?: boolean; /** Result details are a hub job descriptor, not an exec result. */ @@ -95,12 +93,6 @@ const num = (args: Args, key: string): number | undefined => { const value = args?.[key]; return typeof value === "number" && Number.isFinite(value) ? value : undefined; }; -const strList = (args: Args, key: string): string[] | undefined => { - const value = args?.[key]; - if (!Array.isArray(value)) return undefined; - const items = value.filter((item): item is string => typeof item === "string" && item.length > 0); - return items.length > 0 ? items : undefined; -}; /** `run`/`insights`/`profile` all take either inline source or an existing file. */ function describeSource(args: Args): string { @@ -292,24 +284,12 @@ function jvmResultFrame(fallback: (args: Args) => Frame) { const written = jvm.output ?? jvm.jar; return { description: (written ? shortenPath(written) : undefined) ?? jvm.className ?? base.description, - meta: [ - ...(base.meta ?? []), - stoppedEarly ? `stopped at ${jvm.phase}` : undefined, - jvm.entryCount !== undefined ? `${jvm.entryCount} entries` : undefined, - ], + meta: [...(base.meta ?? []), stoppedEarly ? `stopped at ${jvm.phase}` : undefined], }; }; } -/** - * `describeResult` for the two hub-backed job tools. - * - * The endpoint takes the description slot because it is what the caller needs - * next — but `mergeCallAndResult` removes the call frame above this row, so the - * launched target has to ride along in `meta` or the transcript stops saying - * *what* is being debugged or served. (`serve` is partly self-documenting via - * the URL's path; `runtime_debug`'s `ws://…` endpoint says nothing at all.) - */ +/** `describeResult` for the supervised static-server tool. */ function jobResultFrame(fallback: (args: Args) => Frame) { return (details: AnyDetails | undefined, args: Args): Frame => { if (!isJobDetails(details)) return fallback(args); @@ -326,11 +306,6 @@ const runCallFrame = (args: Args): Frame => ({ meta: [str(args, "language") ?? (str(args, "path") ? undefined : "ts"), str(args, "engine"), str(args, "mainClass")], }); -const debugCallFrame = (args: Args): Frame => ({ - description: shortenPath(str(args, "path") ?? "?"), - meta: [str(args, "protocol") ?? "cdp"], -}); - const serveCallFrame = (args: Args): Frame => { const port = num(args, "port"); return { @@ -372,12 +347,6 @@ const RUNTIME_RENDERER_SPECS: Record = { }), }, - build: { - title: "Build", - passFail: true, - describeCall: args => ({ description: strList(args, "targets")?.join(" ") ?? "default targets" }), - }, - insights: { title: "Insights", describeCall: args => { @@ -394,11 +363,6 @@ const RUNTIME_RENDERER_SPECS: Record = { describeCall: args => ({ description: describeSource(args), meta: [str(args, "mode")] }), }, - project_advice: { - title: "Project Advice", - describeCall: args => ({ description: path.basename(str(args, "cwd") ?? getProjectDir()) }), - }, - jvm_disassemble: { title: "JVM Disassemble", describeCall: jvmRunCallFrame, @@ -415,19 +379,6 @@ const RUNTIME_RENDERER_SPECS: Record = { jvm_deps: { title: "JVM Deps", describeCall: jvmDepsCallFrame, describeResult: jvmResultFrame(jvmDepsCallFrame) }, - jvm_javadoc: { - title: "JVM Javadoc", - describeCall: args => ({ description: str(args, "output") ?? "javadoc-out" }), - describeResult: jvmResultFrame(args => ({ description: str(args, "output") ?? "javadoc-out" })), - }, - - runtime_debug: { - title: "Runtime Debug", - job: true, - describeCall: debugCallFrame, - describeResult: jobResultFrame(debugCallFrame), - }, - serve: { title: "Serve", job: true, diff --git a/packages/coding-agent/src/tools/runtime-run.ts b/packages/coding-agent/src/tools/runtime-run.ts index 34dd573afa5..1aad51e6b9e 100644 --- a/packages/coding-agent/src/tools/runtime-run.ts +++ b/packages/coding-agent/src/tools/runtime-run.ts @@ -3,22 +3,22 @@ import { logger } from "@oh-my-pi/pi-utils"; import { type } from "arktype"; import runtimeRunDescription from "../prompts/tools/runtime-run.md" with { type: "text" }; import { disposeCachedRuntimeService } from "../runtime"; -import { formatExecResult } from "../runtime/format"; +import { execResultFailed, formatExecResult } from "../runtime/format"; import { type RuntimeExecResult, RuntimeRpcError } from "../runtime/protocol"; import type { ToolSession } from "."; const runtimeRunSchema = type({ - "code?": type("string").describe("inline source to execute (mutually exclusive with path)"), - "path?": type("string").describe("existing file to run; preserves project cwd/imports"), + "code?": type("string").describe("inline source (exclusive with path)"), + "path?": type("string").describe("existing file; keeps project cwd/imports"), "language?": type("'js' | 'ts' | 'python' | 'java' | 'kotlin'").describe( - "language for inline code (default ts; inferred from path)", + "inline language (default ts; inferred from path)", ), - "engine?": type("'bun' | 'elide'").describe("execution engine (js/ts default bun; other languages use elide)"), - "args?": type("string[]").describe("arguments passed to the program"), - "stdin?": type("string").describe("data piped to the program's stdin"), - "timeoutMs?": type("number").describe("kill the run after this many milliseconds"), - "cwd?": type("string").describe("working directory (defaults to the session cwd)"), - "mainClass?": type("string").describe("Java/Kotlin entrypoint class override"), + "engine?": type("'bun' | 'elide'").describe("engine override (JS/TS default bun)"), + "args?": type("string[]").describe("program arguments"), + "stdin?": type("string").describe("program stdin"), + "timeoutMs?": type("number").describe("timeout (ms)"), + "cwd?": type("string").describe("working directory (session cwd)"), + "mainClass?": type("string").describe("JVM entrypoint override"), }); export type RuntimeRunToolParams = typeof runtimeRunSchema.infer; @@ -52,7 +52,11 @@ export class RuntimeRunTool implements AgentTool { readonly name = "serve"; readonly approval = "exec" as const; @@ -58,18 +53,17 @@ export class RuntimeServeTool implements AgentTool> { const descriptor = await requireRuntimeService(this.session).spawn( { - mode: "serve", directory: params.directory, port: params.port, host: params.host, cwd: params.cwd ?? this.session.cwd, }, signal, + this.session.getSessionId?.() ?? undefined, ); const waitSeconds = resolveWaitSeconds(params.waitSeconds); const job = await startRuntimeJob(this.session, descriptor, { namePrefix: "runtime-serve", - mode: "serve", waitSeconds, signal, launch: this.launch, diff --git a/packages/coding-agent/test/discovery/builtin-skills.test.ts b/packages/coding-agent/test/discovery/builtin-skills.test.ts deleted file mode 100644 index 0fca8fc2fb6..00000000000 --- a/packages/coding-agent/test/discovery/builtin-skills.test.ts +++ /dev/null @@ -1,281 +0,0 @@ -/** - * The bundled `builtin-skills` provider ships the runtime skill set embedded in - * the binary. Because every downstream consumer of a skill (the `skill://` - * protocol, `/skill:` invocation, autoload) reads the skill body back off - * disk from `Skill.filePath`, the provider materializes its embedded sources - * into an agent-owned directory before scanning it — these tests defend that - * round-trip, the drift repair, the stale-entry prune, and the priority - * ordering that lets an authored skill of the same name win. - */ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import * as fs from "node:fs"; -import * as os from "node:os"; -import * as path from "node:path"; -import { getCapability } from "@oh-my-pi/pi-coding-agent/capability"; -import { clearCache } from "@oh-my-pi/pi-coding-agent/capability/fs"; -import { BUILTIN_SKILLS_PROVIDER_ID, type Skill, skillCapability } from "@oh-my-pi/pi-coding-agent/capability/skill"; -import type { LoadContext, LoadResult } from "@oh-my-pi/pi-coding-agent/capability/types"; -// Importing discovery registers all providers as a side effect. -import "@oh-my-pi/pi-coding-agent/discovery"; -import { resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; -import { - BUILTIN_SKILL_SOURCES, - getBuiltinSkillsDir, - materializeBuiltinSkills, -} from "@oh-my-pi/pi-coding-agent/discovery/builtin-skills"; -import { loadSkills } from "@oh-my-pi/pi-coding-agent/extensibility/skills"; -import { getConfigRootDir, removeSyncWithRetries, setAgentDir } from "@oh-my-pi/pi-utils"; - -/** The five runtime skills this fork ships. */ -const EXPECTED_SKILL_NAMES = ["insights", "jvm", "profiling", "runtime", "stateful-debugger"]; - -let tempDir: string; -let agentDir: string; - -const originalAgentDirEnv = process.env.PI_CODING_AGENT_DIR; -const fallbackAgentDir = path.join(getConfigRootDir(), "agent"); - -function skillProvider() { - const cap = getCapability(skillCapability.id); - if (!cap) throw new Error("skills capability missing"); - const provider = cap.providers.find(p => p.id === BUILTIN_SKILLS_PROVIDER_ID); - if (!provider) throw new Error("builtin-skills provider missing"); - return { cap, provider }; -} - -async function loadBuiltinSkills(): Promise> { - const { provider } = skillProvider(); - const ctx: LoadContext = { cwd: tempDir, home: tempDir, repoRoot: null }; - return await (provider.load as (ctx: LoadContext) => Promise>)(ctx); -} - -beforeEach(() => { - clearCache(); - tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "aura-builtin-skills-")); - agentDir = path.join(tempDir, ".aura", "agent"); - setAgentDir(agentDir); -}); - -afterEach(() => { - clearCache(); - setAgentDir(originalAgentDirEnv || fallbackAgentDir); - removeSyncWithRetries(tempDir); -}); - -describe("builtin-skills provider", () => { - it("ships exactly the five runtime skills, each with a name and description", async () => { - const { items, warnings } = await loadBuiltinSkills(); - expect(warnings ?? []).toEqual([]); - expect(items.map(s => s.name).sort()).toEqual(EXPECTED_SKILL_NAMES); - for (const skill of items) { - expect(skill._source.provider, skill.name).toBe(BUILTIN_SKILLS_PROVIDER_ID); - expect(skill.frontmatter?.name, skill.name).toBe(skill.name); - expect((skill.frontmatter?.description ?? "").length, skill.name).toBeGreaterThan(20); - expect(skill.content.length, skill.name).toBeGreaterThan(200); - } - }); - - it("materializes each skill to /builtin-skills//SKILL.md so filePath reads back", async () => { - const { items } = await loadBuiltinSkills(); - const dir = getBuiltinSkillsDir(agentDir); - for (const skill of items) { - expect(skill.path).toBe(path.join(dir, skill.name, "SKILL.md")); - // Downstream consumers (skill://, /skill:) re-read the file itself. - const onDisk = fs.readFileSync(skill.path, "utf8"); - const source = BUILTIN_SKILL_SOURCES.find(s => s.name === skill.name); - expect(source, skill.name).toBeDefined(); - expect(onDisk).toBe(source?.content ?? ""); - } - }); - - it("repairs a materialized skill that drifted from the embedded source", async () => { - await loadBuiltinSkills(); - const target = path.join(getBuiltinSkillsDir(agentDir), "runtime", "SKILL.md"); - fs.writeFileSync(target, "---\nname: runtime\ndescription: tampered\n---\n\ngone\n"); - - const { items } = await loadBuiltinSkills(); - const runtime = items.find(s => s.name === "runtime"); - expect(runtime?.frontmatter?.description).not.toBe("tampered"); - expect(fs.readFileSync(target, "utf8")).toBe( - BUILTIN_SKILL_SOURCES.find(s => s.name === "runtime")?.content ?? "", - ); - }); - - it("prunes only skills the manifest claims — a retired one goes, a user-authored one stays", async () => { - await loadBuiltinSkills(); - const dir = getBuiltinSkillsDir(agentDir); - const manifestPath = path.join(dir, ".bundled.json"); - - // Stand in for a previous release that also shipped "retired-skill": the - // manifest claims it, so it is ours to reclaim. - const retired = path.join(dir, "retired-skill"); - fs.mkdirSync(retired, { recursive: true }); - fs.writeFileSync(path.join(retired, "SKILL.md"), "---\nname: retired-skill\ndescription: old\n---\n\nold\n"); - const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as { names: string[] }; - expect(manifest.names.sort()).toEqual(EXPECTED_SKILL_NAMES); - fs.writeFileSync(manifestPath, JSON.stringify({ names: [...manifest.names, "retired-skill"] })); - - // Shape-identical to a bundled skill, but the manifest never claimed it: - // this is a user-authored skill parked in the same root and MUST survive. - const userOwned = path.join(dir, "hand-written"); - fs.mkdirSync(userOwned, { recursive: true }); - fs.writeFileSync(path.join(userOwned, "SKILL.md"), "---\nname: hand-written\ndescription: mine\n---\n\nmine\n"); - - const { items, warnings } = await loadBuiltinSkills(); - expect(fs.existsSync(retired)).toBe(false); - expect(fs.existsSync(path.join(userOwned, "SKILL.md"))).toBe(true); - expect(items.map(s => s.name)).toContain("hand-written"); - expect(items.map(s => s.name)).not.toContain("retired-skill"); - // Deletions are reported, never silent. - expect((warnings ?? []).join("\n")).toInclude('Pruned retired bundled skill "retired-skill"'); - // The manifest no longer claims the retired name. - expect((JSON.parse(fs.readFileSync(manifestPath, "utf8")) as { names: string[] }).names).not.toContain( - "retired-skill", - ); - }); - - it("keeps a retired skill the user has since added files to, and says so", async () => { - await loadBuiltinSkills(); - const dir = getBuiltinSkillsDir(agentDir); - const manifestPath = path.join(dir, ".bundled.json"); - - const retired = path.join(dir, "retired-skill"); - fs.mkdirSync(retired, { recursive: true }); - fs.writeFileSync(path.join(retired, "SKILL.md"), "---\nname: retired-skill\ndescription: old\n---\n\nold\n"); - fs.writeFileSync(path.join(retired, "notes.md"), "keep me"); - const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as { names: string[] }; - fs.writeFileSync(manifestPath, JSON.stringify({ names: [...manifest.names, "retired-skill"] })); - - const { warnings } = await loadBuiltinSkills(); - expect(fs.existsSync(path.join(retired, "notes.md"))).toBe(true); - expect((warnings ?? []).join("\n")).toInclude('Kept retired bundled skill "retired-skill"'); - }); - - it("survives two materializations racing: no truncated SKILL.md is ever left behind", async () => { - const dir = getBuiltinSkillsDir(agentDir); - // Both passes rewrite every file (nothing on disk yet), so they contend on - // the same targets. A shared staging name would let one rename a partial - // write into place under the other. - const [first, second] = await Promise.all([materializeBuiltinSkills(dir), materializeBuiltinSkills(dir)]); - // A shared staging name shows up here first: one pass deletes the other's - // staging file, and the losing rename fails with ENOENT. - expect([...first, ...second]).toEqual([]); - - for (const source of BUILTIN_SKILL_SOURCES) { - expect(fs.readFileSync(path.join(dir, source.name, "SKILL.md"), "utf8"), source.name).toBe(source.content); - // No staging files survive a successful pass. - expect(fs.readdirSync(path.join(dir, source.name)), source.name).toEqual(["SKILL.md"]); - } - }); - - it("is the lowest-priority skill provider so any authored skill of the same name wins", () => { - const { cap, provider } = skillProvider(); - const others = cap.providers.filter(p => p.id !== BUILTIN_SKILLS_PROVIDER_ID); - expect(others.length).toBeGreaterThan(0); - expect(others.every(p => p.priority > provider.priority)).toBe(true); - }); -}); - -describe("settings suppression", () => { - afterEach(() => { - resetSettingsForTest(); - }); - - it("offers nothing and reclaims the tree when runtime.enabled is off", async () => { - // Materialize first with settings uninitialized (the enabled default), then - // turn the runtime off and reload: the whole thing must go. - await loadBuiltinSkills(); - const dir = getBuiltinSkillsDir(agentDir); - expect(fs.existsSync(path.join(dir, "runtime", "SKILL.md"))).toBe(true); - - resetSettingsForTest(); - await Settings.init({ inMemory: true, overrides: { "runtime.enabled": false } }); - - const { items } = await loadBuiltinSkills(); - expect(items).toEqual([]); - expect(fs.existsSync(dir)).toBe(false); - }); - - it("offers nothing when skills.enableBundled is off", async () => { - resetSettingsForTest(); - await Settings.init({ inMemory: true, overrides: { "skills.enableBundled": false } }); - - const { items } = await loadBuiltinSkills(); - expect(items).toEqual([]); - expect(fs.existsSync(getBuiltinSkillsDir(agentDir))).toBe(false); - }); - - it("leaves a user-authored skill (and the directory) alone when suppressed", async () => { - await loadBuiltinSkills(); - const dir = getBuiltinSkillsDir(agentDir); - const userOwned = path.join(dir, "hand-written"); - fs.mkdirSync(userOwned, { recursive: true }); - fs.writeFileSync(path.join(userOwned, "SKILL.md"), "---\nname: hand-written\ndescription: mine\n---\n\nmine\n"); - - resetSettingsForTest(); - await Settings.init({ inMemory: true, overrides: { "runtime.enabled": false } }); - await loadBuiltinSkills(); - - expect(fs.existsSync(path.join(userOwned, "SKILL.md"))).toBe(true); - expect(fs.existsSync(path.join(dir, "runtime"))).toBe(false); - }); - - it("still materializes when the runtime is on and the toggle is left at its default", async () => { - resetSettingsForTest(); - await Settings.init({ inMemory: true }); - - const { items } = await loadBuiltinSkills(); - expect(items.map(s => s.name).sort()).toEqual(EXPECTED_SKILL_NAMES); - }); -}); - -describe("bundled runtime skills through loadSkills", () => { - it("surfaces every bundled skill by default, attributed to the bundled provider", async () => { - const { skills } = await loadSkills({ cwd: tempDir }); - const bundled = skills.filter(skill => skill.source === `${BUILTIN_SKILLS_PROVIDER_ID}:user`); - expect(bundled.map(skill => skill.name).sort()).toEqual(EXPECTED_SKILL_NAMES); - for (const skill of bundled) { - expect(skill.description.length, skill.name).toBeGreaterThan(20); - expect(skill.hide ?? false, skill.name).toBe(false); - } - }); - - it("drops them when skills.enableBundled is off", async () => { - const { skills } = await loadSkills({ cwd: tempDir, enableBundled: false }); - expect(skills.some(skill => skill.source === `${BUILTIN_SKILLS_PROVIDER_ID}:user`)).toBe(false); - }); - - it("drops one named in ignoredSkills", async () => { - const { skills } = await loadSkills({ cwd: tempDir, ignoredSkills: ["profiling"] }); - const bundled = skills.filter(skill => skill.source === `${BUILTIN_SKILLS_PROVIDER_ID}:user`); - expect(bundled.map(skill => skill.name)).not.toContain("profiling"); - expect(bundled.map(skill => skill.name)).toContain("runtime"); - }); -}); - -describe("bundled runtime skill content", () => { - const byName = new Map(BUILTIN_SKILL_SOURCES.map(source => [source.name, source.content])); - - it("never names the vendor runtime — the noun is 'the runtime'", () => { - for (const [name, content] of byName) { - expect(content.toLowerCase(), name).not.toInclude("elide"); - } - }); - - it("teaches the innate tool names", () => { - expect(byName.get("runtime")).toInclude("`run`"); - expect(byName.get("runtime")).toInclude("`check`"); - expect(byName.get("runtime")).toInclude("`build`"); - expect(byName.get("insights")).toInclude("`insights`"); - expect(byName.get("profiling")).toInclude("`profile`"); - expect(byName.get("jvm")).toInclude("`run`"); - expect(byName.get("stateful-debugger")).toInclude("`runtime_debug`"); - }); - - it("keeps the hub-owned lifecycle for runtime_debug and serve (no separate stop tool)", () => { - const debugger_ = byName.get("stateful-debugger") ?? ""; - expect(debugger_).toInclude("hub"); - expect(debugger_).toInclude("no separate stop tool"); - expect(debugger_.toLowerCase()).not.toInclude("stop_runtime_process"); - }); -}); diff --git a/packages/coding-agent/test/discovery/claude-plugins.test.ts b/packages/coding-agent/test/discovery/claude-plugins.test.ts index 945e89096d8..e8c512f12a8 100644 --- a/packages/coding-agent/test/discovery/claude-plugins.test.ts +++ b/packages/coding-agent/test/discovery/claude-plugins.test.ts @@ -396,6 +396,56 @@ describe("listClaudePluginRoots", () => { expect(found).toBeDefined(); expect(found?.path).toContain(path.join(".claude", "skills", "manifest-skill", "SKILL.md")); }); + + test("promotes only canonical Superpowers workflow skills out of discovery", async () => { + const pluginsDir = path.join(tempDir, ".claude", "plugins"); + const superpowersPath = path.join(tempDir, "plugins", "superpowers"); + const authoredPath = path.join(tempDir, "plugins", "authored-workflows"); + await Promise.all([ + fs.mkdir(pluginsDir, { recursive: true }), + fs.mkdir(path.join(superpowersPath, "skills", "using-superpowers"), { recursive: true }), + fs.mkdir(path.join(superpowersPath, "skills", "frontend-design"), { recursive: true }), + fs.mkdir(path.join(authoredPath, "skills", "using-superpowers"), { recursive: true }), + ]); + const entry = (installPath: string) => ({ + scope: "user", + installPath, + version: "1.0.0", + installedAt: "2026-08-01T00:00:00Z", + lastUpdated: "2026-08-01T00:00:00Z", + }); + await fs.writeFile( + path.join(pluginsDir, "installed_plugins.json"), + JSON.stringify({ + version: 2, + plugins: { + "superpowers@superpowers-marketplace": [entry(superpowersPath)], + "authored-workflows@market": [entry(authoredPath)], + }, + }), + ); + await Promise.all([ + fs.writeFile( + path.join(superpowersPath, "skills", "using-superpowers", "SKILL.md"), + "---\nname: using-superpowers\ndescription: Universal dispatcher\n---\nCore workflow.\n", + ), + fs.writeFile( + path.join(superpowersPath, "skills", "frontend-design", "SKILL.md"), + "---\nname: frontend-design\ndescription: Domain design guidance\n---\nDomain skill.\n", + ), + fs.writeFile( + path.join(authoredPath, "skills", "using-superpowers", "SKILL.md"), + "---\nname: using-superpowers\ndescription: Authored override\n---\nAuthored skill.\n", + ), + ]); + + const result = await loadCapability("skills", { cwd: tempDir }); + const coreSkills = result.all.filter(skill => skill.name === "using-superpowers"); + + expect(result.all.find(skill => skill.name === "frontend-design")).toBeDefined(); + expect(coreSkills).toHaveLength(1); + expect(coreSkills[0]?.path).toStartWith(authoredPath); + }); test("keeps plugin skills out of slash commands while loading them as skills", async () => { const pluginsDir = path.join(tempDir, ".claude", "plugins"); const pluginPath = path.join(tempDir, "plugins", "understand-anything"); diff --git a/packages/coding-agent/test/doctor-cli.test.ts b/packages/coding-agent/test/doctor-cli.test.ts index a44ce218f6c..be5366a0a63 100644 --- a/packages/coding-agent/test/doctor-cli.test.ts +++ b/packages/coding-agent/test/doctor-cli.test.ts @@ -506,14 +506,12 @@ describe("resolveToolGating", () => { const RUNTIME_TOOLS = [ "run", "check", - "build", "insights", "profile", "jvm_disassemble", "jvm_format", "jvm_jar", "jvm_deps", - "jvm_javadoc", ]; test("everything on registers every name", () => { diff --git a/packages/coding-agent/test/doctor-tool-gate-drift.test.ts b/packages/coding-agent/test/doctor-tool-gate-drift.test.ts index f47bfb4cf3a..a21d87a91df 100644 --- a/packages/coding-agent/test/doctor-tool-gate-drift.test.ts +++ b/packages/coding-agent/test/doctor-tool-gate-drift.test.ts @@ -127,20 +127,14 @@ describe("doctor's tool-gate table matches the real registry", () => { const dropped = [...on].filter(name => !off.has(name)).sort(); expect(dropped).toEqual( [ - "build", "check", "insights", "jvm_deps", "jvm_disassemble", "jvm_format", "jvm_jar", - "jvm_javadoc", "profile", - // Read-only, but registered on the same gate: no runtime, no advice. - "project_advice", "run", - // The two long-running flows ride the same gate. - "runtime_debug", "serve", ].sort(), ); diff --git a/packages/coding-agent/test/jvm-tools.test.ts b/packages/coding-agent/test/jvm-tools.test.ts index 89454f43209..4f75dd875d2 100644 --- a/packages/coding-agent/test/jvm-tools.test.ts +++ b/packages/coding-agent/test/jvm-tools.test.ts @@ -5,7 +5,6 @@ import { JvmDepsTool } from "../src/tools/jvm-deps"; import { JvmDisassembleTool } from "../src/tools/jvm-disassemble"; import { JvmFormatTool } from "../src/tools/jvm-format"; import { JvmJarTool } from "../src/tools/jvm-jar"; -import { JvmJavadocTool } from "../src/tools/jvm-javadoc"; const SESSION_CWD = "/work/project"; @@ -161,44 +160,30 @@ describe("jvm_deps", () => { expect(seen()).toEqual({ action: "deps", path: "out/Main.class", cwd: SESSION_CWD }); expect(textOf(out)).toBe("Main.class -> java.base"); }); -}); -describe("jvm_javadoc", () => { - test("reports the written tree, its size, and how to browse it", async () => { + test("source path and output are forwarded without manual compilation or writing", async () => { const { session, seen } = sessionReturning( ok({ - action: "javadoc", - phase: "javadoc", - className: "Widget", - output: "/work/project/apidocs", - entryCount: 42, - topLevel: ["index.html", "Widget.html"], + action: "deps", + phase: "deps", + stdout: "Report.class -> java.sql\n", + output: "/work/project/deps.txt", }), ); - const out = await JvmJavadocTool.createIf(session)!.execute( + const out = await JvmDepsTool.createIf(session)!.execute( "id", - { code: "public class Widget {}", output: "apidocs" }, + { path: "Report.java", output: "deps.txt", overwrite: true }, SIGNAL, ); expect(seen()).toEqual({ - action: "javadoc", - code: "public class Widget {}", - output: "apidocs", + action: "deps", + path: "Report.java", + output: "deps.txt", + overwrite: true, cwd: SESSION_CWD, }); - expect(textOf(out)).toBe( - "Generated API docs for Widget → /work/project/apidocs (42 entries).\n" + - "Top-level: index.html, Widget.html\n" + - "Tip: open /work/project/apidocs/index.html to browse them.", - ); - }); - - test("a javadoc failure renders the generator's output", async () => { - const { session } = sessionReturning( - ok({ action: "javadoc", phase: "javadoc", exitCode: 1, stderr: "error: bad @link" }), - ); - const out = await JvmJavadocTool.createIf(session)!.execute("id", { code: "class X {}" }, SIGNAL); - expect(textOf(out)).toContain("error: bad @link"); + expect(textOf(out)).toContain("Wrote dependency report to /work/project/deps.txt"); + expect(textOf(out)).toContain("Report.class -> java.sql"); }); }); @@ -214,7 +199,6 @@ describe("JVM tools without a runtime service", () => { JvmFormatTool.createIf(session)!, JvmJarTool.createIf(session)!, JvmDepsTool.createIf(session)!, - JvmJavadocTool.createIf(session)!, ]; for (const tool of tools) { await expect( diff --git a/packages/coding-agent/test/runtime-advice-tool.test.ts b/packages/coding-agent/test/runtime-advice-tool.test.ts deleted file mode 100644 index f7cfe7f7043..00000000000 --- a/packages/coding-agent/test/runtime-advice-tool.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import type { RuntimeExecResult } from "../src/runtime/protocol"; -import type { ToolSession } from "../src/tools"; -import { RuntimeAdviceTool } from "../src/tools/runtime-advice"; - -const REPORT: RuntimeExecResult = { - exitCode: 0, - stdout: "## Project Advice\n\n- Project Name: `probe`\n", - stderr: "", - durationMs: 12, - killed: false, -}; - -function sessionWith(service: object | undefined, enabled = true, cwd = "/session/cwd"): ToolSession { - return { - cwd, - settings: { get: (key: string) => (key === "runtime.enabled" ? enabled : undefined) }, - getRuntimeService: () => service as never, - } as unknown as ToolSession; -} - -describe("project_advice tool", () => { - test("is discoverable, gated on runtime.enabled, and read-approved", () => { - expect(RuntimeAdviceTool.createIf(sessionWith(undefined, false))).toBeNull(); - const tool = RuntimeAdviceTool.createIf(sessionWith({})); - expect(tool).not.toBeNull(); - expect(tool!.name).toBe("project_advice"); - expect(tool!.loadMode).toBe("discoverable"); - // Read, not exec: fixed argv, no caller-supplied code, arguments, or output path. - expect(tool!.approval).toBe("read"); - }); - - test("never names the runtime product in model- or user-facing text", () => { - const tool = RuntimeAdviceTool.createIf(sessionWith({})); - const text = `${tool!.description} ${tool!.summary} ${tool!.label}`; - expect(text.toLowerCase()).not.toContain("elide"); - expect(text).toContain("runtime"); - }); - - test("defaults cwd to the session directory and forwards timeoutMs", async () => { - let received: unknown; - const tool = RuntimeAdviceTool.createIf( - sessionWith({ - advice: async (p: unknown) => { - received = p; - return REPORT; - }, - }), - ); - const r = await tool!.execute("id", { timeoutMs: 5_000 }, new AbortController().signal); - expect(received).toEqual({ cwd: "/session/cwd", timeoutMs: 5_000 }); - expect((r.content[0] as { text: string }).text).toContain("Project Advice"); - expect(r.details).toMatchObject({ exitCode: 0 }); - }); - - test("an explicit cwd wins over the session directory", async () => { - let received: unknown; - const tool = RuntimeAdviceTool.createIf( - sessionWith({ - advice: async (p: unknown) => { - received = p; - return REPORT; - }, - }), - ); - await tool!.execute("id", { cwd: "/elsewhere" }, new AbortController().signal); - expect(received).toEqual({ cwd: "/elsewhere" }); - }); - - test("every param is optional — an empty object validates", () => { - const tool = RuntimeAdviceTool.createIf(sessionWith({})); - expect(tool!.parameters.allows({})).toBe(true); - }); - - test("the runtime's own failure is surfaced, not reinterpreted", async () => { - const tool = RuntimeAdviceTool.createIf( - sessionWith({ - advice: async () => ({ - exitCode: 1, - stdout: "", - stderr: "advice is unavailable on this build", - durationMs: 4, - killed: false, - }), - }), - ); - const r = await tool!.execute("id", {}, new AbortController().signal); - const text = (r.content[0] as { text: string }).text; - expect(text).toContain("advice is unavailable on this build"); - expect(text).toContain("(exit code 1)"); - }); - - test("throws when the runtime service is unavailable", async () => { - const tool = RuntimeAdviceTool.createIf(sessionWith(undefined)); - await expect(tool!.execute("id", {}, new AbortController().signal)).rejects.toThrow( - /The runtime service is unavailable on this session/, - ); - }); -}); diff --git a/packages/coding-agent/test/runtime-check-build-tools.test.ts b/packages/coding-agent/test/runtime-check-tool.test.ts similarity index 67% rename from packages/coding-agent/test/runtime-check-build-tools.test.ts rename to packages/coding-agent/test/runtime-check-tool.test.ts index 93d2ea98b24..c8edb86e0b5 100644 --- a/packages/coding-agent/test/runtime-check-build-tools.test.ts +++ b/packages/coding-agent/test/runtime-check-tool.test.ts @@ -1,7 +1,6 @@ import { describe, expect, test } from "bun:test"; import type { RuntimeExecResult } from "../src/runtime/protocol"; import type { ToolSession } from "../src/tools"; -import { RuntimeBuildTool } from "../src/tools/runtime-build"; import { RuntimeCheckTool } from "../src/tools/runtime-check"; const OK: RuntimeExecResult = { exitCode: 0, stdout: "Build successful", stderr: "", durationMs: 10, killed: false }; @@ -13,21 +12,16 @@ function sessionWith(service: object | undefined, enabled = true): ToolSession { } as unknown as ToolSession; } -describe("check/build tools", () => { +describe("check tool", () => { test("createIf gates on runtime.enabled", () => { expect(RuntimeCheckTool.createIf(sessionWith(undefined, false))).toBeNull(); - expect(RuntimeBuildTool.createIf(sessionWith(undefined, false))).toBeNull(); expect(RuntimeCheckTool.createIf(sessionWith({}))?.name).toBe("check"); - expect(RuntimeBuildTool.createIf(sessionWith({}))?.name).toBe("build"); }); - test("check/build advertise essential load mode and exec approval", () => { + test("advertises essential load mode and exec approval", () => { const check = RuntimeCheckTool.createIf(sessionWith({})); expect(check?.loadMode).toBe("essential"); expect(check?.approval).toBe("exec"); - const build = RuntimeBuildTool.createIf(sessionWith({})); - expect(build?.loadMode).toBe("essential"); - expect(build?.approval).toBe("exec"); }); test("check calls service.check with no targets", async () => { @@ -47,21 +41,6 @@ describe("check/build tools", () => { expect(r.details).toMatchObject({ exitCode: 0 }); }); - test("build passes targets through", async () => { - let received: unknown; - const tool = RuntimeBuildTool.createIf( - sessionWith({ - build: async (p: unknown) => { - received = p; - return OK; - }, - }), - ); - expect(tool).not.toBeNull(); - await tool!.execute("id", { targets: [":deps", "--fresh"] } as never, new AbortController().signal); - expect(received).toMatchObject({ targets: [":deps", "--fresh"] }); - }); - test("execute throws when the runtime service is unavailable", async () => { const tool = RuntimeCheckTool.createIf(sessionWith(undefined)); expect(tool).not.toBeNull(); diff --git a/packages/coding-agent/test/runtime-embedded-endpoint.test.ts b/packages/coding-agent/test/runtime-embedded-endpoint.test.ts index 2a15da92fff..fabb116c85a 100644 --- a/packages/coding-agent/test/runtime-embedded-endpoint.test.ts +++ b/packages/coding-agent/test/runtime-embedded-endpoint.test.ts @@ -332,6 +332,20 @@ describe("SelectedRuntimeEndpoint routing", () => { params: { code: "1", language: "js", engine: "elide" }, expected: "embedded", }, + { + name: "auto Python prefers process", + adapter: "auto", + embeddedStatus: validEmbeddedStatus, + params: { code: "print(1)", language: "python" }, + expected: "process", + }, + { + name: "auto inferred Python file prefers process", + adapter: "auto", + embeddedStatus: validEmbeddedStatus, + params: { path: "main.py" }, + expected: "process", + }, { name: "auto JVM with valid library", adapter: "auto", @@ -622,8 +636,9 @@ describe("SelectedRuntimeEndpoint routing", () => { const environment: NodeJS.ProcessEnv = { AUTO_SNAPSHOT: "original-env" }; const args = ["original-arg"]; const params: Record = { - code: "print('original-source')", - language: "python", + code: "console.log('original-source')", + language: "js", + engine: "elide", args, stdin: "original-stdin", timeoutMs: 10_000, @@ -658,8 +673,8 @@ describe("SelectedRuntimeEndpoint routing", () => { expect(unwrapResponse<{ exitCode: number }>(await pending).exitCode).toBe(0); const call = new Message(host.callRequests[0], false).getRoot(EmbeddedCallRequest); const run = call.invocation.invocation.cli.command.run; - expect(run.sourceLanguage).toBe(EngineInvocation_CliInvocation_SourceLanguage.PYTHON); - expect(run.sourceCode.code).toBe("print('original-source')"); + expect(run.sourceLanguage).toBe(EngineInvocation_CliInvocation_SourceLanguage.JAVASCRIPT); + expect(run.sourceCode.code).toBe("console.log('original-source')"); expect(run.scriptArgs.count).toBe(1); expect(call.invocation.args.args.list.get(1).key).toBe("original-arg"); expect(call.invocation.meta.engineConfig.directories.workingDir.path.pathString.path).toBe(originalCwd); @@ -852,7 +867,7 @@ describe("SelectedRuntimeEndpoint routing", () => { } }); - test("routes every non-run method through the process endpoint", async () => { + test("routes every executable non-run method through the process endpoint", async () => { const processEndpoint = new StubEndpoint("process", processStatus); const embedded = new StubEndpoint("embedded", validEmbeddedStatus); const endpoint = new SelectedRuntimeEndpoint({ @@ -860,13 +875,21 @@ describe("SelectedRuntimeEndpoint routing", () => { processEndpoint, embeddedEndpoint: embedded, }); - await endpoint.request(rpcRequest(1, "runtime/check", {})); - await endpoint.request(rpcRequest(2, "runtime/build", {})); - await endpoint.request(rpcRequest(3, "runtime/advice", {})); + for (const [id, method] of [ + [1, "runtime/check"], + [2, "runtime/insights"], + [3, "runtime/profile"], + [4, "runtime/jvm"], + [5, "runtime/spawn"], + ] as const) { + await endpoint.request(rpcRequest(id, method, {})); + } expect(processEndpoint.requests.map(request => request.method)).toEqual([ "runtime/check", - "runtime/build", - "runtime/advice", + "runtime/insights", + "runtime/profile", + "runtime/jvm", + "runtime/spawn", ]); expect(embedded.requests).toHaveLength(0); }); diff --git a/packages/coding-agent/test/runtime-integration.test.ts b/packages/coding-agent/test/runtime-integration.test.ts index 2f160043478..c26ca0ea6a7 100644 --- a/packages/coding-agent/test/runtime-integration.test.ts +++ b/packages/coding-agent/test/runtime-integration.test.ts @@ -158,100 +158,6 @@ describe.skipIf(!realBin)("runtime integration (real binary)", () => { * handle and can guarantee it never leaks — the hub path is covered * separately, without a real runtime. */ - /** - * Both debug protocols, live. This is the bug class the mocked tests missed: - * 1.4.2's DAP banner is `listening on /0.0.0.0:4711` (Java's socket-address - * formatting), and the leading slash has to be dropped or the "endpoint" is - * not attachable. The program suspends waiting for a client, so the process is - * killed unconditionally rather than waited on. - */ - describe("debug launch descriptors", () => { - const scrapeDebugEndpoint = async (protocol: "cdp" | "dap"): Promise => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "aura-debug-live-")); - const guest = path.join(dir, "app.ts"); - await fs.writeFile(guest, "console.log('debug me')\n"); - const descriptor = await svc.spawn({ mode: "debug", path: guest, protocol, cwd: dir }); - expect(descriptor.argv.slice(1, 3)).toEqual(["run", `--debugger=${protocol}`]); - const proc = Bun.spawn(descriptor.argv, { - cwd: descriptor.cwd, - env: { ...process.env, ...descriptor.env }, - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - }); - try { - let output = ""; - let endpoint: string | undefined; - const scan = async (stream: ReadableStream): Promise => { - const decoder = new TextDecoder(); - for await (const chunk of stream) { - output += decoder.decode(chunk, { stream: true }); - endpoint ??= matchRuntimeEndpoint(output, descriptor.endpointPattern); - if (endpoint !== undefined) return; - } - }; - await Promise.race([scan(proc.stdout), scan(proc.stderr), Bun.sleep(60_000)]); - expect(output, "the debugger printed nothing").not.toBe(""); - return endpoint; - } finally { - proc.kill("SIGTERM"); - const exited = await Promise.race([proc.exited.then(() => true), Bun.sleep(2_000).then(() => false)]); - if (!exited) proc.kill("SIGKILL"); - await proc.exited; - await fs.rm(dir, { recursive: true, force: true }); - } - }; - - test("cdp reports an attachable ws:// inspector URL", async () => { - expect(await scrapeDebugEndpoint("cdp")).toMatch(/^ws:\/\/[^/\s]+:\d+\/\S*$/); - }, 120_000); - - test("dap reports a bare host:port, with no leading slash", async () => { - const endpoint = await scrapeDebugEndpoint("dap"); - expect(endpoint).toMatch(/^[^/\s]+:\d+$/); - expect(endpoint?.startsWith("/")).toBe(false); - }, 120_000); - }); - - /** - * `project advice` is the runtime's own feature, and BUCKSHOT recorded it - * crashing outright on 1.4.0-nightly.20260712. It works cleanly on the pinned - * 1.4.x line (verified on 1.4.2+20260720), so this runs unguarded beyond the - * usual real-binary skip — a crash here should fail the suite loudly rather - * than be skipped away, because the tool's whole value is this output. - */ - describe("project advice", () => { - test("reports a detected project from the real directory, with no ANSI escapes", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "aura-advice-live-")); - try { - await fs.writeFile( - path.join(dir, "elide.pkl"), - 'amends "elide:project.pkl"\n\nname = "adviceprobe"\nversion = "9.8.7"\n', - ); - const r = await svc.advice({ cwd: dir, timeoutMs: 120_000 }); - expect(r.exitCode).toBe(0); - const report = `${r.stdout}${r.stderr}`; - // It read *this* directory's manifest, which is why the flow has no workdir. - expect(report).toContain("adviceprobe"); - expect(report).toContain("9.8.7"); - // `--no-color` plus `NO_COLOR=1` is why no escape-stripping pass exists. - expect(report).not.toContain("\u001b["); - } finally { - await fs.rm(dir, { recursive: true, force: true }); - } - }, 180_000); - - test("a directory with no project still yields the runtime's command guidance", async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "aura-advice-bare-")); - try { - const r = await svc.advice({ cwd: dir, timeoutMs: 120_000 }); - expect(r.exitCode).toBe(0); - expect(`${r.stdout}${r.stderr}`.length).toBeGreaterThan(0); - } finally { - await fs.rm(dir, { recursive: true, force: true }); - } - }, 180_000); - }); describe("serve launch descriptor", () => { test("the composed argv serves a directory and prints a scrapable endpoint", async () => { @@ -260,7 +166,7 @@ describe.skipIf(!realBin)("runtime integration (real binary)", () => { // A high, unlikely-to-be-taken port; a collision shows up as a failed // scrape with the runtime's own message, not as a hang. const port = 41_000 + Math.floor(Math.random() * 2_000); - const descriptor = await svc.spawn({ mode: "serve", directory: dir, port, host: "127.0.0.1", cwd: dir }); + const descriptor = await svc.spawn({ directory: dir, port, host: "127.0.0.1", cwd: dir }); expect(descriptor.argv.slice(1)).toEqual([ "serve", dir, diff --git a/packages/coding-agent/test/runtime-jvm-endpoint.test.ts b/packages/coding-agent/test/runtime-jvm-endpoint.test.ts index e0569147b62..3d1ddc8b952 100644 --- a/packages/coding-agent/test/runtime-jvm-endpoint.test.ts +++ b/packages/coding-agent/test/runtime-jvm-endpoint.test.ts @@ -34,7 +34,6 @@ if [ "$1" = "java" ]; then IFS= read -r input || true; printf 'STDIN:%s\\n' "$in if [ "$1" = "jar" ]; then case "$*" in *--create*) : > aura-out.jar ;; esac fi -if [ "$1" = "javadoc" ]; then mkdir -p apidocs && echo "" > apidocs/index.html && echo "unnamed" > apidocs/element-list; fi echo "ARGS:$*" `; @@ -206,6 +205,26 @@ describe("runtime/jvm — argv per action", () => { expect(r.phase).toBe("deps"); expect(await invocations()).toEqual(["javac -- --release 17 Main.java", "jdeps -- Main.class"]); }); + test("deps accepts a Java source path and compiles it in scratch space", async () => { + const source = path.join(dir, "SourceDeps.java"); + await fs.writeFile(source, "public class SourceDeps {}"); + const r = await jvm({ action: "deps", path: "SourceDeps.java", cwd: dir }); + expect(r).toMatchObject({ phase: "deps", language: "java", className: "SourceDeps" }); + expect(await invocations()).toEqual(["javac -- --release 17 SourceDeps.java", "jdeps -- SourceDeps.class"]); + }); + + test("deps writes its successful report to a guarded project output", async () => { + const output = path.join(dir, "deps-report.txt"); + const r = await jvm({ + action: "deps", + language: "java", + code: JAVA_HELLO, + output: "deps-report.txt", + cwd: dir, + }); + expect(r.output).toBe(output); + expect(await fs.readFile(output, "utf8")).toBe("ARGS:jdeps -- Main.class\n"); + }); test("deps analyzes out/ in Kotlin source mode", async () => { await jvm({ action: "deps", language: "kotlin", code: "fun main() {}" }); @@ -219,6 +238,20 @@ describe("runtime/jvm — argv per action", () => { expect(r.phase).toBe("deps"); expect(await invocations()).toEqual([`jdeps -- ${target}`]); }); + test("deps refuses to replace an existing report without overwrite", async () => { + const output = path.join(dir, "existing-deps.txt"); + await fs.writeFile(output, "keep"); + const err = await jvmError({ + action: "deps", + language: "java", + code: JAVA_HELLO, + output: "existing-deps.txt", + cwd: dir, + }); + expect(err.message).toBe(`Refusing to overwrite ${output} — pass overwrite: true to replace it.`); + expect(await fs.readFile(output, "utf8")).toBe("keep"); + expect(await invocations()).toEqual([]); + }); }); describe("runtime/jvm — compile failures", () => { @@ -317,58 +350,11 @@ describe("runtime/jvm — jar", () => { expect(await invocations()).toEqual([`jar -- --list --file ${jarPath}`]); }); - test("deps of a missing artifact names the resolved path", async () => { + test("deps of a missing input names the resolved path", async () => { const err = await jvmError({ action: "deps", path: "ghost.class", cwd: dir }); - expect(err.message).toBe(`No class file or jar found at ${path.join(dir, "ghost.class")}.`); - }); -}); - -describe("runtime/jvm — javadoc", () => { - test("generates into apidocs and copies the tree to the requested output", async () => { - const dest = path.join(dir, "docs-out"); - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, output: "docs-out", cwd: dir }); - expect(await invocations()).toEqual(["javadoc -- -d apidocs Main.java"]); - expect(r.output).toBe(dest); - expect(r.className).toBe("Main"); - expect(r.entryCount).toBe(2); - expect(r.topLevel).toEqual(["element-list", "index.html"]); - expect(await fs.readFile(path.join(dest, "index.html"), "utf8")).toContain(""); - await fs.rm(dest, { recursive: true, force: true }); - }); - - test("defaults the output directory to javadoc-out", async () => { - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, cwd: dir }); - expect(r.output).toBe(path.join(dir, "javadoc-out")); - await fs.rm(path.join(dir, "javadoc-out"), { recursive: true, force: true }); - }); - - test("refuses an existing output directory and names the flag", async () => { - const dest = path.join(dir, "existing-docs"); - await fs.mkdir(dest, { recursive: true }); - await fs.writeFile(path.join(dest, "keep.txt"), "keep"); - const err = await jvmError({ action: "javadoc", code: JAVA_HELLO, output: "existing-docs", cwd: dir }); - expect(err.code).toBe("invalid-params"); - expect(err.message).toBe(`Refusing to overwrite ${dest} — pass overwrite: true to replace it.`); - expect(await invocations()).toEqual([]); - expect(await fs.readFile(path.join(dest, "keep.txt"), "utf8")).toBe("keep"); - }); - - test("overwrite true replaces a previous output directory wholesale", async () => { - const dest = path.join(dir, "stale-docs"); - await fs.mkdir(dest, { recursive: true }); - // Shaped like a previous run — that is what `overwrite` is allowed to replace. - await fs.writeFile(path.join(dest, "index.html"), "old"); - await fs.writeFile(path.join(dest, "help-doc.html"), "old"); - await fs.writeFile(path.join(dest, "stale.html"), "old"); - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, output: "stale-docs", overwrite: true, cwd: dir }); - expect(r.topLevel).toEqual(["element-list", "index.html"]); - expect((await fs.readdir(dest)).sort()).toEqual(["element-list", "index.html"]); - await fs.rm(dest, { recursive: true, force: true }); - }); - - test("javadoc without code is invalid-params", async () => { - const err = await jvmError({ action: "javadoc" }); - expect(err.message).toBe("jvm_javadoc requires `code` (Java source to document)."); + expect(err.message).toBe( + `No JVM source, class, JAR, or class directory found at ${path.join(dir, "ghost.class")}.`, + ); }); }); @@ -385,97 +371,26 @@ describe("runtime/jvm — output paths are bound to the working directory", () = /** Every path an existing project must survive, whatever `overwrite` says. */ const OUT_OF_BOUNDS = [".", "..", "./", path.join("..", "sibling"), path.join("src", "..")]; - test("javadoc refuses an output that is the working directory or above it, even with overwrite", async () => { + test("deps refuses an output that is the working directory or above it, even with overwrite", async () => { for (const output of OUT_OF_BOUNDS) { const root = await project(); - const err = await jvmError({ action: "javadoc", code: JAVA_HELLO, output, overwrite: true, cwd: root }); + const err = await jvmError({ + action: "deps", + language: "java", + code: JAVA_HELLO, + output, + overwrite: true, + cwd: root, + }); expect(err.code, `expected ${output} to be refused`).toBe("invalid-params"); expect(err.message).toContain("output must be a path inside the working directory"); - // Nothing was spawned and nothing was removed. expect(await invocations()).toEqual([]); expect(await fs.readFile(path.join(root, "README.md"), "utf8")).toBe("keep me"); expect(await fs.readdir(root)).toContain("src"); } }); - test("javadoc overwrite refuses a directory that is not a previous docs output", async () => { - const root = await project(); - const err = await jvmError({ action: "javadoc", code: JAVA_HELLO, output: "src", overwrite: true, cwd: root }); - expect(err.code).toBe("invalid-params"); - expect(err.message).toBe( - `Refusing to replace ${path.join(root, "src")} — it does not look like a previous jvm_javadoc output ` + - "(needs index.html plus one of element-list, help-doc.html, member-search-index.js). " + - "Choose a fresh directory, or the output directory of a previous run.", - ); - expect(await invocations()).toEqual([]); - expect(await fs.readFile(path.join(root, "src", "Thing.java"), "utf8")).toBe("class Thing {}"); - }); - - test("javadoc overwrite refuses a static site that merely has an index.html", async () => { - const root = await project(); - const dest = path.join(root, "public"); - await fs.mkdir(dest, { recursive: true }); - await fs.writeFile(path.join(dest, "index.html"), ""); - await fs.writeFile(path.join(dest, "app.js"), "console.log(1)"); - const err = await jvmError({ action: "javadoc", code: JAVA_HELLO, output: "public", overwrite: true, cwd: root }); - expect(err.code).toBe("invalid-params"); - expect(err.message).toContain("does not look like a previous jvm_javadoc output"); - expect(await invocations()).toEqual([]); - expect(await fs.readFile(path.join(dest, "app.js"), "utf8")).toBe("console.log(1)"); - }); - - test("javadoc overwrite replaces a previous docs output", async () => { - const root = await project(); - const dest = path.join(root, "apidocs"); - await fs.mkdir(dest, { recursive: true }); - await fs.writeFile(path.join(dest, "index.html"), "stale"); - await fs.writeFile(path.join(dest, "element-list"), "stale"); - await fs.writeFile(path.join(dest, "Stale.html"), "stale"); - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, output: "apidocs", overwrite: true, cwd: root }); - expect(r.output).toBe(dest); - expect((await fs.readdir(dest)).sort()).toEqual(["element-list", "index.html"]); - expect(await fs.readFile(path.join(dest, "index.html"), "utf8")).toContain(""); - }); - - test("a directory named ..docs is inside the project and is not mistaken for a parent escape", async () => { - const root = await project(); - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, output: "..docs", cwd: root }); - expect(r.output).toBe(path.join(root, "..docs")); - expect(await fs.readFile(path.join(root, "README.md"), "utf8")).toBe("keep me"); - }); - - test("javadoc overwrite accepts an existing empty directory", async () => { - const root = await project(); - await fs.mkdir(path.join(root, "empty-docs"), { recursive: true }); - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, output: "empty-docs", overwrite: true, cwd: root }); - expect(r.entryCount).toBe(2); - }); - - test("javadoc overwrite refuses a plain file standing where the docs would go", async () => { - const root = await project(); - const err = await jvmError({ - action: "javadoc", - code: JAVA_HELLO, - output: "README.md", - overwrite: true, - cwd: root, - }); - expect(err.code).toBe("invalid-params"); - expect(err.message).toContain("does not look like a previous jvm_javadoc output"); - expect(await fs.readFile(path.join(root, "README.md"), "utf8")).toBe("keep me"); - }); - - test("javadoc still writes a fresh nested output", async () => { - const root = await project(); - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, output: "docs/api", cwd: root }); - expect(r.output).toBe(path.join(root, "docs", "api")); - expect((await fs.readdir(path.join(root, "docs", "api"))).sort()).toEqual(["element-list", "index.html"]); - expect(await fs.readFile(path.join(root, "README.md"), "utf8")).toBe("keep me"); - }); - - test("a symlinked prefix cannot smuggle the output outside the working directory", async () => { - // Lexically `link/docs` reads as "inside the project"; really it is somewhere - // else entirely, and the javadoc replace path is a recursive remove. + test("a symlinked prefix cannot smuggle file output outside the working directory", async () => { const root = await project(); const outside = await fs.mkdtemp(path.join(dir, "outside-")); await fs.mkdir(path.join(outside, "docs"), { recursive: true }); @@ -483,7 +398,14 @@ describe("runtime/jvm — output paths are bound to the working directory", () = await fs.symlink(outside, path.join(root, "link"), "dir"); for (const params of [ - { action: "javadoc" as const, code: JAVA_HELLO, output: "link/docs", overwrite: true, cwd: root }, + { + action: "deps" as const, + language: "java" as const, + code: JAVA_HELLO, + output: "link/deps.txt", + overwrite: true, + cwd: root, + }, { action: "jar" as const, language: "java" as const, @@ -502,13 +424,39 @@ describe("runtime/jvm — output paths are bound to the working directory", () = expect(await fs.readdir(outside)).toEqual(["docs"]); }); - test("a symlink pointing inside the working directory still works", async () => { + test("a destination symlink cannot redirect dependency output", async () => { + const root = await project(); + const outside = await fs.mkdtemp(path.join(dir, "outside-file-")); + const target = path.join(outside, "deps.txt"); + await fs.writeFile(target, "keep"); + await fs.symlink(target, path.join(root, "deps.txt")); + const err = await jvmError({ + action: "deps", + language: "java", + code: JAVA_HELLO, + output: "deps.txt", + overwrite: true, + cwd: root, + }); + expect(err.code).toBe("invalid-params"); + expect(err.message).toContain("output must be a path inside the working directory"); + expect(await fs.readFile(target, "utf8")).toBe("keep"); + expect(await invocations()).toEqual([]); + }); + + test("a symlink pointing inside the working directory still accepts dependency output", async () => { const root = await project(); await fs.mkdir(path.join(root, "real"), { recursive: true }); await fs.symlink(path.join(root, "real"), path.join(root, "inside-link"), "dir"); - const r = await jvm({ action: "javadoc", code: JAVA_HELLO, output: "inside-link/api", cwd: root }); - expect(r.output).toBe(path.join(root, "inside-link", "api")); - expect(await fs.readdir(path.join(root, "real", "api"))).toContain("index.html"); + const r = await jvm({ + action: "deps", + language: "java", + code: JAVA_HELLO, + output: "inside-link/deps.txt", + cwd: root, + }); + expect(r.output).toBe(path.join(root, "inside-link", "deps.txt")); + expect(await fs.readFile(path.join(root, "real", "deps.txt"), "utf8")).toContain("jdeps"); }); test("jar create refuses an output that is the working directory or above it", async () => { @@ -556,7 +504,9 @@ describe("runtime/jvm — shared behaviour", () => { test("an empty path takes the source branch instead of analyzing the whole cwd", async () => { const err = await jvmError({ action: "deps", path: "", cwd: dir }); - expect(err.message).toBe("jvm_deps requires either `path` (existing .class/.jar) or `language` + `code`."); + expect(err.message).toBe( + "jvm_deps requires `path` (source, .class, .jar, or class directory) or `language` + `code`.", + ); expect(await invocations()).toEqual([]); }); @@ -578,7 +528,7 @@ describe("runtime/jvm — shared behaviour", () => { "jvm_format requires `language` and `code`.", ); expect((await jvmError({ action: "deps" })).message).toBe( - "jvm_deps requires either `path` (existing .class/.jar) or `language` + `code`.", + "jvm_deps requires `path` (source, .class, .jar, or class directory) or `language` + `code`.", ); }); diff --git a/packages/coding-agent/test/runtime-launch-tools.test.ts b/packages/coding-agent/test/runtime-launch-tools.test.ts index da86e120464..23e1992725e 100644 --- a/packages/coding-agent/test/runtime-launch-tools.test.ts +++ b/packages/coding-agent/test/runtime-launch-tools.test.ts @@ -4,26 +4,20 @@ import type { DaemonSnapshot, DaemonState } from "../src/launch/protocol"; import type { RuntimeLaunchDescriptor } from "../src/runtime/protocol"; import type { ToolSession } from "../src/tools"; import type { LaunchParams, LaunchToolDetails } from "../src/tools/hub/launch"; -import { RuntimeDebugTool } from "../src/tools/runtime-debug"; import { matchRuntimeEndpoint, resolveWaitSeconds } from "../src/tools/runtime-launch"; import { RuntimeServeTool } from "../src/tools/runtime-serve"; // Kept in step with `transport/local.ts` by `runtime-spawn-endpoint.test.ts`, // which asserts the descriptor the endpoint actually emits and scrapes the real // 1.4.2 banners through it. -const CDP_RULES = [{ pattern: "ws://\\S+" }]; -const DAP_RULES = [{ pattern: "listening on\\s+/?(\\S+)", group: 1 }]; const SERVE_RULES = [{ pattern: "Serving static files on\\s+(\\S+)", group: 1, prefix: "http://" }]; -function descriptorFor(mode: "debug" | "serve", overrides: Partial = {}) { +function descriptorFor(overrides: Partial = {}) { return { - argv: - mode === "debug" - ? ["/opt/runtime/bin/elide", "run", "--debugger=cdp", "-l", "ts", "/proj/app.ts"] - : ["/opt/runtime/bin/elide", "serve", "/proj/public", "--no-tui"], + argv: ["/opt/runtime/bin/elide", "serve", "/proj/public", "--no-tui"], cwd: "/proj", env: { NO_COLOR: "1" }, - endpointPattern: mode === "debug" ? CDP_RULES : SERVE_RULES, + endpointPattern: SERVE_RULES, source: "managed" as const, ...overrides, } satisfies RuntimeLaunchDescriptor; @@ -97,15 +91,6 @@ function sessionWith(spawn: (params: unknown) => Promise { - test("scrapes a CDP inspector URL whole", () => { - const out = "Debugger listening. Open ws://127.0.0.1:4242/abc-123 in DevTools.\n"; - expect(matchRuntimeEndpoint(out, CDP_RULES)).toBe("ws://127.0.0.1:4242/abc-123"); - }); - - test("scrapes the DAP capture group, not the whole line", () => { - expect(matchRuntimeEndpoint("DAP server listening on 127.0.0.1:4711\n", DAP_RULES)).toBe("127.0.0.1:4711"); - }); - test("prefixes a bare serve host:port with its scheme", () => { expect(matchRuntimeEndpoint("Serving static files on 127.0.0.1:8080\n", SERVE_RULES)).toBe( "http://127.0.0.1:8080", @@ -125,7 +110,7 @@ describe("matchRuntimeEndpoint", () => { test("output with no endpoint yields undefined rather than a guess", () => { expect(matchRuntimeEndpoint("Starting up...\nloading modules\n", SERVE_RULES)).toBeUndefined(); - expect(matchRuntimeEndpoint("", CDP_RULES)).toBeUndefined(); + expect(matchRuntimeEndpoint("", SERVE_RULES)).toBeUndefined(); }); test("rules are tried in order and an unparseable pattern is skipped, not thrown", () => { @@ -144,24 +129,6 @@ describe("matchRuntimeEndpoint", () => { matchRuntimeEndpoint("Serving static files on 127.0.0.1:8080", [{ pattern: "\\p{" }, ...SERVE_RULES]), ).toBe("http://127.0.0.1:8080"); }); - - test("the real Graal DAP banner yields a bare host:port, with the leading slash dropped", () => { - // 1.4.2 prints Java's InetSocketAddress formatting, which includes a `/`. - // `/0.0.0.0:4711` is not something a DAP client can attach to. - const banner = "[Graal DAP] Starting server and listening on /0.0.0.0:4711\n"; - expect(matchRuntimeEndpoint(banner, DAP_RULES)).toBe("0.0.0.0:4711"); - }); - - test("a DAP banner without the slash still parses", () => { - expect(matchRuntimeEndpoint("DAP listening on 127.0.0.1:4711\n", DAP_RULES)).toBe("127.0.0.1:4711"); - }); - - test("the real CDP banner still resolves to the ws:// URL, not the surrounding words", () => { - const banner = - "Debugger listening on ws://127.0.0.1:9229/0dc12963/inspect\n" + - "For help, see: https://www.graalvm.org/tools/chrome-debugger\n"; - expect(matchRuntimeEndpoint(banner, CDP_RULES)).toBe("ws://127.0.0.1:9229/0dc12963/inspect"); - }); }); describe("resolveWaitSeconds", () => { @@ -175,190 +142,10 @@ describe("resolveWaitSeconds", () => { }); }); -describe("runtime_debug", () => { - test("is discoverable, exec-approved, gated on runtime.enabled, and not named `debug`", () => { - const session = sessionWith(async () => descriptorFor("debug")); - expect(RuntimeDebugTool.createIf(sessionWith(async () => descriptorFor("debug"), false))).toBeNull(); - const tool = RuntimeDebugTool.createIf(session); - expect(tool).not.toBeNull(); - expect(tool!.name).toBe("runtime_debug"); - expect(tool!.name).not.toBe("debug"); - expect(tool!.loadMode).toBe("discoverable"); - expect(tool!.approval).toBe("exec"); - }); - - test("starts the descriptor through hub, without a PTY, and returns the scraped endpoint", async () => { - const hub = fakeHub({ logs: "Debugger listening on ws://127.0.0.1:4242/tab-1" }); - let spawned: unknown; - const tool = new RuntimeDebugTool( - sessionWith(async params => { - spawned = params; - return descriptorFor("debug"); - }), - hub.launch, - ); - const result = await tool.execute("id", { path: "app.ts" }); - expect(spawned).toMatchObject({ mode: "debug", path: "app.ts", protocol: "cdp", cwd: "/proj" }); - - const start = hub.calls[0]!; - expect(start.op).toBe("start"); - expect(start.application).toBe("/opt/runtime/bin/elide"); - expect(start.args).toEqual(["run", "--debugger=cdp", "-l", "ts", "/proj/app.ts"]); - expect(start.cwd).toBe("/proj"); - expect(start.env).toEqual({ NO_COLOR: "1" }); - expect(start.pty).toBe(false); - // hub's own readiness wait does the waiting; the pattern is the descriptor's. - expect(start.ready).toEqual({ log: "(?:ws://\\S+)", timeout: 15 }); - expect(hub.calls[1]).toMatchObject({ op: "logs", name: start.name, head: true }); - - const text = (result.content[0] as { text: string }).text; - expect(text).toContain("ws://127.0.0.1:4242/tab-1"); - expect(text).toContain("Chrome DevTools"); - expect(text).toContain(`hub {op:"stop", name:"${start.name}"}`); - expect(result.details).toMatchObject({ - mode: "debug", - jobName: start.name, - endpoint: "ws://127.0.0.1:4242/tab-1", - timedOut: false, - }); - // The startup output the model sees is the process's own, without hub's marker. - expect(result.details?.startupOutput).not.toContain("cursor=42"); - }); - - test("the dap protocol threads through and uses the dap attach hint", async () => { - const hub = fakeHub({ logs: "DAP listening on 127.0.0.1:4711" }); - const tool = new RuntimeDebugTool( - sessionWith(async () => descriptorFor("debug", { endpointPattern: DAP_RULES })), - hub.launch, - ); - const result = await tool.execute("id", { path: "app.ts", protocol: "dap" }); - const text = (result.content[0] as { text: string }).text; - expect(text).toContain("DAP debugger listening at 127.0.0.1:4711"); - expect(text).toContain("VS Code"); - }); - - test("no endpoint in the wait window returns the job plus startup output, not an error", async () => { - const hub = fakeHub({ logs: "warming up\nresolving imports", state: "running" }); - const tool = new RuntimeDebugTool( - sessionWith(async () => descriptorFor("debug")), - hub.launch, - ); - const result = await tool.execute("id", { path: "app.ts", waitSeconds: 3 }); - const text = (result.content[0] as { text: string }).text; - expect(result.isError).toBeUndefined(); - expect(text).toContain("did not report an endpoint within the wait window (3s)"); - expect(text).toContain("it may still be starting"); - expect(text).toContain("Startup output:\nwarming up\nresolving imports"); - expect(text).toContain(hub.calls[0]!.name!); - expect(result.details).toMatchObject({ timedOut: true, state: "running" }); - expect(result.details?.endpoint).toBeUndefined(); - }); - - test("the endpoint comes from hub's matched line even when the banner is not in the log read", async () => { - // The readiness buffer saw the banner; the startup lines read back did not - // (a busy process scrolls it past the first 200 lines). - const hub = fakeHub({ - logs: "line1\nline2\nline3", - readySees: "Debugger listening on ws://127.0.0.1:9229/deep/inspect", - }); - const tool = new RuntimeDebugTool( - sessionWith(async () => descriptorFor("debug")), - hub.launch, - ); - const result = await tool.execute("id", { path: "app.ts" }); - expect(result.details?.endpoint).toBe("ws://127.0.0.1:9229/deep/inspect"); - expect(result.details?.readyMatch).toBe("ws://127.0.0.1:9229/deep/inspect"); - expect(result.details?.timedOut).toBe(false); - }); - - test("a banner that matches but yields no endpoint is reported as a stale rule, not a timeout", async () => { - // A rule whose capture group can match empty: readiness fires, extraction - // does not. Conflating this with a timeout is what hides a bad rule. - const rules = [{ pattern: "listening(.*)", group: 1 }]; - const hub = fakeHub({ logs: "listening" }); - const tool = new RuntimeDebugTool( - sessionWith(async () => descriptorFor("debug", { endpointPattern: rules })), - hub.launch, - ); - const result = await tool.execute("id", { path: "app.ts" }); - const text = (result.content[0] as { text: string }).text; - expect(result.details?.endpoint).toBeUndefined(); - expect(result.details?.timedOut).toBe(false); - expect(result.details?.readyMatch).toBe("listening"); - expect(text).toContain("no endpoint could be extracted"); - expect(text).toContain("scraping rule is probably stale"); - expect(text).not.toContain("wait window"); - }); - - test("without the hub tool the guidance names only remedies that actually exist", async () => { - const hub = fakeHub({ logs: "Debugger listening on ws://127.0.0.1:9229/x" }); - const session = sessionWith(async () => descriptorFor("debug")); - (session as { isToolActive?: (name: string) => boolean }).isToolActive = name => name !== "hub"; - const result = await new RuntimeDebugTool(session, hub.launch).execute("id", { path: "app.ts" }); - const text = (result.content[0] as { text: string }).text; - expect(text).toContain("no hub tool"); - // Must not name a tool the session lacks... - expect(text).not.toContain('hub {op:"stop"'); - // ...and must not name `/jobs`, which lists async tool jobs (read-only) and - // never broker daemons — it can neither show nor stop this job. - expect(text).not.toContain("/jobs"); - // The remedies that are true: the broker takes non-detached daemons down with - // it, and a session with hub can stop it. - expect(text).toContain("background broker exits"); - expect(text).toContain("session that has hub"); - expect(text).toMatch(/out of band/); - // The endpoint is still reported — only the lifecycle advice changes. - expect(result.details?.endpoint).toBe("ws://127.0.0.1:9229/x"); - }); - - test("the hub-less caveat also reaches the no-endpoint and failed-launch bodies", async () => { - const session = sessionWith(async () => descriptorFor("debug")); - (session as { isToolActive?: (name: string) => boolean }).isToolActive = name => name !== "hub"; - for (const hub of [fakeHub({ logs: "warming up", state: "running" }), fakeHub({ logs: "", state: "failed" })]) { - const result = await new RuntimeDebugTool(session, hub.launch).execute("id", { path: "app.ts" }); - const text = (result.content[0] as { text: string }).text; - expect(text).not.toContain("/jobs"); - expect(text).toContain("background broker exits"); - } - }); - - test("a failed launch is reported as an error, with the job name kept", async () => { - const hub = fakeHub({ logs: "", state: "failed" }); - const tool = new RuntimeDebugTool( - sessionWith(async () => descriptorFor("debug")), - hub.launch, - ); - const result = await tool.execute("id", { path: "app.ts" }); - expect(result.isError).toBe(true); - expect((result.content[0] as { text: string }).text).toContain(hub.calls[0]!.name!); - // No log read is attempted for a process that never started. - expect(hub.calls.map(c => c.op)).toEqual(["start"]); - }); - - test("a PATH-resolved binary surfaces the shim note", async () => { - const hub = fakeHub({ logs: "ws://127.0.0.1:4242/x" }); - const tool = new RuntimeDebugTool( - sessionWith(async () => descriptorFor("debug", { source: "path", shimWarning: "wrapper script warning" })), - hub.launch, - ); - const result = await tool.execute("id", { path: "app.ts" }); - expect((result.content[0] as { text: string }).text).toContain("Note: wrapper script warning"); - }); - - test("throws the standard explanation when the session has no runtime service", async () => { - const tool = new RuntimeDebugTool({ - cwd: "/proj", - settings: { get: () => true }, - getRuntimeService: () => undefined, - } as unknown as ToolSession); - await expect(tool.execute("id", { path: "app.ts" })).rejects.toThrow(/runtime service is unavailable/); - }); -}); - describe("serve", () => { test("is discoverable, exec-approved, and gated on runtime.enabled", () => { - expect(RuntimeServeTool.createIf(sessionWith(async () => descriptorFor("serve"), false))).toBeNull(); - const tool = RuntimeServeTool.createIf(sessionWith(async () => descriptorFor("serve"))); + expect(RuntimeServeTool.createIf(sessionWith(async () => descriptorFor(), false))).toBeNull(); + const tool = RuntimeServeTool.createIf(sessionWith(async () => descriptorFor())); expect(tool!.name).toBe("serve"); expect(tool!.loadMode).toBe("discoverable"); expect(tool!.approval).toBe("exec"); @@ -370,22 +157,22 @@ describe("serve", () => { const tool = new RuntimeServeTool( sessionWith(async params => { spawned = params; - return descriptorFor("serve"); + return descriptorFor(); }), hub.launch, ); const result = await tool.execute("id", { directory: "public", port: 8080, host: "127.0.0.1" }); - expect(spawned).toMatchObject({ mode: "serve", directory: "public", port: 8080, host: "127.0.0.1" }); + expect(spawned).toEqual({ directory: "public", port: 8080, host: "127.0.0.1", cwd: "/proj" }); const text = (result.content[0] as { text: string }).text; expect(text).toContain("Serving /proj/public at http://127.0.0.1:8080"); expect(text).toContain(`hub {op:"logs", name:"${hub.calls[0]!.name}"}`); - expect(result.details).toMatchObject({ mode: "serve", endpoint: "http://127.0.0.1:8080", timedOut: false }); + expect(result.details).toMatchObject({ endpoint: "http://127.0.0.1:8080", timedOut: false }); }); test("no URL in the wait window falls back to the startup output", async () => { const hub = fakeHub({ logs: "binding socket", state: "running" }); const tool = new RuntimeServeTool( - sessionWith(async () => descriptorFor("serve")), + sessionWith(async () => descriptorFor()), hub.launch, ); const result = await tool.execute("id", { directory: "public" }); @@ -398,7 +185,7 @@ describe("serve", () => { test("job names are distinct per call, so concurrent servers never collide", async () => { const hub = fakeHub({ logs: "Serving static files on 127.0.0.1:8080" }); const tool = new RuntimeServeTool( - sessionWith(async () => descriptorFor("serve")), + sessionWith(async () => descriptorFor()), hub.launch, ); const a = await tool.execute("id", { directory: "public" }); diff --git a/packages/coding-agent/test/runtime-local-endpoint.test.ts b/packages/coding-agent/test/runtime-local-endpoint.test.ts index ee25b6970a6..f23a6da66d8 100644 --- a/packages/coding-agent/test/runtime-local-endpoint.test.ts +++ b/packages/coding-agent/test/runtime-local-endpoint.test.ts @@ -46,14 +46,6 @@ describe("LocalRuntimeEndpoint", () => { expect(out.stdout.trim()).toBe("ARGS:build --no-color"); }); - test("build passes targets through", async () => { - const ep = new LocalRuntimeEndpoint({ explicitPath: fakeBin, autoDownload: false }); - const out = unwrapResponse( - await ep.request(createRequest("runtime/build", { targets: [":deps", "--fresh"] })), - ); - expect(out.stdout.trim()).toBe("ARGS:build --no-color :deps --fresh"); - }); - test("status reports version without provisioning", async () => { const ep = new LocalRuntimeEndpoint({ explicitPath: fakeBin, autoDownload: false }); const out = unwrapResponse(await ep.request(createRequest("runtime/status", undefined))); @@ -217,55 +209,6 @@ describe("LocalRuntimeEndpoint", () => { }); }); - describe("runtime/advice", () => { - test("maps to the pinned project-advice argv and takes no other input", async () => { - const ep = new LocalRuntimeEndpoint({ explicitPath: fakeBin, autoDownload: false }); - const out = unwrapResponse(await ep.request(createRequest("runtime/advice", {}))); - expect(out.exitCode).toBe(0); - expect(out.stdout.trim()).toBe("ARGS:project advice --error-format=plain --no-color"); - }); - - test("empty params are valid — there is nothing to configure but where to look", async () => { - const ep = new LocalRuntimeEndpoint({ explicitPath: fakeBin, autoDownload: false }); - // Undefined params, not even an object: `advice` must not demand a field. - const out = unwrapResponse(await ep.request(createRequest("runtime/advice", undefined))); - expect(out.stdout.trim()).toBe("ARGS:project advice --error-format=plain --no-color"); - }); - - test("runs in the caller's real directory, with no temp workdir", async () => { - // The whole point of the flow: advice detects manifests in place, so the - // process cwd must be exactly what the caller named. - const pwdBin = path.join(dir, "pwd-echo"); - await fs.writeFile(pwdBin, `#!/bin/sh\necho "CWD:$(pwd)"\n`, { mode: 0o755 }); - const project = path.join(dir, "project"); - await fs.mkdir(project, { recursive: true }); - const ep = new LocalRuntimeEndpoint({ explicitPath: pwdBin, autoDownload: false }); - const out = unwrapResponse( - await ep.request(createRequest("runtime/advice", { cwd: project })), - ); - expect(out.stdout.trim()).toBe(`CWD:${await fs.realpath(project)}`); - }); - - test("omitting cwd leaves the endpoint process directory in place", async () => { - const pwdBin = path.join(dir, "pwd-echo2"); - await fs.writeFile(pwdBin, `#!/bin/sh\necho "CWD:$(pwd)"\n`, { mode: 0o755 }); - const ep = new LocalRuntimeEndpoint({ explicitPath: pwdBin, autoDownload: false }); - const out = unwrapResponse(await ep.request(createRequest("runtime/advice", {}))); - expect(out.stdout.trim()).toBe(`CWD:${await fs.realpath(process.cwd())}`); - }); - - test("a missing runtime is a typed runtime-missing error", async () => { - const ep = new LocalRuntimeEndpoint({ explicitPath: path.join(dir, "nope"), autoDownload: false }); - const res = await ep.request(createRequest("runtime/advice", {})); - try { - unwrapResponse(res); - throw new Error("expected error"); - } catch (e) { - expect((e as RuntimeRpcError).code).toBe("runtime-missing"); - } - }); - }); - test("timeout kills the process and reports killed", async () => { const slowBin = path.join(dir, "slow"); await fs.writeFile(slowBin, `#!/bin/sh\nsleep 5\n`, { mode: 0o755 }); diff --git a/packages/coding-agent/test/runtime-run-tool.test.ts b/packages/coding-agent/test/runtime-run-tool.test.ts index da2f8dd4232..2052eb6b734 100644 --- a/packages/coding-agent/test/runtime-run-tool.test.ts +++ b/packages/coding-agent/test/runtime-run-tool.test.ts @@ -12,11 +12,15 @@ import type { ToolSession } from "../src/tools"; import { wrapToolWithMetaNotice } from "../src/tools/output-meta"; import { RuntimeRunTool } from "../src/tools/runtime-run"; -function sessionWith(overrides: { enabled?: boolean; run?: (p: unknown) => Promise }): ToolSession { +function sessionWith(overrides: { + enabled?: boolean; + run?: (p: unknown, signal?: AbortSignal, sessionId?: string) => Promise; +}): ToolSession { const service = overrides.run ? { run: overrides.run } : undefined; return { settings: { get: (key: string) => (key === "runtime.enabled" ? (overrides.enabled ?? true) : undefined) }, getRuntimeService: () => (overrides.enabled === false ? undefined : (service as never)), + getSessionId: () => "session-a", } as unknown as ToolSession; } @@ -34,10 +38,12 @@ describe("run tool", () => { test("execute forwards params to the service and formats the result", async () => { let received: unknown; + let receivedSessionId: string | undefined; const tool = RuntimeRunTool.createIf( sessionWith({ - run: async p => { + run: async (p, _signal, sessionId) => { received = p; + receivedSessionId = sessionId; return { exitCode: 0, stdout: "hello\n", stderr: "", durationMs: 5, killed: false }; }, }), @@ -48,11 +54,27 @@ describe("run tool", () => { new AbortController().signal, ); expect((received as { code: string }).code).toBe("console.log('hello')"); + expect(receivedSessionId).toBe("session-a"); const block = result?.content[0] as { type: "text"; text: string } | undefined; expect(block?.text).toContain("hello"); expect(result?.details).toMatchObject({ exitCode: 0 }); }); + test.each([ + { + name: "nonzero exit", + exec: { exitCode: 2, stdout: "", stderr: "boom", durationMs: 3, killed: false }, + }, + { + name: "killed execution", + exec: { exitCode: 0, stdout: "", stderr: "", durationMs: 3, killed: true }, + }, + ])("marks $name as a tool error", async ({ exec }) => { + const tool = RuntimeRunTool.createIf(sessionWith({ run: async () => exec })); + const result = await tool?.execute("id1", { code: "fail()" } as never, new AbortController().signal); + expect(result?.isError).toBe(true); + }); + test("evicts an internally failed cached service so the next call gets a fresh runtime", async () => { const options = { adapter: "process" as const, autoDownload: false, explicitPath: "/runtime-fixture" }; const scope: RuntimeServiceScope = { diff --git a/packages/coding-agent/test/runtime-service.test.ts b/packages/coding-agent/test/runtime-service.test.ts index 65696f6aa76..97cba0246ae 100644 --- a/packages/coding-agent/test/runtime-service.test.ts +++ b/packages/coding-agent/test/runtime-service.test.ts @@ -1,9 +1,11 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; +import { type Span, trace } from "@opentelemetry/api"; import { okResponse, type RuntimeRpcRequest, type RuntimeRpcResponse } from "../src/runtime/protocol"; import { type RuntimeEndpoint, RuntimeService } from "../src/runtime/service"; +import { subscribeTelemetry, type TelemetryEvent } from "../src/telemetry/events"; class RecordingEndpoint implements RuntimeEndpoint { requests: RuntimeRpcRequest[] = []; @@ -32,20 +34,18 @@ describe("RuntimeService", () => { const svc = new RuntimeService(ep); await svc.run({ code: "console.log(1)" }); await svc.check({}); - await svc.build({ targets: [":compile"] }); await svc.insights({ code: "x", insight: "y" }); await svc.profile({ code: "x", mode: "cpusampling" }); - await svc.spawn({ mode: "serve", directory: "public" }); - await svc.advice({}); + await svc.jvm({ action: "deps", path: "app.jar" }); + await svc.spawn({ directory: "public" }); await svc.status(); expect(ep.requests.map(r => r.method)).toEqual([ "runtime/run", "runtime/check", - "runtime/build", "runtime/insights", "runtime/profile", + "runtime/jvm", "runtime/spawn", - "runtime/advice", "runtime/status", ]); }); @@ -57,6 +57,190 @@ describe("RuntimeService", () => { expect(r.stdout).toBe("ok"); }); + test("publishes bounded runtime facts and annotates the active tool span", async () => { + const events: TelemetryEvent[] = []; + const attributes: Record = {}; + const activeSpan = { + setAttribute(key: string, value: string | number | boolean) { + attributes[key] = value; + return this; + }, + } as unknown as Span; + const spanSpy = spyOn(trace, "getActiveSpan").mockReturnValue(activeSpan); + const unsubscribe = subscribeTelemetry(event => events.push(event)); + try { + const result = await new RuntimeService(new RecordingEndpoint()).run( + { + code: "console.log(1)", + language: "ts", + }, + undefined, + "session-a", + ); + expect(result.exitCode).toBe(0); + } finally { + unsubscribe(); + spanSpy.mockRestore(); + } + + const event = events.find(candidate => candidate.type === "runtime.call.completed"); + expect(event).toMatchObject({ + type: "runtime.call.completed", + sessionId: "session-a", + method: "runtime/run", + language: "ts", + outcome: "ok", + exitCode: 0, + killed: false, + }); + expect(event).not.toHaveProperty("stdout"); + expect(event).not.toHaveProperty("stderr"); + expect(event).not.toHaveProperty("params"); + expect(attributes).toMatchObject({ + "aura.runtime.method": "runtime/run", + "aura.runtime.language": "ts", + "aura.runtime.outcome": "ok", + "aura.runtime.exit_code": 0, + "aura.runtime.killed": false, + }); + expect(attributes["aura.runtime.duration_ms"]).toBeNumber(); + }); + + test("classifies runtime protocol failures once without changing the thrown error", async () => { + const events: TelemetryEvent[] = []; + const unsubscribe = subscribeTelemetry(event => events.push(event)); + const service = new RuntimeService({ + async request(req) { + return { + jsonrpc: "2.0", + id: req.id, + error: { code: "timeout", message: "guest timed out" }, + }; + }, + }); + try { + await expect(service.run({ code: "while(true){}", language: "js" })).rejects.toMatchObject({ + name: "RuntimeRpcError", + code: "timeout", + }); + } finally { + unsubscribe(); + } + + const runtimeEvents = events.filter(candidate => candidate.type === "runtime.call.completed"); + expect(runtimeEvents).toHaveLength(1); + expect(runtimeEvents[0]).toMatchObject({ + method: "runtime/run", + language: "js", + outcome: "timeout", + errorType: "timeout", + }); + expect(runtimeEvents[0]).not.toHaveProperty("message"); + }); + + test("classifies non-zero, killed, cancelled, and unexpected failures exactly once without mutation", async () => { + const controller = new AbortController(); + controller.abort(); + const unexpected = new Error("endpoint exploded"); + const cases = [ + { + name: "non-zero", + response: { exitCode: 2, stdout: "", stderr: "bad", durationMs: 1, killed: false }, + expected: { outcome: "error", exitCode: 2, killed: false, errorType: "non_zero_exit" }, + }, + { + name: "killed", + response: { exitCode: 137, stdout: "", stderr: "", durationMs: 1, killed: true }, + expected: { outcome: "timeout", exitCode: 137, killed: true, errorType: "killed" }, + }, + { + name: "cancelled", + error: new Error("cancelled"), + signal: controller.signal, + expected: { outcome: "cancelled", errorType: "cancelled" }, + }, + { + name: "unexpected", + error: unexpected, + expected: { outcome: "error", errorType: "unknown" }, + }, + ] as const; + for (const testCase of cases) { + const events: TelemetryEvent[] = []; + const unsubscribe = subscribeTelemetry(event => events.push(event)); + const service = new RuntimeService({ + async request(req) { + if ("error" in testCase) throw testCase.error; + return okResponse(req.id, testCase.response); + }, + }); + let result: unknown; + let failure: unknown; + try { + result = await service.run( + { code: "process.exit(2)", language: "js" }, + "signal" in testCase ? testCase.signal : undefined, + "session-a", + ); + } catch (error) { + failure = error; + } finally { + unsubscribe(); + } + const runtimeEvents = events.filter(event => event.type === "runtime.call.completed"); + expect(runtimeEvents, testCase.name).toHaveLength(1); + expect(runtimeEvents[0], testCase.name).toMatchObject({ + sessionId: "session-a", + method: "runtime/run", + language: "js", + ...testCase.expected, + }); + if ("error" in testCase) expect(failure, testCase.name).toBe(testCase.error); + else expect(result, testCase.name).toEqual(testCase.response); + } + }); + + test("records JVM action and resolved language plus the spawn method", async () => { + const events: TelemetryEvent[] = []; + const unsubscribe = subscribeTelemetry(event => events.push(event)); + const service = new RuntimeService(new RecordingEndpoint()); + try { + await service.jvm({ action: "disassemble", language: "java", code: "class Main {}" }); + await service.spawn({ directory: "public" }); + } finally { + unsubscribe(); + } + const completed = events.filter(event => event.type === "runtime.call.completed"); + expect(completed).toEqual([ + expect.objectContaining({ method: "runtime/jvm", action: "disassemble", language: "java" }), + expect.objectContaining({ method: "runtime/spawn" }), + ]); + expect(completed[1]?.action).toBeUndefined(); + expect(completed[1]?.language).toBeUndefined(); + }); + + test("telemetry sink and span failures preserve the runtime result", async () => { + const events: TelemetryEvent[] = []; + const failingUnsubscribe = subscribeTelemetry(() => { + throw new Error("sink failed"); + }); + const collectingUnsubscribe = subscribeTelemetry(event => events.push(event)); + const spanSpy = spyOn(trace, "getActiveSpan").mockReturnValue({ + setAttribute() { + throw new Error("span failed"); + }, + } as unknown as Span); + try { + const result = await new RuntimeService(new RecordingEndpoint()).run({ code: "1" }, undefined, "session-a"); + expect(result.exitCode).toBe(0); + } finally { + failingUnsubscribe(); + collectingUnsubscribe(); + spanSpy.mockRestore(); + } + expect(events.filter(event => event.type === "runtime.call.completed")).toHaveLength(1); + }); + test("close is idempotent and waits for endpoint settlement", async () => { const endpoint = new ClosingEndpoint(); const service = new RuntimeService(endpoint); diff --git a/packages/coding-agent/test/runtime-spawn-endpoint.test.ts b/packages/coding-agent/test/runtime-spawn-endpoint.test.ts index 0f80090d6ba..beb564d7ddd 100644 --- a/packages/coding-agent/test/runtime-spawn-endpoint.test.ts +++ b/packages/coding-agent/test/runtime-spawn-endpoint.test.ts @@ -38,55 +38,9 @@ async function descriptor(params: Record): Promise(await endpoint().request(createRequest("runtime/spawn", params))); } -describe("runtime/spawn — debug descriptors", () => { - test("composes the CDP argv and carries the ws:// recognition rule", async () => { - const d = await descriptor({ mode: "debug", path: guest, cwd: dir }); - expect(d.argv).toEqual([ - fakeBin, - "run", - "--debugger=cdp", - "--error-format=plain", - "--no-color", - "-l", - "ts", - guest, - ]); - expect(d.cwd).toBe(dir); - expect(d.env).toEqual({ NO_COLOR: "1" }); - expect(d.endpointPattern).toEqual([{ pattern: "ws://\\S+" }]); - expect(d.source).toBe("flag"); - expect(d.shimWarning).toBeUndefined(); - }); - - test("dap selects the debugger flag and the `listening on` rule", async () => { - const d = await descriptor({ mode: "debug", path: guest, protocol: "dap", cwd: dir }); - expect(d.argv).toContain("--debugger=dap"); - // The optional `/` is consumed, not captured: see the real banner test below. - expect(d.endpointPattern).toEqual([{ pattern: "listening on\\s+/?(\\S+)", group: 1 }]); - }); - - test("language is inferred from the extension and overridable", async () => { - const py = path.join(dir, "app.py"); - await fs.writeFile(py, "print(1)\n"); - expect((await descriptor({ mode: "debug", path: py, cwd: dir })).argv).toContain("python"); - expect((await descriptor({ mode: "debug", path: py, language: "js", cwd: dir })).argv).toContain("js"); - }); - - test("guest args go after `--`, and timeoutMs becomes the runtime's own timeout flag", async () => { - const d = await descriptor({ mode: "debug", path: guest, args: ["a", "b"], timeoutMs: 2500, cwd: dir }); - expect(d.argv.join(" ")).toContain("--timeout 2500ms"); - expect(d.argv.slice(-3)).toEqual(["--", "a", "b"]); - }); - - test("a missing or absent path is invalid-params, not a launch", async () => { - await expect(descriptor({ mode: "debug", cwd: dir })).rejects.toMatchObject({ code: "invalid-params" }); - await expect(descriptor({ mode: "debug", path: path.join(dir, "nope.ts"), cwd: dir })).rejects.toMatchObject({ - code: "invalid-params", - }); - }); - - test("a directory in place of the program file is refused", async () => { - await expect(descriptor({ mode: "debug", path: staticDir, cwd: dir })).rejects.toMatchObject({ +describe("runtime/spawn — removed debug descriptors", () => { + test("rejects the retired debug mode instead of silently serving", async () => { + await expect(descriptor({ mode: "debug", path: guest, directory: staticDir, cwd: dir })).rejects.toMatchObject({ code: "invalid-params", }); }); @@ -94,7 +48,7 @@ describe("runtime/spawn — debug descriptors", () => { describe("runtime/spawn — serve descriptors", () => { test("composes the serve argv with --no-tui and the static-files rule", async () => { - const d = await descriptor({ mode: "serve", directory: "public", cwd: dir }); + const d = await descriptor({ directory: "public", cwd: dir }); expect(d.argv).toEqual([fakeBin, "serve", staticDir, "--no-tui"]); expect(d.endpointPattern).toEqual([ { pattern: "Serving static files on\\s+(\\S+)", group: 1, prefix: "http://" }, @@ -102,53 +56,33 @@ describe("runtime/spawn — serve descriptors", () => { }); test("port and host are appended only when supplied", async () => { - const d = await descriptor({ mode: "serve", directory: staticDir, port: 8123, host: "0.0.0.0", cwd: dir }); + const d = await descriptor({ directory: staticDir, port: 8123, host: "0.0.0.0", cwd: dir }); expect(d.argv.slice(-4)).toEqual(["--port", "8123", "--host", "0.0.0.0"]); }); test("a non-integer or out-of-range port is invalid-params", async () => { for (const port of [0, 70_000, 1.5]) { - await expect(descriptor({ mode: "serve", directory: staticDir, port, cwd: dir })).rejects.toMatchObject({ + await expect(descriptor({ directory: staticDir, port, cwd: dir })).rejects.toMatchObject({ code: "invalid-params", }); } }); test("a missing directory, or a file in its place, is invalid-params", async () => { - await expect(descriptor({ mode: "serve", cwd: dir })).rejects.toMatchObject({ code: "invalid-params" }); - await expect(descriptor({ mode: "serve", directory: "nope", cwd: dir })).rejects.toMatchObject({ + await expect(descriptor({ cwd: dir })).rejects.toMatchObject({ code: "invalid-params" }); + await expect(descriptor({ directory: "nope", cwd: dir })).rejects.toMatchObject({ code: "invalid-params", }); - await expect(descriptor({ mode: "serve", directory: guest, cwd: dir })).rejects.toMatchObject({ + await expect(descriptor({ directory: guest, cwd: dir })).rejects.toMatchObject({ code: "invalid-params", }); }); }); -/** - * The rules the endpoint ships must parse the banners the pinned runtime really - * prints. These are verbatim captures from 1.4.2 — the one thing a unit test can - * pin that a mocked banner cannot, and the bug class that shipped a `/0.0.0.0:4711` - * "endpoint" no DAP client could attach to. - */ -describe("runtime/spawn — the shipped rules parse real 1.4.2 banners", () => { - test("cdp", async () => { - const d = await descriptor({ mode: "debug", path: guest, protocol: "cdp", cwd: dir }); - const banner = - "Debugger listening on ws://127.0.0.1:9229/0dc12963/inspect\n" + - "For help, see: https://www.graalvm.org/tools/chrome-debugger\n" + - "E.g. in Chrome open: devtools://devtools/bundled/js_app.html?ws=127.0.0.1:9229/0dc12963/inspect\n"; - expect(matchRuntimeEndpoint(banner, d.endpointPattern)).toBe("ws://127.0.0.1:9229/0dc12963/inspect"); - }); - - test("dap — the leading slash never reaches the caller", async () => { - const d = await descriptor({ mode: "debug", path: guest, protocol: "dap", cwd: dir }); - const banner = "[Graal DAP] Starting server and listening on /0.0.0.0:4711\n"; - expect(matchRuntimeEndpoint(banner, d.endpointPattern)).toBe("0.0.0.0:4711"); - }); - +/** The endpoint rule must parse the banner printed by the pinned runtime. */ +describe("runtime/spawn — the shipped rule parses the real serve banner", () => { test("serve", async () => { - const d = await descriptor({ mode: "serve", directory: staticDir, cwd: dir }); + const d = await descriptor({ directory: staticDir, cwd: dir }); const banner = 'Serving from directory: "/tmp/pub"\nServing /tmp/pub at http://"127.0.0.1":8080\n' + "Serving static files on 127.0.0.1:8080\n"; @@ -159,7 +93,7 @@ describe("runtime/spawn — the shipped rules parse real 1.4.2 banners", () => { describe("runtime/spawn — resolution", () => { test("a missing runtime is a runtime-missing error, never a partial descriptor", async () => { const ep = new LocalRuntimeEndpoint({ explicitPath: path.join(dir, "gone"), autoDownload: false }); - const res = await ep.request(createRequest("runtime/spawn", { mode: "serve", directory: staticDir, cwd: dir })); + const res = await ep.request(createRequest("runtime/spawn", { directory: staticDir, cwd: dir })); expect(() => unwrapResponse(res)).toThrow(RuntimeRpcError); expect(() => unwrapResponse(res)).toThrow(/not installed/); }); @@ -170,7 +104,7 @@ describe("runtime/spawn — resolution", () => { resolve: async () => ({ binaryPath: fakeBin, source: "path" }), }); const d = unwrapResponse( - await onPath.request(createRequest("runtime/spawn", { mode: "serve", directory: staticDir, cwd: dir })), + await onPath.request(createRequest("runtime/spawn", { directory: staticDir, cwd: dir })), ); expect(d.source).toBe("path"); expect(d.shimWarning).toMatch(/PATH/); @@ -186,13 +120,17 @@ describe("runtime/spawn — resolution", () => { resolve: async () => ({ binaryPath: fakeBin, source: "managed" }), }); const m = unwrapResponse( - await managed.request(createRequest("runtime/spawn", { mode: "serve", directory: staticDir, cwd: dir })), + await managed.request(createRequest("runtime/spawn", { directory: staticDir, cwd: dir })), ); expect(m.shimWarning).toBeUndefined(); }); - test("an unknown mode is invalid-params", async () => { - await expect(descriptor({ mode: "repl" })).rejects.toMatchObject({ code: "invalid-params" }); + test("an obsolete or unknown mode is invalid-params", async () => { + for (const mode of ["debug", "repl"]) { + await expect(descriptor({ mode, directory: staticDir, cwd: dir })).rejects.toMatchObject({ + code: "invalid-params", + }); + } }); test("invalid params are rejected before the binary is resolved, so no download is triggered", async () => { @@ -212,11 +150,10 @@ describe("runtime/spawn — resolution", () => { }, }); for (const params of [ - { mode: "repl" }, - { mode: "debug", cwd: dir }, - { mode: "debug", path: path.join(dir, "missing.ts"), cwd: dir }, - { mode: "serve", directory: "nope", cwd: dir }, - { mode: "serve", directory: staticDir, port: 0, cwd: dir }, + { mode: "repl", directory: staticDir, cwd: dir }, + { mode: "debug", directory: staticDir, cwd: dir }, + { directory: "nope", cwd: dir }, + { directory: staticDir, port: 0, cwd: dir }, ]) { const res = await ep.request(createRequest("runtime/spawn", params)); expect(() => unwrapResponse(res)).toThrow(RuntimeRpcError); @@ -226,9 +163,7 @@ describe("runtime/spawn — resolution", () => { // Sanity: valid params DO resolve, so the assertion above is about ordering // rather than a resolve call that never happens. - unwrapResponse( - await ep.request(createRequest("runtime/spawn", { mode: "serve", directory: staticDir, cwd: dir })), - ); + unwrapResponse(await ep.request(createRequest("runtime/spawn", { directory: staticDir, cwd: dir }))); expect(resolved).toBe(1); expect(provisioned).toBe(1); }); diff --git a/packages/coding-agent/test/runtime-tool-registry.test.ts b/packages/coding-agent/test/runtime-tool-registry.test.ts index 3c0f9af7ed9..ee65afeddc7 100644 --- a/packages/coding-agent/test/runtime-tool-registry.test.ts +++ b/packages/coding-agent/test/runtime-tool-registry.test.ts @@ -1,14 +1,15 @@ import { describe, expect, test } from "bun:test"; +import { toolWireSchema } from "@oh-my-pi/pi-ai"; import { BUILTIN_TOOLS, type ToolSession } from "../src/tools"; import { BUILTIN_TOOL_NAMES, normalizeToolName } from "../src/tools/builtin-names"; import { ESSENTIAL_BUILTIN_TOOL_NAMES } from "../src/tools/essential-tools"; -const RUNTIME_TOOLS = ["run", "check", "build", "insights", "profile", "runtime_debug", "serve"] as const; +const RUNTIME_TOOLS = ["run", "check", "insights", "profile", "serve"] as const; -/** The two long-running flows, supervised by hub rather than by the runtime layer. */ -const LAUNCH_TOOLS = ["runtime_debug", "serve"] as const; +/** The long-running flow supervised by hub rather than by the runtime layer. */ +const LAUNCH_TOOLS = ["serve"] as const; -const JVM_TOOLS = ["jvm_disassemble", "jvm_format", "jvm_jar", "jvm_deps", "jvm_javadoc"] as const; +const JVM_TOOLS = ["jvm_disassemble", "jvm_format", "jvm_jar", "jvm_deps"] as const; function stubSession(enabled: boolean): ToolSession { return { @@ -17,6 +18,17 @@ function stubSession(enabled: boolean): ToolSession { } as unknown as ToolSession; } +async function providerPayloadBytes(names: readonly (keyof typeof BUILTIN_TOOLS)[]): Promise { + let bytes = 0; + for (const name of names) { + const tool = await BUILTIN_TOOLS[name](stubSession(true)); + if (!tool) throw new Error(`Expected ${name} to be available`); + bytes += Buffer.byteLength(tool.description ?? ""); + bytes += Buffer.byteLength(JSON.stringify(toolWireSchema(tool))); + } + return bytes; +} + describe("runtime tool registry", () => { test("all runtime tools are builtin names", () => { for (const name of RUNTIME_TOOLS) expect(BUILTIN_TOOL_NAMES).toContain(name); @@ -26,32 +38,35 @@ describe("runtime tool registry", () => { for (const name of RUNTIME_TOOLS) expect(normalizeToolName(name)).toBe(name); }); - test("run/check/build are essential; insights/profile/debug/serve are not", () => { + test("run/check are essential; removed runtime surfaces stay absent", () => { expect(ESSENTIAL_BUILTIN_TOOL_NAMES.run).toBe(true); expect(ESSENTIAL_BUILTIN_TOOL_NAMES.check).toBe(true); - expect(ESSENTIAL_BUILTIN_TOOL_NAMES.build).toBe(true); + expect(BUILTIN_TOOL_NAMES).not.toContain("build"); + expect(BUILTIN_TOOL_NAMES).not.toContain("project_advice"); + expect("build" in BUILTIN_TOOLS).toBe(false); + expect("project_advice" in BUILTIN_TOOLS).toBe(false); + expect(BUILTIN_TOOL_NAMES).not.toContain("jvm_javadoc"); + expect("jvm_javadoc" in BUILTIN_TOOLS).toBe(false); + expect(BUILTIN_TOOL_NAMES).not.toContain("runtime_debug"); + expect("runtime_debug" in BUILTIN_TOOLS).toBe(false); for (const name of ["insights", "profile", ...LAUNCH_TOOLS]) { expect(name in ESSENTIAL_BUILTIN_TOOL_NAMES).toBe(false); } }); + test("keeps inherent runtime provider payloads compact", async () => { + expect(await providerPayloadBytes(["run", "check"])).toBeLessThanOrEqual(1_800); + expect(await providerPayloadBytes([...RUNTIME_TOOLS, ...JVM_TOOLS])).toBeLessThanOrEqual(8_500); + }); - test("the runtime debug tool does NOT claim the built-in `debug` name", async () => { - // `debug` is the interactive stepping debugger (DebugTool) and must stay so: - // a collision here would silently replace it in the registry. + test("the interactive debugger remains the sole debug surface", async () => { const debugSession = { settings: { get: (key: string) => key === "runtime.enabled" || key === "debug.enabled" }, getRuntimeService: () => undefined, } as unknown as ToolSession; const debugTool = await BUILTIN_TOOLS.debug(debugSession); expect(debugTool?.name).toBe("debug"); - expect(debugTool?.label).not.toBe("Runtime Debug"); - const runtimeDebug = await BUILTIN_TOOLS.runtime_debug(stubSession(true)); - expect(runtimeDebug?.name).toBe("runtime_debug"); - // The registry is keyed by name, so a second `debug` entry could only appear - // as a duplicate in the name list — assert there are none at all. + expect(BUILTIN_TOOL_NAMES).not.toContain("runtime_debug"); expect(new Set(BUILTIN_TOOL_NAMES).size).toBe(BUILTIN_TOOL_NAMES.length); - // And no launch tool is reachable under a legacy alias for another tool. - for (const name of LAUNCH_TOOLS) expect(normalizeToolName(name)).toBe(name); }); test("launch tools are discoverable, exec-approved, and never say the product name", async () => { @@ -77,7 +92,7 @@ describe("runtime tool registry", () => { }); describe("JVM tool registry", () => { - test("the five specialized JVM tools are builtin names and jvm_run is removed", () => { + test("the four specialized JVM tools are builtin names and jvm_run is removed", () => { for (const name of JVM_TOOLS) expect(BUILTIN_TOOL_NAMES).toContain(name); expect(BUILTIN_TOOL_NAMES).not.toContain("jvm_run"); }); @@ -109,23 +124,3 @@ describe("JVM tool registry", () => { } }); }); - -describe("project_advice registry", () => { - test("is a builtin name, collides with no legacy alias, and is not essential", () => { - expect(BUILTIN_TOOL_NAMES).toContain("project_advice"); - expect(normalizeToolName("project_advice")).toBe("project_advice"); - expect("project_advice" in ESSENTIAL_BUILTIN_TOOL_NAMES).toBe(false); - expect(new Set(BUILTIN_TOOL_NAMES).size).toBe(BUILTIN_TOOL_NAMES.length); - }); - - test("gates on runtime.enabled and is the one read-approved runtime tool", async () => { - expect(await BUILTIN_TOOLS.project_advice(stubSession(false))).toBeNull(); - const tool = await BUILTIN_TOOLS.project_advice(stubSession(true)); - expect(tool?.name).toBe("project_advice"); - expect(tool?.loadMode).toBe("discoverable"); - expect(tool?.approval).toBe("read"); - for (const name of [...RUNTIME_TOOLS, ...JVM_TOOLS]) { - expect((await BUILTIN_TOOLS[name](stubSession(true)))?.approval).toBe("exec"); - } - }); -}); diff --git a/packages/coding-agent/test/runtime-tool-renderers.test.ts b/packages/coding-agent/test/runtime-tool-renderers.test.ts index 83547710c7e..60cdf478a64 100644 --- a/packages/coding-agent/test/runtime-tool-renderers.test.ts +++ b/packages/coding-agent/test/runtime-tool-renderers.test.ts @@ -71,21 +71,17 @@ describe("runtime tool renderers: registration", () => { } }); - it("covers the five core runtime tools, the five specialized JVM tools, and the two job tools", () => { + it("covers runtime execution and analysis, four specialized JVM tools, and one job tool", () => { expect([...RUNTIME_RENDERER_TOOL_NAMES].sort()).toEqual( [ - "build", "check", "insights", "jvm_deps", "jvm_disassemble", "jvm_format", "jvm_jar", - "jvm_javadoc", "profile", - "project_advice", "run", - "runtime_debug", "serve", ].sort(), ); @@ -162,7 +158,7 @@ describe("run", () => { }); }); -describe("check and build", () => { +describe("check", () => { it("shows pass on a clean check and fail on a broken one", () => { expect(settled("check", { content: [{ type: "text", text: "ok" }], details: exec() }, {})).toContain("passed"); const failed = settled( @@ -173,19 +169,6 @@ describe("check and build", () => { expect(failed).toContain("failed"); expect(failed).toContain("exit 2"); }); - - it("shows the requested build targets on the call line and pass/fail on the result", () => { - expect(call("build", { targets: [":jvm", ":native"] })).toContain(":jvm :native"); - expect(call("build", {})).toContain("default targets"); - const text = settled( - "build", - { content: [{ type: "text", text: "BUILD OK" }], details: exec() }, - { targets: [":jvm"] }, - ); - expect(text).toContain("Build"); - expect(text).toContain(":jvm"); - expect(text).toContain("passed"); - }); }); describe("insights and profile", () => { @@ -246,7 +229,7 @@ describe("jvm tools", () => { expect(text).toContain("error: bad"); }); - it("shows the written output path for jvm_jar and jvm_javadoc", () => { + it("shows written output paths for jvm_jar and jvm_deps", () => { const jar = settled( "jvm_jar", { @@ -257,46 +240,25 @@ describe("jvm tools", () => { ); expect(jar).toContain("/tmp/app.jar"); - const javadoc = settled( - "jvm_javadoc", + const deps = settled( + "jvm_deps", { content: [{ type: "text", text: "done" }], - details: exec({ action: "javadoc", phase: "javadoc", output: "/tmp/javadoc-out", entryCount: 7 }), + details: exec({ action: "deps", phase: "deps", output: "/tmp/deps.txt" }), }, - { code: "class Main {}" }, + { path: "Main.java", output: "deps.txt" }, ); - expect(javadoc).toContain("/tmp/javadoc-out"); - expect(javadoc).toContain("7"); + expect(deps).toContain("/tmp/deps.txt"); }); }); -describe("runtime_debug and serve", () => { - it("shows the endpoint and the hub job handle", () => { - const debug = settled( - "runtime_debug", - { - content: [{ type: "text", text: "CDP debugger listening at ws://127.0.0.1:4242/x" }], - details: { - mode: "debug", - jobName: "runtime-debug-cdp-1a2b3c4d", - endpoint: "ws://127.0.0.1:4242/x", - timedOut: false, - startupOutput: "", - argv: ["elide"], - cwd: "/repo", - }, - }, - { path: "app.ts" }, - ); - expect(debug).toContain("ws://127.0.0.1:4242/x"); - expect(debug).toContain("runtime-debug-cdp-1a2b3c4d"); - - const serve = settled( +describe("serve", () => { + it("shows the endpoint, hub job handle, and served directory", () => { + const text = settled( "serve", { content: [{ type: "text", text: "Serving /repo/public at http://127.0.0.1:8080" }], details: { - mode: "serve", jobName: "runtime-serve-9f8e7d6c", endpoint: "http://127.0.0.1:8080", timedOut: false, @@ -307,17 +269,17 @@ describe("runtime_debug and serve", () => { }, { directory: "public" }, ); - expect(serve).toContain("http://127.0.0.1:8080"); - expect(serve).toContain("runtime-serve-9f8e7d6c"); + expect(text).toContain("http://127.0.0.1:8080"); + expect(text).toContain("runtime-serve-9f8e7d6c"); + expect(text).toContain("public"); }); - it("says so when no endpoint was scraped inside the wait window", () => { + it("warns when no endpoint was scraped inside the wait window", () => { const text = settled( "serve", { content: [{ type: "text", text: "no endpoint" }], details: { - mode: "serve", jobName: "runtime-serve-1", timedOut: true, startupOutput: "", @@ -329,101 +291,35 @@ describe("runtime_debug and serve", () => { ); expect(text).toContain("no endpoint"); expect(text).toContain("timed out"); - // A launched-but-unscraped job is not a failure (the process may be fine), - // so the frame warns rather than erroring — but it must not read as clean. - expect(text.startsWith(" ")).toBe(true); - expect(Bun.stripANSI(uiTheme.symbol("status.warning"))).toBeTruthy(); expect(text).toContain(Bun.stripANSI(uiTheme.symbol("status.warning"))); }); - it("keeps the launched target visible once the endpoint replaces the description", () => { - // The settled row takes the endpoint as its description and - // `mergeCallAndResult` removes the call frame above it, so unless the - // target is carried into `meta` the transcript loses the one thing that - // says WHAT is being debugged or served. - const debug = settled( - "runtime_debug", - { - content: [{ type: "text", text: "listening" }], - details: { - mode: "debug", - jobName: "runtime-debug-cdp-1", - endpoint: "ws://127.0.0.1:4242/x", - timedOut: false, - startupOutput: "", - argv: [], - cwd: "/repo", - }, - }, - { path: "src/app.ts", protocol: "cdp" }, - ); - expect(debug).toContain("ws://127.0.0.1:4242/x"); - expect(debug).toContain("src/app.ts"); - - const serve = settled( - "serve", - { - content: [{ type: "text", text: "serving" }], - details: { - mode: "serve", - jobName: "runtime-serve-1", - endpoint: "http://127.0.0.1:8080", - timedOut: false, - startupOutput: "", - argv: [], - cwd: "/repo", - }, - }, - { directory: "public", port: 8080 }, - ); - expect(serve).toContain("http://127.0.0.1:8080"); - expect(serve).toContain("public"); - }); - - it("errors — not warns — when the launch itself failed", () => { - // A rejected launch also has no endpoint, so the endpoint-less warning path - // must not swallow it. + it("errors rather than warns when the launch itself failed", () => { const text = settled( - "runtime_debug", + "serve", { content: [{ type: "text", text: "hub refused to start the process" }], isError: true, details: { - mode: "debug", - jobName: "runtime-debug-cdp-2", + jobName: "runtime-serve-2", timedOut: false, startupOutput: "", argv: [], cwd: "/repo", }, }, - { path: "app.ts" }, + { directory: "public" }, ); expect(text).toContain(Bun.stripANSI(uiTheme.symbol("status.error"))); expect(text).not.toContain(Bun.stripANSI(uiTheme.symbol("status.warning"))); }); - it("shows the target on the call line", () => { - expect(call("runtime_debug", { path: "app.ts", protocol: "dap" })).toContain("app.ts"); - expect(call("runtime_debug", { path: "app.ts", protocol: "dap" })).toContain("dap"); + it("shows the target and port on the call line", () => { expect(call("serve", { directory: "public", port: 9000 })).toContain("public"); expect(call("serve", { directory: "public", port: 9000 })).toContain("9000"); }); }); -describe("project_advice", () => { - it("shows the project name", () => { - expect(call("project_advice", { cwd: "/home/dev/my-project" })).toContain("my-project"); - const text = settled( - "project_advice", - { content: [{ type: "text", text: "Run tests with: elide test" }], details: exec() }, - { cwd: "/home/dev/my-project" }, - ); - expect(text).toContain("my-project"); - expect(text).toContain("Run tests with"); - }); -}); - describe("spilled output", () => { it("shows the artifact reference the central spill attached", () => { const text = settled( @@ -458,17 +354,13 @@ describe("naming rule", () => { const frames = [ call("run", { code: "x" }), call("check", {}), - call("build", { targets: [":jvm"] }), call("insights", { code: "x" }), call("profile", { mode: "cputracing", code: "x" }), - call("project_advice", {}), call("run", { language: "java", code: "class Main {}" }), call("jvm_deps", { path: "app.jar" }), call("jvm_format", { language: "kotlin", code: "fun main(){}" }), call("jvm_disassemble", { language: "java", code: "class Main {}" }), call("jvm_jar", { action: "inspect", jar: "app.jar" }), - call("jvm_javadoc", { code: "class Main {}" }), - call("runtime_debug", { path: "app.ts" }), call("serve", { directory: "public" }), ]; for (const frame of frames) { diff --git a/packages/coding-agent/test/skills.test.ts b/packages/coding-agent/test/skills.test.ts index f7cf490781a..5b8c43e6b37 100644 --- a/packages/coding-agent/test/skills.test.ts +++ b/packages/coding-agent/test/skills.test.ts @@ -41,9 +41,6 @@ const DISABLE_ALL_BUILTIN_SKILLS = { enablePiProject: false, enableAgentsUser: false, enableAgentsProject: false, - // Fork-added: the bundled runtime skills provider is agent-native and has its - // own toggle, so "every built-in source off" must switch it off explicitly. - enableBundled: false, } as const; describe("skills", () => { diff --git a/packages/coding-agent/test/system-prompt-inventory.test.ts b/packages/coding-agent/test/system-prompt-inventory.test.ts index 6d74a4e4066..60c2c51f813 100644 --- a/packages/coding-agent/test/system-prompt-inventory.test.ts +++ b/packages/coding-agent/test/system-prompt-inventory.test.ts @@ -147,6 +147,66 @@ describe("system prompt tool inventory", () => { } as ToolSession; } + it("renders runtime selection as inherent policy only for registered capabilities", async () => { + const runtimeTools = new Map(TOOLS); + for (const name of [ + "run", + "eval", + "check", + "insights", + "profile", + "serve", + "jvm_disassemble", + "jvm_format", + "jvm_jar", + "jvm_deps", + ]) { + runtimeTools.set(name, { + label: name, + description: `${name} description`, + parameters: { type: "object", properties: {} }, + }); + } + const renderWith = async (tools: Map): Promise => { + const { systemPrompt } = await buildSystemPrompt({ + cwd: tempDir, + contextFiles: [], + skills: [], + rules: [], + toolNames: [...tools.keys()], + tools, + workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, + nativeTools: true, + inlineToolDescriptors: false, + }); + return systemPrompt.join("\n\n"); + }; + + const inherent = await renderWith(runtimeTools); + const coreTools = new Map( + [...runtimeTools].filter(([name]) => ["read", "bash", "run", "eval", "check"].includes(name)), + ); + const core = await renderWith(coreTools); + const shellOnly = await renderWith(TOOLS); + + expect(inherent).toContain("INHERENT CAPABILITIES"); + expect(inherent).toContain("Direct program execution"); + expect(inherent).toContain("Persistent exploration"); + expect(inherent).toContain("Validation without artifacts"); + expect(inherent).not.toContain("Artifact production"); + expect(inherent).not.toContain("Project-declared build/run guidance"); + expect(inherent).toContain("JVM bytecode disassembly"); + expect(inherent).toContain("NEVER invoke the runtime binary through"); + expect(inherent).toContain("Standalone Java/Kotlin"); + expect(inherent).toContain("do not repeat equivalent"); + expect(shellOnly).not.toContain("## Runtime execution"); + expect(shellOnly).toContain("## Engineering method"); + + const corePrefix = core.slice(0, core.indexOf("# Skills & Rules")); + const specializedPrefix = inherent.slice(0, inherent.indexOf("# Skills & Rules")); + expect(specializedPrefix).toBe(corePrefix); + }); + it("preserves the one-argument full metadata builder", () => { const metadata = buildSystemPromptToolMetadata(new Map([[SDK_TOOL.name, SDK_TOOL]])); diff --git a/packages/coding-agent/test/telemetry-sink.test.ts b/packages/coding-agent/test/telemetry-sink.test.ts index 83f31ba0e1c..78491358a5f 100644 --- a/packages/coding-agent/test/telemetry-sink.test.ts +++ b/packages/coding-agent/test/telemetry-sink.test.ts @@ -148,6 +148,55 @@ describe("otlp sink", () => { await provider.shutdown(); }); + it("exports bounded runtime call count and wall-clock duration dimensions", async () => { + const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE); + const provider = new MeterProvider({ + readers: [new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 })], + }); + const recorder = new AuraMetricRecorder(provider.getMeter("test")); + const logs: Array<{ eventName: string; attributes: Record }> = []; + const unregister = registerOtlpSink({ + recorder, + emitLog: (_level, _body, attributes, eventName) => logs.push({ eventName, attributes }), + }); + + emitTelemetryEvent({ + type: "runtime.call.completed", + sessionId: "s1", + method: "runtime/run", + language: "python", + outcome: "error", + durationMs: 125, + exitCode: 2, + killed: false, + errorType: "non_zero_exit", + }); + + const metrics = await collect(exporter, provider); + const calls = metrics.find(metric => metric.descriptor.name === "aura.runtime.calls"); + const duration = metrics.find(metric => metric.descriptor.name === "aura.runtime.duration"); + const attributes = calls?.dataPoints[0]?.attributes; + expect(calls?.dataPoints[0]?.value).toBe(1); + expect(duration?.dataPoints[0]?.value).toMatchObject({ sum: 125, count: 1 }); + expect(attributes).toMatchObject({ + "aura.runtime.method": "runtime/run", + "aura.runtime.language": "python", + "aura.runtime.outcome": "error", + }); + expect(attributes).not.toHaveProperty("error.message"); + expect(logs).toContainEqual({ + eventName: "aura.runtime.call.completed", + attributes: expect.objectContaining({ + "session.id": "s1", + "aura.runtime.duration_ms": 125, + "error.type": "non_zero_exit", + }), + }); + + unregister(); + await provider.shutdown(); + }); + it("counts error.reported with phase attribute", async () => { const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE); const provider = new MeterProvider({ diff --git a/packages/metaharness/README.md b/packages/metaharness/README.md index 55804997488..bfec318ae78 100644 --- a/packages/metaharness/README.md +++ b/packages/metaharness/README.md @@ -36,7 +36,7 @@ direct-vs-runtime microbenchmarks from the repository root: bun run bench:runtime ``` -The command materializes 12 deterministic Harbor tasks, runs paired arms with +The command materializes 10 deterministic Harbor tasks, runs paired arms with alternating AB/BA order, and writes both a frozen launch manifest and comparison report: @@ -48,12 +48,12 @@ the runtime tools are added to the second arm. The report pairs outcomes within task strata, uses a deterministic 10,000-sample task bootstrap for 95% confidence intervals, reports end-to-end arm time separately from trial time, and renders the pre-registered decision. -The runtime arm always exposes the production-essential `run`, `check`, and -`build` tools, then adds only the discoverable runtime tools applicable to the -task (for example, `insights` for instrumentation or `jvm_disassemble` for -bytecode inspection). The frozen manifest records the exact runtime tool list -per task. This keeps the causal treatment realistic without charging every -trial for unrelated debugger, profiler, JVM, server, and advisory tool schemas. +The runtime arm always exposes the production-essential `run` and `check` +tools, then adds only the discoverable runtime tools applicable to the task +(for example, `insights` for instrumentation or `jvm_disassemble` for bytecode +inspection). The frozen manifest records the exact runtime tool list per task. +This keeps the causal treatment realistic without charging every trial for +unrelated debugger, profiler, JVM, and server tool schemas. Before any model trial, the orchestrator builds the generated TypeScript task image and runs its verifier against a deterministic valid solution. Every task @@ -63,6 +63,50 @@ independent of the agent install. The smoke covers `node:fs`, TypeScript syntax, top-level await, and the expected sorted BFS output. A smoke failure stops the campaign. The frozen manifest records this verifier Bun's version and SHA-256. +To add a pinned vanilla OMP whole-product control, pass both its revision and +compiled binary: + +```bash +bun run bench:runtime --agent-only \ + --historical-revision= \ + --historical-binary=/absolute/path/to/omp-linux-x64 +``` + +The historical arm receives only the Bash-baseline tools and is reported +separately because it changes the whole product revision. The local Harbor +adapter writes gateway routing under both `~/.aura/agent` and `~/.omp/agent`, +so this control uses the same host-side credentials without exposing them to +the task container. + +### Inherent capability prompt comparison + +Run the current-configuration smoke while tuning inherent runtime and +engineering policy: + +```bash +bun run bench:inherent --prefix=inherent-smoke +``` + +This runs `typescript-execution` and `jvm-dependencies` once against current +source. The report requires every task to pass, select its task-specific runtime +tool before `bash`, and load no promoted runtime/core-workflow skill. + +For a matched comparison, provide a pinned pre-change binary: + +```bash +bun run bench:inherent --legacy-binary=/absolute/path/to/aura-linux-x64 \ + --prefix=inherent-capabilities +``` + +Comparison mode runs each `(task, attempt)` as its own one-attempt job and +alternates arm order across three attempts. Both revisions use the same model, +reasoning level, fixtures, and task-specific tool lists. It adds no-increase +gates for median paired tool-call and input-token deltas. Reports record a +SHA-256 for the current treatment sources and, in +comparison mode, the pinned legacy binary. Set `AURA_LEGACY_BINARY` instead of +passing the flag when automating repeat runs. + + An adapter decision run additionally compares independent process and embedded runtime services: diff --git a/packages/metaharness/agent/omp_local.py b/packages/metaharness/agent/omp_local.py index cfc00333022..3d69220a945 100644 --- a/packages/metaharness/agent/omp_local.py +++ b/packages/metaharness/agent/omp_local.py @@ -14,8 +14,9 @@ external deps + the platform native addon, and run `bun .../dist/cli.js`. * binary (`--binary`): a self-contained compiled omp binary is uploaded. -Auth never enters the container: a generated `~/.aura/agent/models.yml` routes the -configured providers' `baseUrl` at the host's pm2 auth-gateway (default +Auth never enters the container: generated `models.yml` files in Aura's and +vanilla OMP's config directories route the configured providers' `baseUrl` at the +host's pm2 auth-gateway (default `http://host.docker.internal:4000`, `transport: pi-native`), so the gateway resolves credentials host-side. No provider API keys are passed in. @@ -322,7 +323,7 @@ async def install(self, environment: BaseEnvironment) -> None: else: self._cli = await self._install_local(environment) - # 3) Auth + model config under $HOME/.aura/agent. + # 3) Auth + model config for both Aura and vanilla OMP binaries. if self._gateway_on: # Gateway routing — no provider keys ever enter the container. await self._write_models_yaml(environment) @@ -453,8 +454,9 @@ async def _write_models_yaml(self, environment: BaseEnvironment) -> None: await self.exec_as_agent( environment, command=( - f'mkdir -p "$HOME/.aura/agent"; ' - f'cp {shlex.quote(staged)} "$HOME/.aura/agent/models.yml"' + f'mkdir -p "$HOME/.aura/agent" "$HOME/.omp/agent"; ' + f'cp {shlex.quote(staged)} "$HOME/.aura/agent/models.yml"; ' + f'cp {shlex.quote(staged)} "$HOME/.omp/agent/models.yml"' ), ) @@ -474,7 +476,7 @@ def _generate_models_yaml(self) -> str: return "\n".join(lines) async def _write_config(self, environment: BaseEnvironment) -> None: - """Write $HOME/.aura/agent/config.yml: the web_search toggle. + """Write the web_search toggle for Aura and vanilla OMP. web_search can't authenticate through the gateway, so it's off by default. """ @@ -489,8 +491,9 @@ async def _write_config(self, environment: BaseEnvironment) -> None: await self.exec_as_agent( environment, command=( - f'mkdir -p "$HOME/.aura/agent"; ' - f'cp {shlex.quote(_CONFIG_DST)} "$HOME/.aura/agent/config.yml"' + f'mkdir -p "$HOME/.aura/agent" "$HOME/.omp/agent"; ' + f'cp {shlex.quote(_CONFIG_DST)} "$HOME/.aura/agent/config.yml"; ' + f'cp {shlex.quote(_CONFIG_DST)} "$HOME/.omp/agent/config.yml"' ), ) diff --git a/packages/metaharness/agent/test_omp_local.py b/packages/metaharness/agent/test_omp_local.py index 7bd07342114..afd79f2fbaf 100644 --- a/packages/metaharness/agent/test_omp_local.py +++ b/packages/metaharness/agent/test_omp_local.py @@ -7,7 +7,7 @@ class OmpLocalConfigPathTest(unittest.IsolatedAsyncioTestCase): - async def test_gateway_configuration_is_written_to_aura_config_directory(self) -> None: + async def test_gateway_configuration_is_written_to_aura_and_omp_config_directories(self) -> None: agent = OmpLocal.__new__(OmpLocal) agent._models_yaml_path = "" agent._gateway_providers = ["openai-codex"] @@ -31,6 +31,12 @@ async def capture_command(self, environment, *, command, **kwargs): self.assertTrue( any('"$HOME/.aura/agent/config.yml"' in command for command in commands) ) + self.assertTrue( + any('"$HOME/.omp/agent/models.yml"' in command for command in commands) + ) + self.assertTrue( + any('"$HOME/.omp/agent/config.yml"' in command for command in commands) + ) if __name__ == "__main__": diff --git a/packages/metaharness/package.json b/packages/metaharness/package.json index 7ed953fc1df..62cef5789c0 100644 --- a/packages/metaharness/package.json +++ b/packages/metaharness/package.json @@ -21,6 +21,7 @@ "lint": "biome lint .", "serve": "bun run src/server.ts", "bench:edit": "bun adapters/edit/cli.ts", + "bench:inherent": "bun src/inherent-capability-benchmark.ts", "dev": "bun --hot src/server.ts", "bench:runtime": "bun src/runtime-benchmark.ts", "test": "bun test" diff --git a/packages/metaharness/src/inherent-capability-benchmark.test.ts b/packages/metaharness/src/inherent-capability-benchmark.test.ts new file mode 100644 index 00000000000..2d371a37502 --- /dev/null +++ b/packages/metaharness/src/inherent-capability-benchmark.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it } from "bun:test"; +import { + analyzeInherentBenchmark, + buildInherentBenchmarkLaunches, + INHERENT_BENCHMARK_TASK_IDS, + inherentBenchmarkToolsForTask, + parseInherentBenchmarkCli, + runInherentTelemetryProbe, + scanInherentTranscript, +} from "./inherent-capability-benchmark"; +import type { ArmSummary, RuntimeTaskMeasurement } from "./runtime-benchmark"; + +function summary( + arm: "baseline" | "runtime", + options: { pass?: number; toolCalls?: number[]; inputTokens?: number[] } = {}, +): ArmSummary { + const toolCalls = options.toolCalls ?? [4, 4, 5, 5, 6, 6]; + const inputTokens = options.inputTokens ?? [100, 100, 110, 110, 120, 120]; + const trials = toolCalls.map((calls, index) => ({ + taskId: INHERENT_BENCHMARK_TASK_IDS[index % INHERENT_BENCHMARK_TASK_IDS.length], + trialName: `trial-${index}`, + status: "pass" as const, + detail: "", + durationMs: 100, + tokIn: inputTokens[index], + tokOut: 10, + tokCache: 0, + costUsd: 0, + toolCalls: calls, + runtimeUsed: arm === "runtime", + })); + const taskMeasurements = INHERENT_BENCHMARK_TASK_IDS.map(taskId => ({ + taskId, + group: taskId === "typescript-execution" ? ("execution" as const) : ("jvm" as const), + trials: trials.filter(trial => trial.taskId === taskId), + })) satisfies RuntimeTaskMeasurement[]; + const pass = options.pass ?? trials.length; + return { + arm, + tasks: INHERENT_BENCHMARK_TASK_IDS.length, + trials: trials.length, + completedTrials: trials.length, + pass, + fail: trials.length - pass, + error: 0, + costUsd: 0, + tokIn: inputTokens.reduce((sum, value) => sum + value, 0), + tokOut: trials.length * 10, + tokCache: 0, + durationMs: trials.length * 100, + elapsedMs: trials.length * 100, + medianDurationMs: 100, + toolCalls: toolCalls.reduce((sum, value) => sum + value, 0), + runtimeTasks: arm === "runtime" ? INHERENT_BENCHMARK_TASK_IDS.length : 0, + runtimeTrials: arm === "runtime" ? trials.length : 0, + taskMeasurements, + }; +} + +describe("inherent capability benchmark", () => { + it("runs one-attempt jobs in alternating matched arm order", () => { + const options = parseInherentBenchmarkCli(["--legacy-binary=/tmp/legacy-omp", "--prefix=inherent-test"], {}); + const launches = buildInherentBenchmarkLaunches(options, "/tmp/tasks"); + expect(launches).toHaveLength(12); + expect(launches.map(launch => [launch.attempt, launch.arm, launch.taskId])).toEqual([ + [1, "legacy", "typescript-execution"], + [1, "inherent", "typescript-execution"], + [1, "inherent", "jvm-dependencies"], + [1, "legacy", "jvm-dependencies"], + [2, "inherent", "typescript-execution"], + [2, "legacy", "typescript-execution"], + [2, "legacy", "jvm-dependencies"], + [2, "inherent", "jvm-dependencies"], + [3, "legacy", "typescript-execution"], + [3, "inherent", "typescript-execution"], + [3, "inherent", "jvm-dependencies"], + [3, "legacy", "jvm-dependencies"], + ]); + for (const launch of launches) { + expect(launch.args).toContain("--attempts=1"); + expect(launch.args).toContain(`--agent-arg=${inherentBenchmarkToolsForTask(launch.taskId).join(",")}`); + } + expect(launches[0].args).toContain("--binary=/tmp/legacy-omp"); + expect(launches[1].args).toContain("--install=source"); + }); + + it("defaults to a one-attempt current-only smoke", () => { + const options = parseInherentBenchmarkCli(["--prefix=inherent-smoke"], {}); + expect(options.attempts).toBe(1); + expect(options.legacyBinary).toBeUndefined(); + const launches = buildInherentBenchmarkLaunches(options, "/tmp/tasks"); + expect(launches.map(launch => [launch.arm, launch.taskId])).toEqual([ + ["inherent", "typescript-execution"], + ["inherent", "jvm-dependencies"], + ]); + }); + + it("extracts first execution choice and promoted skill loads from emitted transcript events", () => { + const transcript = [ + JSON.stringify({ type: "tool_execution_start", toolName: "read", args: { path: "/app/events.jsonl" } }), + JSON.stringify({ type: "tool_execution_start", toolName: "run", args: { path: "/app/aggregate.py" } }), + JSON.stringify({ type: "tool_execution_start", toolName: "read", args: { path: "skill://runtime" } }), + JSON.stringify({ + type: "tool_execution_start", + toolName: "read", + args: { path: "skill://superpowers:receiving-code-review" }, + }), + JSON.stringify({ type: "tool_execution_start", toolName: "read", args: { path: "skill://frontend-design" } }), + ].join("\n"); + expect(scanInherentTranscript(transcript)).toEqual({ firstCapabilityTool: "run", coreSkillLoads: 2 }); + }); + + it("passes only when behavior improves without tool-call or token regression", () => { + const legacy = summary("baseline", { + toolCalls: [6, 6, 7, 7, 8, 8], + inputTokens: [150, 150, 160, 160, 170, 170], + }); + const inherent = summary("runtime"); + const traces = Array.from({ length: 3 }, () => [ + { taskId: "typescript-execution", facts: { firstCapabilityTool: "run" as const, coreSkillLoads: 0 } }, + { taskId: "jvm-dependencies", facts: { firstCapabilityTool: "jvm_deps" as const, coreSkillLoads: 0 } }, + ]).flat(); + const analysis = analyzeInherentBenchmark(legacy, inherent, traces); + expect(analysis.verdict).toBe("pass"); + expect(analysis).toMatchObject({ + inherentPassRate: 1, + firstExecutionSelectionRate: 1, + coreSkillLoads: 0, + }); + }); + + it("fails on skill loading, wrong execution selection, or efficiency regression", () => { + const legacy = summary("baseline"); + const inherent = summary("runtime", { + pass: 5, + toolCalls: [7, 7, 8, 8, 9, 9], + inputTokens: [130, 130, 140, 140, 150, 150], + }); + const analysis = analyzeInherentBenchmark(legacy, inherent, [ + { taskId: "typescript-execution", facts: { firstCapabilityTool: "bash", coreSkillLoads: 1 } }, + ]); + expect(analysis.verdict).toBe("fail"); + expect(analysis.reasons).toEqual([ + "inherent arm did not pass every trial", + "inherent arm is missing transcript evidence", + "inherent arm did not select the task-specific runtime tool before bash in every trial", + "inherent arm loaded a promoted runtime or core workflow skill", + "median paired tool calls increased", + "median paired input tokens increased", + ]); + }); + + it("fails when any inherent trial lacks runtime adoption", () => { + const inherent = summary("runtime"); + inherent.runtimeTrials -= 1; + const traces = Array.from({ length: 3 }, () => [ + { taskId: "typescript-execution", facts: { firstCapabilityTool: "run" as const, coreSkillLoads: 0 } }, + { taskId: "jvm-dependencies", facts: { firstCapabilityTool: "jvm_deps" as const, coreSkillLoads: 0 } }, + ]).flat(); + const analysis = analyzeInherentBenchmark(undefined, inherent, traces); + expect(analysis.verdict).toBe("fail"); + expect(analysis.reasons).toContain("inherent arm did not use a runtime tool in every trial"); + }); + + it("preflights bounded success and failure runtime telemetry", async () => { + const probe = await runInherentTelemetryProbe(); + expect(probe.success).toMatchObject({ + sessionId: "benchmark-preflight", + language: "python", + outcome: "ok", + exitCode: 0, + }); + expect(probe.failure).toMatchObject({ + sessionId: "benchmark-preflight", + language: "python", + outcome: "error", + exitCode: 2, + errorType: "non_zero_exit", + }); + expect(probe.success.durationMs).toBeGreaterThanOrEqual(0); + expect(probe.failure.durationMs).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/packages/metaharness/src/inherent-capability-benchmark.ts b/packages/metaharness/src/inherent-capability-benchmark.ts new file mode 100755 index 00000000000..790c29dba19 --- /dev/null +++ b/packages/metaharness/src/inherent-capability-benchmark.ts @@ -0,0 +1,517 @@ +#!/usr/bin/env bun +import * as fs from "node:fs"; +import * as path from "node:path"; +import { isRecord } from "@oh-my-pi/pi-utils"; +import { type ArmSummary, type RuntimeTaskMeasurement, runtimeToolsForTask, summarizeArm } from "./runtime-benchmark"; +import { materializeRuntimeTasks, smokeTypeScriptTaskVerifier } from "./runtime-benchmark-suite"; + +const REPO_ROOT = path.resolve(import.meta.dir, "..", "..", ".."); +const PKG_DIR = path.resolve(import.meta.dir, ".."); +const DEFAULT_JOBS_DIR = path.join(REPO_ROOT, "runs", "harbor"); +const INHERENT_TREATMENT_FILES = [ + "packages/coding-agent/src/prompts/system/system-prompt.md", + "packages/coding-agent/src/prompts/tools/runtime-run.md", + "packages/coding-agent/src/prompts/tools/runtime-check.md", + "packages/coding-agent/src/prompts/tools/runtime-insights.md", + "packages/coding-agent/src/prompts/tools/runtime-profile.md", + "packages/coding-agent/src/prompts/tools/runtime-serve.md", + "packages/coding-agent/src/prompts/tools/jvm-disassemble.md", + "packages/coding-agent/src/prompts/tools/jvm-format.md", + "packages/coding-agent/src/prompts/tools/jvm-jar.md", + "packages/coding-agent/src/prompts/tools/jvm-deps.md", + "packages/coding-agent/src/tools/builtin-names.ts", + "packages/coding-agent/src/tools/essential-tools.ts", + "packages/coding-agent/src/tools/index.ts", + "packages/coding-agent/src/discovery/claude-plugins.ts", + "packages/coding-agent/src/discovery/index.ts", + "packages/coding-agent/src/capability/skill.ts", + "packages/coding-agent/src/config/settings-schema.ts", +] as const; +const CORE_SKILLS: Readonly> = { + runtime: true, + insights: true, + profiling: true, + jvm: true, + "stateful-debugger": true, + "using-superpowers": true, + brainstorming: true, + "writing-plans": true, + "test-driven-development": true, + "systematic-debugging": true, + "verification-before-completion": true, + "dispatching-parallel-agents": true, + "subagent-driven-development": true, + "using-git-worktrees": true, + "requesting-code-review": true, + "receiving-code-review": true, + "finishing-a-development-branch": true, + "executing-plans": true, +}; + +export const INHERENT_BENCHMARK_TASK_IDS = ["typescript-execution", "jvm-dependencies"] as const; +export type InherentBenchmarkTaskId = (typeof INHERENT_BENCHMARK_TASK_IDS)[number]; + +export function inherentBenchmarkToolsForTask(taskId: InherentBenchmarkTaskId): string[] { + return runtimeToolsForTask(taskId); +} + +export type InherentBenchmarkArm = "legacy" | "inherent"; + +export interface InherentBenchmarkOptions { + model: string; + thinking: string; + attempts: number; + prefix: string; + jobsDir: string; + gatewayUrl: string; + hostNetwork: boolean; + legacyBinary?: string; +} + +export interface InherentBenchmarkLaunch { + arm: InherentBenchmarkArm; + taskId: (typeof INHERENT_BENCHMARK_TASK_IDS)[number]; + attempt: number; + jobName: string; + args: string[]; +} + +export interface InherentTranscriptFacts { + firstCapabilityTool: "run" | "check" | "jvm_deps" | "bash" | undefined; + coreSkillLoads: number; +} + +export interface InherentBenchmarkAnalysis { + verdict: "pass" | "fail"; + comparison: boolean; + inherentPassRate: number; + firstExecutionSelectionRate: number; + coreSkillLoads: number; + legacyMedianToolCalls: number; + inherentMedianToolCalls: number; + legacyMedianInputTokens: number; + inherentMedianInputTokens: number; + reasons: string[]; +} + +export interface InherentTelemetryFact { + sessionId: string; + language: string; + outcome: string; + durationMs: number; + exitCode: number; + errorType?: string; +} + +export interface InherentTelemetryProbe { + success: InherentTelemetryFact; + failure: InherentTelemetryFact; +} + +interface InherentBenchmarkIdentities { + currentSourceSha256: string; + legacyBinarySha256?: string; +} + +export function buildInherentBenchmarkLaunches( + opts: InherentBenchmarkOptions, + taskRoot: string, +): InherentBenchmarkLaunch[] { + const launches: InherentBenchmarkLaunch[] = []; + for (let attempt = 1; attempt <= opts.attempts; attempt++) { + for (const [index, taskId] of INHERENT_BENCHMARK_TASK_IDS.entries()) { + const legacyFirst = (attempt + index) % 2 === 1; + const armOrder: readonly InherentBenchmarkArm[] = opts.legacyBinary + ? legacyFirst + ? ["legacy", "inherent"] + : ["inherent", "legacy"] + : ["inherent"]; + for (const arm of armOrder) { + const jobName = `${opts.prefix}-a${attempt}-${arm === "legacy" ? "baseline" : "runtime"}-${taskId}`; + const args = [ + `--path=${path.join(taskRoot, taskId)}`, + arm === "legacy" ? `--binary=${opts.legacyBinary}` : "--install=source", + `--model=${opts.model}`, + `--thinking=${opts.thinking}`, + "--attempts=1", + "--tasks=1", + "--concurrency=1", + `--jobs-dir=${opts.jobsDir}`, + `--gateway-url=${opts.gatewayUrl}`, + `--job-name=${jobName}`, + "--agent-arg=--tools", + `--agent-arg=${inherentBenchmarkToolsForTask(taskId).join(",")}`, + ]; + if (opts.hostNetwork) args.push("--host-network"); + launches.push({ arm, taskId, attempt, jobName, args }); + } + } + } + return launches; +} + +/** Extract prompt-architecture signals from a runner JSONL transcript. */ +export function scanInherentTranscript(content: string): InherentTranscriptFacts { + let firstCapabilityTool: InherentTranscriptFacts["firstCapabilityTool"]; + let coreSkillLoads = 0; + for (const line of content.split("\n")) { + if (!line.startsWith("{")) continue; + try { + const event: unknown = JSON.parse(line); + if (!isRecord(event) || event.type !== "tool_execution_start" || typeof event.toolName !== "string") continue; + if ( + firstCapabilityTool === undefined && + (event.toolName === "run" || + event.toolName === "check" || + event.toolName === "jvm_deps" || + event.toolName === "bash") + ) { + firstCapabilityTool = event.toolName; + } + if (event.toolName !== "read" || !isRecord(event.args) || typeof event.args.path !== "string") continue; + const match = /^skill:\/\/(?:superpowers:)?([^/:?#]+)/.exec(event.args.path); + if (match?.[1] && CORE_SKILLS[match[1]]) coreSkillLoads++; + } catch {} + } + return { firstCapabilityTool, coreSkillLoads }; +} + +function parseTelemetryFact(value: unknown): InherentTelemetryFact { + if ( + !isRecord(value) || + typeof value.sessionId !== "string" || + typeof value.language !== "string" || + typeof value.outcome !== "string" || + typeof value.durationMs !== "number" || + typeof value.exitCode !== "number" || + (value.errorType !== undefined && typeof value.errorType !== "string") + ) { + throw new Error("runtime telemetry preflight returned invalid evidence"); + } + return { + sessionId: value.sessionId, + language: value.language, + outcome: value.outcome, + durationMs: value.durationMs, + exitCode: value.exitCode, + errorType: value.errorType, + }; +} + +export async function runInherentTelemetryProbe(): Promise { + const script = path.join(REPO_ROOT, "packages", "coding-agent", "scripts", "runtime-telemetry-preflight.ts"); + const child = Bun.spawn(["bun", script], { + cwd: REPO_ROOT, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + Bun.readableStreamToText(child.stdout), + Bun.readableStreamToText(child.stderr), + child.exited, + ]); + if (exitCode !== 0) throw new Error(`runtime telemetry preflight exited ${exitCode}: ${stderr.trim()}`); + let value: unknown; + try { + value = JSON.parse(stdout); + } catch { + throw new Error("runtime telemetry preflight returned invalid JSON"); + } + if (!isRecord(value)) throw new Error("runtime telemetry preflight returned invalid evidence"); + return { success: parseTelemetryFact(value.success), failure: parseTelemetryFact(value.failure) }; +} + +function median(values: readonly number[]): number { + if (values.length === 0) return 0; + const sorted = values.toSorted((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; +} + +function benchmarkPairs(summary: ArmSummary | undefined): Map { + const pairs = new Map(); + if (!summary) return pairs; + for (const task of summary.taskMeasurements) { + for (const trial of task.trials) { + const separator = trial.trialName.indexOf(":"); + const attempt = separator < 0 ? trial.trialName : trial.trialName.slice(0, separator); + pairs.set(`${task.taskId}:${attempt}`, { toolCalls: trial.toolCalls, tokIn: trial.tokIn }); + } + } + return pairs; +} + +export function analyzeInherentBenchmark( + legacy: ArmSummary | undefined, + inherent: ArmSummary, + inherentTranscripts: ReadonlyArray<{ taskId: string; facts: InherentTranscriptFacts }>, +): InherentBenchmarkAnalysis { + const comparison = legacy !== undefined; + const inherentPassRate = inherent.trials === 0 ? 0 : inherent.pass / inherent.trials; + const expectedFirstTool: Readonly> = { + "typescript-execution": "run", + "jvm-dependencies": "jvm_deps", + }; + const selectedCorrectly = inherentTranscripts.filter( + entry => + INHERENT_BENCHMARK_TASK_IDS.includes(entry.taskId as InherentBenchmarkTaskId) && + entry.facts.firstCapabilityTool === expectedFirstTool[entry.taskId as InherentBenchmarkTaskId], + ).length; + const firstExecutionSelectionRate = + inherentTranscripts.length === 0 ? 0 : selectedCorrectly / inherentTranscripts.length; + const coreSkillLoads = inherentTranscripts.reduce((sum, entry) => sum + entry.facts.coreSkillLoads, 0); + const legacyToolCalls = legacy?.taskMeasurements.flatMap(task => task.trials.map(trial => trial.toolCalls)) ?? []; + const inherentToolCalls = inherent.taskMeasurements.flatMap(task => task.trials.map(trial => trial.toolCalls)); + const legacyInputTokens = legacy?.taskMeasurements.flatMap(task => task.trials.map(trial => trial.tokIn)) ?? []; + const inherentInputTokens = inherent.taskMeasurements.flatMap(task => task.trials.map(trial => trial.tokIn)); + const legacyMedianToolCalls = median(legacyToolCalls); + const inherentMedianToolCalls = median(inherentToolCalls); + const legacyMedianInputTokens = median(legacyInputTokens); + const inherentMedianInputTokens = median(inherentInputTokens); + const reasons: string[] = []; + if (inherent.trials === 0 || inherent.completedTrials !== inherent.trials) + reasons.push("inherent arm has missing or incomplete trials"); + if (inherentPassRate !== 1) reasons.push("inherent arm did not pass every trial"); + if (inherent.runtimeTrials !== inherent.trials) + reasons.push("inherent arm did not use a runtime tool in every trial"); + if (inherentTranscripts.length !== inherent.trials) reasons.push("inherent arm is missing transcript evidence"); + if (firstExecutionSelectionRate !== 1) + reasons.push("inherent arm did not select the task-specific runtime tool before bash in every trial"); + if (coreSkillLoads !== 0) reasons.push("inherent arm loaded a promoted runtime or core workflow skill"); + if (comparison) { + const legacyPairs = benchmarkPairs(legacy); + const inherentPairs = benchmarkPairs(inherent); + const toolCallDeltas: number[] = []; + const inputTokenDeltas: number[] = []; + for (const [pair, current] of inherentPairs) { + const baseline = legacyPairs.get(pair); + if (!baseline) continue; + toolCallDeltas.push(current.toolCalls - baseline.toolCalls); + inputTokenDeltas.push(current.tokIn - baseline.tokIn); + } + if (legacyPairs.size !== inherentPairs.size || toolCallDeltas.length !== inherentPairs.size) + reasons.push("comparison arm is missing matched task-attempt evidence"); + if (median(toolCallDeltas) > 0) reasons.push("median paired tool calls increased"); + if (median(inputTokenDeltas) > 0) reasons.push("median paired input tokens increased"); + } + return { + verdict: reasons.length === 0 ? "pass" : "fail", + comparison, + inherentPassRate, + firstExecutionSelectionRate, + coreSkillLoads, + legacyMedianToolCalls, + inherentMedianToolCalls, + legacyMedianInputTokens, + inherentMedianInputTokens, + reasons, + }; +} + +export function parseInherentBenchmarkCli( + argv: string[], + env: NodeJS.ProcessEnv = process.env, +): InherentBenchmarkOptions { + const envLegacyBinary = env.AURA_LEGACY_BINARY?.trim() || undefined; + const opts: InherentBenchmarkOptions = { + model: "openai-codex/gpt-5.6-sol", + thinking: "xhigh", + attempts: 1, + prefix: `inherent${Date.now()}`, + jobsDir: DEFAULT_JOBS_DIR, + gatewayUrl: "http://127.0.0.1:4000", + hostNetwork: true, + legacyBinary: envLegacyBinary, + }; + let attemptsExplicit = false; + for (let index = 0; index < argv.length; index++) { + const [flag, inline] = argv[index].split("=", 2); + const take = () => inline ?? argv[++index]; + switch (flag) { + case "--model": + opts.model = take(); + break; + case "--thinking": + opts.thinking = take(); + break; + case "--attempts": + opts.attempts = Number(take()); + attemptsExplicit = true; + break; + case "--prefix": + opts.prefix = take(); + break; + case "--jobs-dir": + opts.jobsDir = path.resolve(take()); + break; + case "--gateway-url": + opts.gatewayUrl = take(); + break; + case "--legacy-binary": + opts.legacyBinary = path.resolve(take()); + break; + case "--no-host-network": + opts.hostNetwork = false; + break; + default: + throw new Error(`unknown inherent benchmark flag: ${flag}`); + } + } + if (opts.legacyBinary && !attemptsExplicit) opts.attempts = 3; + if (!Number.isSafeInteger(opts.attempts) || opts.attempts < 1) + throw new Error("--attempts must be a positive integer"); + return opts; +} + +async function readBenchmarkIdentities(opts: InherentBenchmarkOptions): Promise { + const sourceHasher = new Bun.CryptoHasher("sha256"); + for (const relativePath of INHERENT_TREATMENT_FILES) { + sourceHasher.update(relativePath); + sourceHasher.update(await Bun.file(path.join(REPO_ROOT, relativePath)).bytes()); + } + const legacyBinarySha256 = opts.legacyBinary + ? new Bun.CryptoHasher("sha256").update(await Bun.file(opts.legacyBinary).bytes()).digest("hex") + : undefined; + return { currentSourceSha256: sourceHasher.digest("hex"), legacyBinarySha256 }; +} + +export function summarizeInherentArm(opts: InherentBenchmarkOptions, arm: InherentBenchmarkArm): ArmSummary { + const summaries = Array.from({ length: opts.attempts }, (_, index) => + summarizeArm(opts.jobsDir, `${opts.prefix}-a${index + 1}`, arm === "legacy" ? "baseline" : "runtime", [ + ...INHERENT_BENCHMARK_TASK_IDS, + ]), + ); + const taskMeasurements: RuntimeTaskMeasurement[] = INHERENT_BENCHMARK_TASK_IDS.map(taskId => { + const source = summaries[0]?.taskMeasurements.find(task => task.taskId === taskId); + if (!source) throw new Error(`missing ${arm} benchmark task summary: ${taskId}`); + return { + taskId, + group: source.group, + trials: summaries.flatMap((summary, index) => { + const task = summary.taskMeasurements.find(entry => entry.taskId === taskId); + if (!task) throw new Error(`missing ${arm} benchmark task summary: ${taskId}`); + return task.trials.map(trial => ({ ...trial, trialName: `${index + 1}:${trial.trialName}` })); + }), + }; + }); + const trials = taskMeasurements.flatMap(task => task.trials); + return { + arm: arm === "legacy" ? "baseline" : "runtime", + tasks: INHERENT_BENCHMARK_TASK_IDS.length, + trials: trials.length, + completedTrials: trials.length, + pass: trials.filter(trial => trial.status === "pass").length, + fail: trials.filter(trial => trial.status === "fail").length, + error: trials.filter(trial => trial.status === "error").length, + costUsd: trials.reduce((sum, trial) => sum + trial.costUsd, 0), + tokIn: trials.reduce((sum, trial) => sum + trial.tokIn, 0), + tokOut: trials.reduce((sum, trial) => sum + trial.tokOut, 0), + tokCache: trials.reduce((sum, trial) => sum + trial.tokCache, 0), + durationMs: trials.reduce((sum, trial) => sum + trial.durationMs, 0), + elapsedMs: summaries.reduce((sum, summary) => sum + summary.elapsedMs, 0), + medianDurationMs: median(trials.map(trial => trial.durationMs)), + toolCalls: trials.reduce((sum, trial) => sum + trial.toolCalls, 0), + runtimeTasks: taskMeasurements.filter(task => task.trials.some(trial => trial.runtimeUsed)).length, + runtimeTrials: trials.filter(trial => trial.runtimeUsed).length, + taskMeasurements, + }; +} + +function readInherentTranscripts( + opts: InherentBenchmarkOptions, + summary: ArmSummary, +): Array<{ taskId: string; facts: InherentTranscriptFacts }> { + return summary.taskMeasurements.flatMap(task => + task.trials.map(trial => { + const separator = trial.trialName.indexOf(":"); + if (separator < 1) throw new Error(`invalid aggregated trial name: ${trial.trialName}`); + const attempt = trial.trialName.slice(0, separator); + const rawTrialName = trial.trialName.slice(separator + 1); + const transcriptPath = path.join( + opts.jobsDir, + `${opts.prefix}-a${attempt}-runtime-${task.taskId}`, + rawTrialName, + "agent", + "omp.txt", + ); + if (!fs.existsSync(transcriptPath)) + throw new Error(`missing inherent benchmark transcript: ${transcriptPath}`); + return { taskId: task.taskId, facts: scanInherentTranscript(fs.readFileSync(transcriptPath, "utf8")) }; + }), + ); +} + +function formatInherentReport( + opts: InherentBenchmarkOptions, + legacy: ArmSummary | undefined, + analysis: InherentBenchmarkAnalysis, + identities: InherentBenchmarkIdentities, + telemetry: InherentTelemetryProbe, +): string { + const percent = (value: number) => `${(value * 100).toFixed(1)}%`; + const legacyIdentity = opts.legacyBinary + ? `- Legacy binary: \`${opts.legacyBinary}\`\n- Legacy binary SHA-256: \`${identities.legacyBinarySha256}\`\n` + : ""; + return ( + `# Inherent Harness Capability Benchmark\n\n` + + `- Model: \`${opts.model}\`\n` + + `- Attempts per task: ${opts.attempts}\n` + + `- Tasks: ${INHERENT_BENCHMARK_TASK_IDS.map(task => `\`${task}\``).join(", ")}\n` + + `- Task tools: ${INHERENT_BENCHMARK_TASK_IDS.map( + task => + `\`${task}\` = ${inherentBenchmarkToolsForTask(task) + .map(tool => `\`${tool}\``) + .join(", ")}`, + ).join("; ")}\n` + + `- Current treatment SHA-256: \`${identities.currentSourceSha256}\`\n` + + `- Telemetry preflight: \`${telemetry.success.outcome}\` → \`${telemetry.failure.outcome}\` (` + + `${telemetry.success.language}, ${telemetry.success.durationMs.toFixed(3)}/${telemetry.failure.durationMs.toFixed(3)} ms)\n` + + legacyIdentity + + `\n| Signal | Legacy skills | Inherent prompt |\n|---|---:|---:|\n` + + `| Pass rate | ${legacy ? percent(legacy.trials === 0 ? 0 : legacy.pass / legacy.trials) : "—"} | ${percent(analysis.inherentPassRate)} |\n` + + `| Median tool calls | ${legacy ? analysis.legacyMedianToolCalls : "—"} | ${analysis.inherentMedianToolCalls} |\n` + + `| Median input tokens | ${legacy ? analysis.legacyMedianInputTokens : "—"} | ${analysis.inherentMedianInputTokens} |\n` + + `| Correct runtime tool selected first | — | ${percent(analysis.firstExecutionSelectionRate)} |\n` + + `| Promoted skill loads | — | ${analysis.coreSkillLoads} |\n\n` + + `## Verdict: ${analysis.verdict.toUpperCase()}\n\n` + + (analysis.reasons.length === 0 + ? "All gates passed.\n" + : `${analysis.reasons.map(reason => `- ${reason}`).join("\n")}\n`) + ); +} + +async function main(): Promise { + const opts = parseInherentBenchmarkCli(process.argv.slice(2)); + const identities = await readBenchmarkIdentities(opts); + const telemetry = await runInherentTelemetryProbe(); + const taskRoot = path.join(opts.jobsDir, "_bench", opts.prefix, "tasks"); + await materializeRuntimeTasks(taskRoot); + await smokeTypeScriptTaskVerifier(taskRoot); + const launches = buildInherentBenchmarkLaunches(opts, taskRoot); + for (const [index, launch] of launches.entries()) { + process.stdout.write( + `[${index + 1}/${launches.length}] attempt ${launch.attempt} · ${launch.arm} · ${launch.taskId}\n`, + ); + const runner = Bun.spawn(["bun", "src/runner.ts", ...launch.args], { + cwd: PKG_DIR, + stdout: "inherit", + stderr: "inherit", + }); + const exitCode = await runner.exited; + if (exitCode !== 0) throw new Error(`${launch.jobName} exited ${exitCode}`); + } + const legacy = opts.legacyBinary ? summarizeInherentArm(opts, "legacy") : undefined; + const inherent = summarizeInherentArm(opts, "inherent"); + const analysis = analyzeInherentBenchmark(legacy, inherent, readInherentTranscripts(opts, inherent)); + const reportPath = path.join(opts.jobsDir, "_bench", `${opts.prefix}-inherent-capabilities.md`); + await Bun.write(reportPath, formatInherentReport(opts, legacy, analysis, identities, telemetry)); + process.stdout.write(`Inherent capability benchmark report: ${reportPath}\n`); + if (analysis.verdict === "fail") process.exitCode = 1; +} + +if (import.meta.main) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/packages/metaharness/src/runtime-benchmark-suite.test.ts b/packages/metaharness/src/runtime-benchmark-suite.test.ts index 6bb2d8b70fa..39b9c9c3877 100644 --- a/packages/metaharness/src/runtime-benchmark-suite.test.ts +++ b/packages/metaharness/src/runtime-benchmark-suite.test.ts @@ -26,7 +26,7 @@ const EXPECTED_TASKS = [ "java-execution", "bytecode-inspection", "executable-jar", - "jvm-dependency-docs", + "jvm-dependencies", ]; describe("runtime capability suite", () => { diff --git a/packages/metaharness/src/runtime-benchmark-suite.ts b/packages/metaharness/src/runtime-benchmark-suite.ts index 9fd45f22fb0..c97ab1e5a6e 100644 --- a/packages/metaharness/src/runtime-benchmark-suite.ts +++ b/packages/metaharness/src/runtime-benchmark-suite.ts @@ -4,7 +4,7 @@ import * as path from "node:path"; export type RuntimeCapabilityGroup = "execution" | "project" | "debugging" | "profiling" | "jvm"; -export type TaskRuntimeTool = "insights" | "profile" | "jvm_disassemble" | "jvm_jar" | "jvm_deps" | "jvm_javadoc"; +export type TaskRuntimeTool = "insights" | "profile" | "jvm_disassemble" | "jvm_jar" | "jvm_deps"; export interface RuntimeTaskDefinition { id: string; @@ -242,11 +242,11 @@ test "$(java -jar app.jar 6 7)" = "42" jar --list --file app.jar | grep -q 'Main.class'`), }, { - id: "jvm-dependency-docs", + id: "jvm-dependencies", group: "jvm", - runtimeTools: ["jvm_deps", "jvm_javadoc"], + runtimeTools: ["jvm_deps"], instruction: - "Compile /app/Report.java, write its real module dependencies to /app/deps.txt using dependency analysis, and generate Javadoc under /app/api-docs. Both outputs must describe the supplied source rather than placeholders.", + "Compile /app/Report.java, write its real module dependencies to /app/deps.txt, and verify it prints 1970-01-01. Use the most direct available dependency-analysis capability; the report must include java.sql.", files: { "Report.java": '/** Formats a deterministic SQL date. */\npublic class Report { /** Returns the epoch date. */ public static String epoch(){ return java.sql.Date.valueOf("1970-01-01").toString(); } public static void main(String[] a){System.out.println(epoch());} }\n', @@ -254,7 +254,6 @@ jar --list --file app.jar | grep -q 'Main.class'`), verify: verifier(`cd /app javac --release 17 Report.java grep -q 'java.sql' deps.txt -test -f api-docs/index.html test "$(java Report)" = "1970-01-01"`), }, ]; diff --git a/packages/metaharness/src/runtime-benchmark.test.ts b/packages/metaharness/src/runtime-benchmark.test.ts index cf9946fb41a..8f0bc55c4a6 100644 --- a/packages/metaharness/src/runtime-benchmark.test.ts +++ b/packages/metaharness/src/runtime-benchmark.test.ts @@ -205,7 +205,7 @@ describe("runtime benchmark orchestration", () => { expect(launches.filter(launch => launch.arm === "runtime")).toHaveLength(RUNTIME_TASKS.length); expect(launches[0].args).toContain(`--agent-arg=${BASELINE_TOOLS.join(",")}`); expect(launches.find(launch => launch.arm === "runtime")?.args).toContain( - `--agent-arg=${[...BASELINE_TOOLS, "run", "check", "build"].join(",")}`, + `--agent-arg=${[...BASELINE_TOOLS, "run", "check"].join(",")}`, ); expect(launches.every(launch => launch.args.includes("--host-network"))).toBe(true); }); @@ -220,18 +220,16 @@ describe("runtime benchmark orchestration", () => { taskRoot: "/tmp/tasks", gatewayUrl: "http://127.0.0.1:4000", hostNetwork: true, - taskIds: ["project-validation", "instrumentation", "jvm-dependency-docs"], + taskIds: ["project-validation", "instrumentation", "jvm-dependencies"], }); const runtimeArgs = (taskId: string) => launches.find(launch => launch.arm === "runtime" && launch.taskId === taskId)?.args; - const essentials = [...BASELINE_TOOLS, "run", "check", "build"]; + const essentials = [...BASELINE_TOOLS, "run", "check"]; expect(runtimeArgs("project-validation")).toContain(`--agent-arg=${essentials.join(",")}`); expect(runtimeArgs("project-validation")?.join(" ")).not.toContain("project_advice"); expect(runtimeArgs("instrumentation")).toContain(`--agent-arg=${[...essentials, "insights"].join(",")}`); - expect(runtimeArgs("jvm-dependency-docs")).toContain( - `--agent-arg=${[...essentials, "jvm_deps", "jvm_javadoc"].join(",")}`, - ); + expect(runtimeArgs("jvm-dependencies")).toContain(`--agent-arg=${[...essentials, "jvm_deps"].join(",")}`); }); it("maps packaged runtime files into source-mounted task containers", () => { @@ -668,7 +666,7 @@ describe("runtime benchmark orchestration", () => { tools: { baseline: BASELINE_TOOLS, runtimeByTask: { - "python-execution": [...BASELINE_TOOLS, "run", "check", "build"], + "python-execution": [...BASELINE_TOOLS, "run", "check"], }, historical: BASELINE_TOOLS, }, diff --git a/packages/metaharness/src/runtime-benchmark.ts b/packages/metaharness/src/runtime-benchmark.ts index b341c917d89..799290e3cdd 100755 --- a/packages/metaharness/src/runtime-benchmark.ts +++ b/packages/metaharness/src/runtime-benchmark.ts @@ -26,21 +26,17 @@ const DEFAULT_JOBS_DIR = path.join(REPO_ROOT, "runs", "harbor"); const RUNTIME_TOOL_NAMES: Record = { run: true, check: true, - build: true, insights: true, profile: true, - runtime_debug: true, serve: true, jvm_disassemble: true, jvm_format: true, jvm_jar: true, jvm_deps: true, - jvm_javadoc: true, - project_advice: true, }; export const BASELINE_TOOLS = ["read", "write", "edit", "bash", "grep", "glob"]; -export const ESSENTIAL_RUNTIME_TOOLS = [...BASELINE_TOOLS, "run", "check", "build"]; +export const ESSENTIAL_RUNTIME_TOOLS = [...BASELINE_TOOLS, "run", "check"]; export type BenchmarkArm = "baseline" | "runtime" | "historical"; @@ -976,10 +972,7 @@ export async function runMicrobenchmarks(iterations: number): Promise runProcess([bun, "run", "build"], projectDir), - async () => { - const result = await service.build({ cwd: projectDir }); - if (result.exitCode !== 0) throw new Error(result.stderr); - }, - ); await add( "Java compile + run", async () => {