diff --git a/.coderabbit.yaml b/.coderabbit.yaml index dcc1ac3e..612dcc37 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -48,7 +48,6 @@ reviews: path_filters: - "!dist/**" - "!build/**" - - "!graphify-out/**" - "!**/*.pyc" - "!**/__pycache__/**" # Prose and working-notes paths: keeps bot reviews focused on shipped diff --git a/.gitignore b/.gitignore index 16dd1fbe..95d1c1db 100644 --- a/.gitignore +++ b/.gitignore @@ -33,14 +33,9 @@ src/pythinker_code/dashboard/static/ # Generated reports .firecrawl/ -# Graphify generated graph, cache, wiki, and Obsidian vault outputs -graphify-out/ -graphify-out*/ src/pythinker_code/.understand-anything/ # understand-anything knowledge graph (any location) .understand-anything/ -.graphify_*.json -.graphify_*.txt tests_ai/report.json tests_ai/terminal_bench_2_cache/ jobs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a014bf9a..f71021d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,9 +15,48 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Agent-spec loading is more defensive and truthful.** Subagent `path`, `extend`, and + `system_prompt_path` references that resolve outside their spec's directory (or the built-in + agents directory) are now rejected instead of loaded, and the markdown agent catalogue no longer + reclassifies an unexpected parser error as a harmless "invalid field" skip — only genuinely + malformed frontmatter is skipped. - **Thinking and subagent activity now render cleanly in the terminal.** Live reasoning previews render complete Markdown without exposing top-level HTML comments, activity-tree rows remain visually stable, and the coral shimmer is reserved for the active verb spinner. +- **Agent request compatibility is now executable and reviewable.** Provider handoff, + prompt ordering, persisted-versus-effective history, context JSONL restoration, + agent projections, and Toolset lifecycle behavior now have explicit compatibility + contracts guarding future agent-core changes. +- **Skill discovery is bounded without making skills unreachable.** Pythinker now + searches one deterministic `SkillCatalog`, keeps exhaustive exact-name resolution, + and sends only task-relevant candidates to the model within an 8,000-character + request budget. The exhaustive `Runtime.skills` mapping remains available during + the compatibility window. +- **Agent requests now have one observable assembly path.** Required guidance fails + closed, optional guidance reports sanitized degradation outcomes, and the new + `/prompt-manifest` command explains the latest request composition without storing + raw prompts, user text, or provenance paths. +- **Conversation history updates are transactional.** Normal appends persist before + changing memory, while compaction, pruning, revert, and clear flows use atomic + replacement with coherent cancellation and rollback behavior. Concurrent revert + conflicts now stop after a bounded retry budget instead of starving indefinitely. + Existing JSONL records and restoration behavior remain compatible. +- **Agent definitions now resolve through one source-aware catalogue.** YAML and + Markdown definitions share deterministic precedence, collision diagnostics, and + safe provenance handling. Unknown fields warn in this release, become errors in + the following minor release, and the `LaborMarket`, `AgentTypeDefinition`, and + generated-wrapper adapters remain through that strict-default release. +- **Tool execution and MCP lifecycle behavior now have deterministic fault coverage.** + Publication rebuilds preserve the previous MCP tool registry if registration + fails. Characterization crossed the execution-overhead threshold, but a controlled + private extraction measured slightly worse and was reverted, so + `PythinkerToolset` remains the implementation boundary. +- **Agent-core seams hardened from review.** Persisted usage/checkpoint records reject + boolean and negative token counts, `update_token_count` validates at the boundary, a + temporary system-prompt descriptor is closed if `fdopen` fails, request finalization + surfaces every provider acknowledgement failure, a failed skill projection is always + recorded as failed (never blurred to not-applicable), and request-assembly telemetry no + longer emits unbounded per-request token values as metric attributes. ## 0.57.0 (2026-07-05) diff --git a/CLAUDE.md b/CLAUDE.md index 0cb90013..bacf99de 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ Read both, in order: 1. **`AGENTS.md`** — non-negotiable repository rules. Always applies. 2. **`AGENTS.local`** — machine-specific / private local instructions (gitignored). Read it after - `AGENTS.md`. It may add workflow detail (e.g. the code-graph / graphify workflow) but must not + `AGENTS.md`. It may add workflow detail (e.g. a local code-graph workflow) but must not weaken or override the rules in `AGENTS.md`. @AGENTS.md diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 97da6de5..74f2bafd 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -14,6 +14,11 @@ export default withMermaid(defineConfig({ title: 'Pythinker Code Docs', description: 'Pythinker Code Documentation', + // Internal superpowers working docs (plans/specs/reports) are gitignored and + // never linked in the published site's nav; exclude them from VitePress's + // page auto-discovery so their Markdown is not compiled by the Vue SFC parser. + srcExclude: ['**/superpowers/**'], + locales: { en: { label: 'English', diff --git a/docs/en/contributing/toolset-characterization.md b/docs/en/contributing/toolset-characterization.md new file mode 100644 index 00000000..8eabfeb9 --- /dev/null +++ b/docs/en/contributing/toolset-characterization.md @@ -0,0 +1,138 @@ +# Toolset characterization + +The Toolset characterization harness is a directional, local engineering aid. It uses +deterministic no-op tools and fake MCP-style inventories to make Pythinker's framework costs +visible without network access, credentials, hosted telemetry, or new runtime dependencies. Its +numbers describe the machine and checkout that produced them; they are not universal product +performance claims. + +Task 12 establishes the schema, evaluator, and measurement harness. It deliberately makes no +extraction decision. Task 13 runs the full fault matrix and real five-run measurements before any +private `_ToolExecutionPipeline`, `_McpLifecycle`, or `_ToolRegistry` extraction may be considered. + +## Run it + +From the repository root: + +```bash +uv run python scripts/benchmark_toolset.py --scenario all --runs 5 \ + --output toolset-characterization.json +``` + +Use `--scenario execution`, `dedupe`, `advertisement`, or `mcp` to select one family. The default is +`all`. `--smoke` selects the smallest fixture in each chosen family and is intended only to validate +the runner and schema: + +```bash +uv run python scripts/benchmark_toolset.py --scenario all --runs 1 --smoke +``` + +Without `--output`, the report is written as JSON to standard output. A non-positive run count, an +unknown scenario, or an unwritable output path exits nonzero. + +## Fixtures and intervals + +Fixture construction and warm-up are outside every named measured interval. All clocks use +`time.monotonic_ns()`. + +- Execution runs parallel-safe and exclusive calls at concurrency 1, 10, and 100 through the public + `PythinkerToolset` facade. For `execution_mixed`, fixture `size` is a reader/writer pair count: + sizes 1, 10, and 100 therefore use concurrency and expected operation counts 2, 20, and 200. + `end_to_end` starts immediately before `handle` dispatch and ends after all returned results + settle. `tool_call` records absolute entry/exit intervals inside + deterministic no-op tools; overlapping intervals are merged before their critical-path duration + is subtracted from `end_to_end` for `framework_overhead`. Each call's read/write gate wait begins + immediately before requesting the shared or exclusive context and ends immediately after + admission, before `tool.call`. `read_write_gate_wait` is the union of those absolute wait + intervals, so overlapping queued calls count once and remain directly comparable with + `end_to_end` for the 25 percent gate. Every mixed scale uses an event-held reader from the first + pair and queues its exclusive writer before releasing it; the remaining pairs are dispatched in + call order behind that barrier, making contention deterministic without sleeps. The remaining + named lifecycle subphases are present with `measurement_status: unmeasured` + and an explanation because current Toolset exposes no stable boundary that would isolate them + without changing production behavior. +- Dedupe dispatches two same-step calls with identical 1 KiB, 100 KiB, or 1 MiB payloads. It uses + the same execution intervals and keeps payload construction outside the measured region. +- Advertisement projects 50, 500, and 5,000 real built-in, `PluginTool`, and `MCPTool` categories. + It records visibility policy enabled/disabled with hidden/unhidden entries, four aggregate reads + plus twenty individually timed repeated reads without a registry change, and a rebuild after + deterministic MCP publication. Each measured run records all twenty raw projection samples and + their nearest-rank p95; the registry repeatability decision uses the five within-run p95 values, + never a single projection or the 5,000-tool stress result. Setup remains + outside every named projection interval. Category and projection counts make the fixture behavior + independently checkable; the registry hash length-prefixes both policy projections in order. +- MCP runs the current background `load_mcp_tools`/`wait_for_mcp_tools`/`cleanup` lifecycle for 1, + 10, and 50 configured servers, replacing network connection with a deterministic local inventory + adapter. `mcp_lifecycle` starts immediately before background loading and ends when + `wait_for_mcp_tools` settles after final registry publication. `time_to_first_inventory` ends when + the inventory is visible through the public Toolset registry (the local all-at-once publication + can make first and settled effectively the same boundary), and the report records the visible + count at that exact publication boundary; `time_to_settled_inventory` ends after + `wait_for_mcp_tools` returns; and `cleanup` covers `PythinkerToolset.cleanup()` entry through + return. `startup_to_ready` begins immediately before the actual `Runtime.create` call and ends at + that same settled-inventory point. + +Concurrency and cancellation coordination use asyncio events and task completion, never timing +sleeps. The harness reports cancellation completion and post-run task leakage. Its fake adapters do +not start processes or sessions, so those leak counts remain explicit zeros. Cancellation probes +enter real queued reader and writer states through `PythinkerToolset.handle`, cancel them, then prove +that a later call recovers. Task 13 exercises the full failure matrix. + +## Repeatability and reruns + +Threshold decisions use exactly five isolated measured runs after warm-up. A threshold is crossed +only when at least four of five values exceed it and the median also exceeds it. No crossings means +uncrossed. Two or three crossings also remain uncrossed because the repeatability rule did not pass. +A single crossing is treated as an outlier and therefore inconclusive; it requests one complete +five-run rerun. The rerun replaces the inconclusive set for the decision, and a second rerun is never +requested. + +The pure evaluator records both primary and rerun values, the selected crossing count and median, +the final `crossed`, `uncrossed`, or `inconclusive` state, and whether a rerun is still required. + +## Extraction thresholds + +Task 13 applies these approved gates with the repeatability rule: + +- Execution pipeline: non-tool framework overhead exceeds 10 percent of p95 latency for short + in-process tools, or three independent recent changes repeatedly touch the same lifecycle region + and extraction deletes that shared state. +- MCP lifecycle: the defined lifecycle interval exceeds 20 percent of startup-to-ready with ten + servers; cleanup exceeds six seconds; a task, process, session, or publication leak is reproduced; + or three modules directly require MCP lifecycle state. +- Registry: advertisement exceeds 5 ms p95 at 500 tools, or recurring collision/rebuild/visibility + defects would be eliminated by one owner. The 5,000-tool stress fixture cannot trigger extraction + by itself. +- Read/write gate: gate wait exceeds 25 percent of end-to-end p95 in a realistic mixed workload, or + a second real consumer appears. + +No threshold crossing, file length alone, duplicate old/new state, a new public interface, a +callback cycle, or a red characterization/cancellation test mandates no extraction. + +## JSON schema + +The versioned report contains: + +- `environment`: Python version and implementation plus platform information; +- `scenarios[].fixture`: scenario kind, size, concurrency, payload size, and composition; mixed + execution explicitly reports `composition: reader/writer pairs`; +- `warmups` and `iterations`; +- `phases`: measured/unmeasured status, raw nanosecond samples, optional raw within-run sample + groups, median, nearest-rank p95, throughput where meaningful, and an explanation for unavailable + boundaries; +- deterministic `registry_hash`; +- `allocation_peak_bytes` and `retained_object_delta` from `tracemalloc`; +- `cancellation`: completion status and completion duration; +- `leaks`: pending task, process, and session counts; +- `task_count_peak`: the maximum observed asyncio task count during a measured sample; +- `operation_count`, `category_counts`, and `projection_counts`: independently checkable fixture + execution and advertisement behavior; +- `lifecycle_status`: `completed` for local execution/registry fixtures or `settled` after the fake + MCP inventory is fully published; and +- `decisions`: threshold, five primary values, optional five-value rerun, crossing count, median, + primary/rerun/final states, and rerun-required state. + +Partial, smoke, and non-five-run reports keep an empty `decisions` array. The documented full +`--scenario all --runs 5` command deterministically derives all seven Task 13 decisions from the raw +scenario samples and writes them in the same report. A human-readable crossed/uncrossed/inconclusive +decision record remains a Task 13 deliverable after deterministic fault tests are complete. diff --git a/docs/en/customization/agents.md b/docs/en/customization/agents.md index cdfa3019..6aad3f47 100644 --- a/docs/en/customization/agents.md +++ b/docs/en/customization/agents.md @@ -59,6 +59,21 @@ Recognized frontmatter fields are `name`, `description`, `tools`, `model`, and warning. Discovered markdown agents appear as `Agent` tool subagent types alongside the built-in types. +### Agent definition validation rollout + +Pythinker currently resolves YAML and repository markdown definitions into one agent catalogue. +Unknown definition fields are accepted with one aggregated startup warning per source in this +release. Warnings identify the field path but do not include field values, prompt content, or raw +absolute source paths. + +The production policy will reject unknown fields in the immediately following minor release. +Fix warnings before upgrading: required YAML definitions will then fail to load, while an invalid +optional markdown definition will be skipped with a diagnostic. The existing `LaborMarket` +compatibility interface and generated markdown YAML wrappers will remain for that strict-default +release. Their earliest removal is one additional minor release later, and only after direct +catalogue launch has equivalent prompt, model, tool-policy, required-MCP, foreground, and +background behavior. + ## Custom agent files Agents are defined in YAML format. Load a custom agent with the `--agent-file` flag: diff --git a/docs/en/reference/slash-commands.md b/docs/en/reference/slash-commands.md index 7df9b1b2..e064a62a 100644 --- a/docs/en/reference/slash-commands.md +++ b/docs/en/reference/slash-commands.md @@ -38,6 +38,13 @@ Aliases: `/report` List the available subagent types, showing each agent's name, when to use it, its default model, and its tool posture. +### `/prompt-manifest` + +Show the latest request-assembly status for the current session. The output includes opaque, +stable fragment identifiers, admission outcomes, and token estimates. It never includes prompt +content, user content, raw source names, raw file paths, credentials, or stack traces. Before the +first assembled request, the command reports that no manifest is available. + ## Account and configuration ### `/login` diff --git a/docs/en/release-notes/changelog.md b/docs/en/release-notes/changelog.md index 9480bfba..2de137d0 100644 --- a/docs/en/release-notes/changelog.md +++ b/docs/en/release-notes/changelog.md @@ -17,9 +17,48 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Agent-spec loading is more defensive and truthful.** Subagent `path`, `extend`, and + `system_prompt_path` references that resolve outside their spec's directory (or the built-in + agents directory) are now rejected instead of loaded, and the markdown agent catalogue no longer + reclassifies an unexpected parser error as a harmless "invalid field" skip — only genuinely + malformed frontmatter is skipped. - **Thinking and subagent activity now render cleanly in the terminal.** Live reasoning previews render complete Markdown without exposing top-level HTML comments, activity-tree rows remain visually stable, and the coral shimmer is reserved for the active verb spinner. +- **Agent request compatibility is now executable and reviewable.** Provider handoff, + prompt ordering, persisted-versus-effective history, context JSONL restoration, + agent projections, and Toolset lifecycle behavior now have explicit compatibility + contracts guarding future agent-core changes. +- **Skill discovery is bounded without making skills unreachable.** Pythinker now + searches one deterministic `SkillCatalog`, keeps exhaustive exact-name resolution, + and sends only task-relevant candidates to the model within an 8,000-character + request budget. The exhaustive `Runtime.skills` mapping remains available during + the compatibility window. +- **Agent requests now have one observable assembly path.** Required guidance fails + closed, optional guidance reports sanitized degradation outcomes, and the new + `/prompt-manifest` command explains the latest request composition without storing + raw prompts, user text, or provenance paths. +- **Conversation history updates are transactional.** Normal appends persist before + changing memory, while compaction, pruning, revert, and clear flows use atomic + replacement with coherent cancellation and rollback behavior. Concurrent revert + conflicts now stop after a bounded retry budget instead of starving indefinitely. + Existing JSONL records and restoration behavior remain compatible. +- **Agent definitions now resolve through one source-aware catalogue.** YAML and + Markdown definitions share deterministic precedence, collision diagnostics, and + safe provenance handling. Unknown fields warn in this release, become errors in + the following minor release, and the `LaborMarket`, `AgentTypeDefinition`, and + generated-wrapper adapters remain through that strict-default release. +- **Tool execution and MCP lifecycle behavior now have deterministic fault coverage.** + Publication rebuilds preserve the previous MCP tool registry if registration + fails. Characterization crossed the execution-overhead threshold, but a controlled + private extraction measured slightly worse and was reverted, so + `PythinkerToolset` remains the implementation boundary. +- **Agent-core seams hardened from review.** Persisted usage/checkpoint records reject + boolean and negative token counts, `update_token_count` validates at the boundary, a + temporary system-prompt descriptor is closed if `fdopen` fails, request finalization + surfaces every provider acknowledgement failure, a failed skill projection is always + recorded as failed (never blurred to not-applicable), and request-assembly telemetry no + longer emits unbounded per-request token values as metric attributes. ## 0.57.0 (2026-07-05) diff --git a/docs/history/CHANGELOG-pre-0.8.0.md b/docs/history/CHANGELOG-pre-0.8.0.md index f4353fc4..23903a4c 100644 --- a/docs/history/CHANGELOG-pre-0.8.0.md +++ b/docs/history/CHANGELOG-pre-0.8.0.md @@ -115,7 +115,7 @@ Subagent roles overhaul, Kimi K2 provider support, and a ripgrep-free Grep fallb - Pure-Python `rg`-free fallback (`_python_grep`) honoring `pattern`, `path`, `glob`, `type` (bash / c / cpp / go / java / js / json / md / py / rust / sh / toml / ts / txt / yaml / zsh), `ignore_case`, `multiline`, `context` / `before_context` / `after_context`, `line_number`, `output_mode` (`content` / `files_with_matches` / `count_matches`), `offset`, `head_limit`, and the standard sensitive-file redaction. `.gitignore` / `.ignore` and the VCS metadata directories (`.git`, `.svn`, `.hg`, `.bzr`, `.jj`, `.sl`) are respected unless `include_ignored=true`. - `_find_existing_rg` now honors `PYTHINKER_RG_PATH` and additionally probes `/usr/bin`, `/usr/local/bin`, `~/.cargo/bin`, `~/.local/bin`, and `~/.pi/agent/bin` before falling through to download. - Downloader retries against the upstream GitHub releases mirror (`https://github.com/BurntSushi/ripgrep/releases/download//...`) when the CDN mirror is unreachable, and the failure path now degrades into the Python fallback instead of raising. -- `.gitignore`: ignore `graphify-out*/`, `.graphify_*.json`, `.graphify_*.txt`, and the local reference-scan scratch area. +- `.gitignore`: ignore generated code-graph outputs and the local reference-scan scratch area. - `AGENTS.md` rewritten to reflect the new subagent roster and workflow. ## 2.3.0 (2026-05-09) diff --git a/docs/superpowers/plans/2026-07-10-agent-core-deepening.md b/docs/superpowers/plans/2026-07-10-agent-core-deepening.md new file mode 100644 index 00000000..96e237e1 --- /dev/null +++ b/docs/superpowers/plans/2026-07-10-agent-core-deepening.md @@ -0,0 +1,380 @@ +# Agent Core Deepening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. Steps use checkbox syntax for tracking. + +**Goal:** Deliver the current-release implementation of all six approved agent-core-deepening phases while preserving provider, JSONL, tool, slash, agent, and configuration compatibility. + +**Architecture:** Add four deep policy owners—SkillCatalog, RequestAssembler, transactional Context, and ResolvedAgentCatalogue—behind temporary compatibility projections. Characterize PythinkerToolset with deterministic tests and a local benchmark harness, then extract at most one private state machine only if an approved threshold is reproducibly crossed. + +**Tech Stack:** Python 3.14, asyncio, Pydantic, pytest, uv, Ruff, Pyright, ty, and standard-library filesystem/timing primitives. + +## Global Constraints + +- Use uv or repository make targets for every Python command. +- Add no third-party runtime or benchmark dependency. +- Preserve pythinker_core.step, provider adapters, JSONL record shapes, tool names, slash behavior, agent precedence, and public configuration unless this plan names the change. +- Keep Runtime.skills, Runtime.labor_market, AgentTypeDefinition, generated Markdown wrappers, and PythinkerToolset as compatibility surfaces for their approved migration windows. +- Required guidance fails closed and is never truncated; optional guidance reports degradation explicitly. +- Never persist raw prompts, request manifests, prompt fragments, raw provenance paths, credentials, or user text as diagnostics or telemetry. +- Keep Context.file_backend on the local Path seam and make no cross-process safety claim. +- Phase 5 ships WARN as production unknown-field policy and tests FORBID; strict production activation belongs to the following minor release. +- Phase 6 treats a reproducible no-go decision as completion and forbids extraction based only on file length. +- Each production slice follows red then green at the named public seam, followed by focused tests and make check-pythinker-code. +- Each shipped-code phase adds an Unreleased bullet to CHANGELOG.md. Generated docs changelog changes only through npm run sync from docs/. + +## Settled Interpretation + +- The skill search query is the latest real user turn supplied explicitly by the soul; reminder-only and tool-result messages are not treated as the task. +- Explicit skill mentions are distinct, left-to-right $ and /skill: tokens. Plain prose names use normal search ranking. +- Request assembly owns main agent-turn provider requests and the /btw aligned-history path. Compaction and blind-advisor calls remain specialized internal model operations outside the dynamic source matrix. +- Phase 6 writes evidence to docs/superpowers/reports/2026-07-10-toolset-characterization.json and the matching .md decision record. + +--- + +### Task 1: Phase 1 compatibility characterization + +**Files:** +- Create: tests/core/test_provider_handoff_contract.py +- Modify: tests/core/test_skills_prompt.py +- Modify: tests/core/test_context.py +- Modify: tests/core/test_load_agent.py +- Modify: tests/core/test_agent_list_injection.py + +**Interfaces:** +- Consumes: current provider handoff, skill rendering, JSONL restoration, agent projection, and Toolset facade. +- Produces: executable compatibility contracts used by later tasks. + +- [ ] Write a provider-handoff test that captures the four positional arguments to pythinker_core.step and independently asserts stable system prompt bytes, AGENTS.md position, dynamic reminder ordering, normalization, and persisted-versus-effective history. + +~~~python +async def test_agent_step_has_one_characterized_provider_handoff(soul, monkeypatch): + captured: list[tuple[str, tuple[Message, ...]]] = [] + + async def capture(_provider, system_prompt, _toolset, history, **_kwargs): + captured.append((system_prompt, tuple(history))) + return StepResult(message=Message(role="assistant", content=[]), usage=None) + + monkeypatch.setattr(pythinker_core, "step", capture) + await soul._step() + assert captured == [(soul.agent.system_prompt, expected_effective_history)] + assert tuple(soul.context.history) == expected_persisted_history +~~~ + +- [ ] Add literal mixed-record JSONL and AgentTypeDefinition projection fixtures. Do not derive expected values with production serializers. +- [ ] Run the pre-change gate: + +~~~bash +uv run pytest -q tests/core/test_provider_handoff_contract.py tests/core/test_skills_prompt.py tests/core/test_context.py tests/core/test_load_agent.py tests/core/test_agent_list_injection.py tests/core/test_mcp_lifecycle.py tests/core/test_toolset_concurrency.py +~~~ + +- [ ] Commit with subject: test(core): characterize agent request contracts + +### Task 2: Phase 2 exhaustive SkillCatalog seam + +**Files:** +- Create: src/pythinker_code/skill/catalog.py +- Create: tests/core/test_skill_catalog.py +- Modify: src/pythinker_code/skill/__init__.py + +**Interfaces:** +- Consumes: ScopedSkillsRoot, Skill, normalize_skill_name, and existing discovery. +- Produces: SkillCatalog.discover, resolve, search, prompt_view, and exhaustive_mapping. + +- [ ] Write failing exact-name, case, alias, first-root precedence, reversed-input determinism, and unavailable-diagnostic tests. +- [ ] Implement frozen SkillMatch, SkillPromptView, SkillProjectionOutcome, status/diagnostic types, and one catalogue-owned winning index. + +~~~python +class SkillCatalog: + @classmethod + async def discover(cls, roots: Sequence[ScopedSkillsRoot]) -> SkillCatalog: ... + def resolve(self, name: str) -> Skill | None: ... + def search(self, query: str, *, limit: int) -> tuple[SkillMatch, ...]: ... + def prompt_view(self, query: str, *, max_characters: int) -> SkillProjectionOutcome: ... + def exhaustive_mapping(self) -> Mapping[str, Skill]: ... +~~~ + +- [ ] Retain malformed-source diagnostics during discovery rather than reconstructing them in ReadSkill. +- [ ] Verify with uv run pytest -q tests/core/test_skill_catalog.py tests/core/test_skills_prompt.py tests/tools/test_skill_tool.py and make check-pythinker-code. +- [ ] Commit with subject: feat(skills): add exhaustive skill catalogue + +### Task 3: Phase 2 bounded runtime discovery + +**Files:** +- Create: tests/fixtures/skill_catalog_recall.json +- Create: tests/core/test_pythinkersoul_skill_projection.py +- Modify: src/pythinker_code/skill/catalog.py +- Modify: src/pythinker_code/skill/__init__.py +- Modify: src/pythinker_code/soul/agent.py +- Modify: src/pythinker_code/soul/pythinkersoul.py +- Modify: src/pythinker_code/tools/skill/__init__.py +- Modify: src/pythinker_code/agents/default/system.md +- Modify: src/pythinker_code/cli/system_prompt.py +- Modify: tests/conftest.py +- Modify: tests/tools/test_skill_tool.py + +**Interfaces:** +- Consumes: Task 2 catalogue. +- Produces: bounded deterministic candidates, shared Runtime.skill_catalog, exhaustive Runtime.skills adapter, and bounded ReadSkill suggestions. + +- [ ] Write red tests for relevance tiers, reversed insertion order, explicit/active priority, 8,000-character hard cap, complete names only, omission counts, no absolute paths/body reads, 100 percent fixture recall, and 1,000-entry warm median below 100 ms with an operation-count assertion. +- [ ] Implement exact/phrase/name-token/description-token ranking, then scope/name/path tie breakers. +- [ ] Add required Runtime.skill_catalog, derive Runtime.skills from exhaustive_mapping, and share catalogue identity in copy_for_subagent. +- [ ] Replace static exhaustive prompt content with stable invocation policy and route prompt inspection through the same discovery implementation. +- [ ] Add the temporary request-only soul projection. A degraded/failed projection stores safe status/reason and never adds an exhaustive fallback. +- [ ] Migrate ReadSkill while preserving local specialization and MCP fallback; distinguish not_found, unavailable, and mcp_fallback. +- [ ] Verify the catalogue, prompt, runtime, soul projection, ReadSkill, MCP bridge, and wire skill suites; run make check-pythinker-code. +- [ ] Commit with subject: feat(skills): bound provider-visible discovery + +### Task 4: Phase 3 request admission engine + +**Files:** +- Create: src/pythinker_code/soul/request_assembly.py +- Create: tests/core/test_request_assembly.py +- Modify: src/pythinker_code/soul/dynamic_injection.py + +**Interfaces:** +- Consumes: Message, shared token estimator, normalize_history, and a SkillCatalog projection port. +- Produces: immutable fragments/outcomes/manifest/input/result and categorized errors. + +- [ ] Write failing tests for required-first admission, non-budgeted AGENTS.md, stable equal-priority ordering, truncatable versus omitted content, redaction, aggregate accounting, and required failure. +- [ ] Implement the approved enums and frozen data contracts. + +~~~python +@dataclass(frozen=True, slots=True) +class RequestAssemblyInput: + system_prompt: str + persisted_history: tuple[Message, ...] + current_task: str + budget_tokens: int + +@dataclass(frozen=True, slots=True) +class AssembledRequest: + system_prompt: str + provider_history: tuple[Message, ...] + history_appends: tuple[Message, ...] + manifest: RequestManifest +~~~ + +- [ ] Recompute all estimates inside the assembler, reserve required content, and keep request-only content out of history_appends. +- [ ] Preserve the legacy budget API as a projection over explicit outcomes; do not retain a second admission algorithm. +- [ ] Verify request assembly, legacy budget, normalization, and make check-pythinker-code. +- [ ] Commit with subject: feat(soul): add observable request assembler + +### Task 5: Phase 3 provider and soul integration + +**Files:** +- Create: tests/core/test_request_assembly_providers.py +- Create: tests/core/test_request_assembly_soul.py +- Modify: src/pythinker_code/soul/pythinkersoul.py +- Modify: src/pythinker_code/soul/dynamic_injection.py +- Modify: src/pythinker_code/soul/dynamic_injections/permissions_state.py +- Modify: src/pythinker_code/soul/dynamic_injections/model_defense.py +- Modify: src/pythinker_code/soul/btw.py + +**Interfaces:** +- Consumes: Tasks 3 and 4. +- Produces: one agent-turn assembly path, explicit provider outcomes, acknowledgement/rearm, and latest manifest. + +- [ ] Write red tests for every legacy provider as best-effort, exception degradation, required permission/model-defense failure, allowed not_applicable, disabled optional bus behavior, and persistence-failure retry. +- [ ] Assign requiredness/persistence in trusted composition: AGENTS.md required request-only/non-budgeted; permissions and applicable model defense required; unnamed providers best-effort. +- [ ] Move one-shot acknowledgement after successful persistence. +- [ ] Replace _step orchestration: assemble, store manifest, persist history_appends once, then invoke provider. Persistence failure stores context_persistence_failed and skips provider invocation. +- [ ] Delete _collect_injections and the temporary Phase 2 adapter only after byte-equivalence tests pass. +- [ ] Align /btw provider history through the assembler while keeping the side question request-only. +- [ ] Verify request assembly, provider, main-step, retry, /btw, and make check-pythinker-code. +- [ ] Commit with subject: feat(soul): route agent turns through request assembly + +### Task 6: Phase 3 manifest observability + +**Files:** +- Create: tests/core/test_prompt_manifest_slash.py +- Modify: src/pythinker_code/soul/slash.py +- Modify: src/pythinker_code/soul/pythinkersoul.py +- Modify: src/pythinker_code/telemetry/metrics.py +- Modify: tests_e2e/test_wire_protocol.py +- Modify: docs/en/reference/slash-commands.md + +**Interfaces:** +- Consumes: latest in-memory RequestManifest. +- Produces: /prompt-manifest text and content-free aggregate telemetry. + +- [ ] Write failing no-data, success, degraded, failure, and redaction tests. +- [ ] Register a normal soul slash command; add no wire event. +- [ ] Emit only counts, budgets, stable source IDs, and duration to telemetry. +- [ ] Verify slash, wire protocol, telemetry, and make check-pythinker-code. +- [ ] Commit with subject: feat(soul): expose sanitized prompt manifests + +### Task 7: Phase 4 reducer and disk-first appends + +**Files:** +- Create: tests/core/test_context_transactions.py +- Modify: src/pythinker_code/soul/context.py + +**Interfaces:** +- Consumes: existing JSONL records and restore repair. +- Produces: one serializer/reducer, one mutation lock, disk-first append_messages, checkpoint, usage, and system-prompt writes. + +- [ ] Write failing serialization/open/write/flush fault tests that assert exact old bytes and memory; include checkpoint ID and usage-counter failure. +- [ ] Extract an immutable reducer for system prompt, repaired history, authoritative/pending tokens, next checkpoint ID, and tail state. +- [ ] Serialize before locking, append one complete batch, flush, then swap precomputed memory without another await. Keep append_message as a compatibility delegate. +- [ ] Add event/barrier concurrency and queued-writer cancellation tests without sleeps. +- [ ] Verify context transaction, restore, repair, pending-token suites, and make check-pythinker-code. +- [ ] Commit with subject: feat(context): make incremental writes disk first + +### Task 8: Phase 4 atomic replacement + +**Files:** +- Modify: src/pythinker_code/soul/context.py +- Modify: tests/core/test_context_transactions.py + +**Interfaces:** +- Consumes: Task 7 reducer/serializer/lock and next_available_rotation. +- Produces: ContextReplacement, ContextCommit, ContextPersistenceError, and replace_history. + +- [ ] Write failing temp creation, record write, flush, temp fsync, archive, replace, directory fsync, cleanup, and cancellation tests. +- [ ] Implement a restrictive unique same-directory temp file, exact-byte numbered archive, os.replace, no-await memory swap, POSIX directory fsync, and BaseException cleanup preserving the primary cause. +- [ ] Shield only replace-plus-swap and re-raise cancellation after coherence. +- [ ] Verify replace/archive/cancel/concurrency tests and make check-pythinker-code. +- [ ] Commit with subject: feat(context): replace history atomically + +### Task 9: Phase 4 soul-flow migration + +**Files:** +- Modify: src/pythinker_code/soul/pythinkersoul.py +- Modify: src/pythinker_code/soul/slash.py +- Modify: src/pythinker_code/soul/context.py +- Modify: tests/core/test_context_pruning.py +- Modify: tests/core/test_compaction_restore.py +- Modify: tests_e2e/test_wire_sessions.py + +**Interfaces:** +- Consumes: Task 8 replace_history. +- Produces: one-commit prune, compact, revert, and clear flows with no compensating rebuild. + +- [ ] Rewrite failure tests at replace_history and assert exact old bytes/state. +- [ ] Prepare all prune/compact semantic messages and tokens first, call replace_history once, preserve CompactionEnd in finally, and rearm providers after commit. +- [ ] Delete both clear/rebuild rollback blocks. +- [ ] Delegate revert_to and clear to replacement; make /clear include current system prompt in one reset. +- [ ] Verify prune, compact, context, and wire clear/compact suites; run make check-pythinker-code. +- [ ] Commit with subject: feat(context): migrate history rewrites to transactions + +### Task 10: Phase 5 validation and catalogue + +**Files:** +- Create: src/pythinker_code/subagents/catalogue.py +- Create: tests/core/test_agent_catalogue_validation.py +- Create: tests/core/test_agent_catalogue_markdown.py +- Modify: src/pythinker_code/agentspec.py +- Modify: src/pythinker_code/subagents/discovery.py + +**Interfaces:** +- Consumes: existing recursive YAML loader and Markdown parser/materializer. +- Produces: UnknownFieldPolicy, safe diagnostics/provenance, immutable catalogue entries, and source adapters. + +- [ ] Write failing WARN/FORBID tests for top-level, agent, nested subagent, inherited, and Markdown unknown fields; assert stable paths and no raw values/absolute paths. +- [ ] Inspect raw mappings before Pydantic. WARN aggregates once per source and continues; FORBID fails required YAML and rejects only the optional Markdown entry. +- [ ] Use exactly name.casefold() for normalize_agent_name. +- [ ] Resolve existing behavior without duplicating inheritance; defensively freeze/copy nested collections. +- [ ] Preserve YAML fatality, Markdown isolation/root precedence, plugin order, deterministic enumeration, shadow diagnostics, required collision failure, and optional collision warning. +- [ ] Verify catalogue validation, Markdown, agent spec, discovery, and make check-pythinker-code. +- [ ] Commit with subject: feat(agents): resolve definitions through a catalogue + +### Task 11: Phase 5 compatibility publication + +**Files:** +- Create: tests/core/test_agent_catalogue_compat.py +- Modify: src/pythinker_code/soul/agent.py +- Modify: src/pythinker_code/subagents/runner.py +- Modify: src/pythinker_code/tools/agent/__init__.py +- Modify: src/pythinker_code/soul/dynamic_injections/agent_list.py +- Modify: src/pythinker_code/ui/shell/slash.py +- Modify: tests/conftest.py +- Modify: docs/en/customization/agents.md + +**Interfaces:** +- Consumes: Task 10. +- Produces: shared Runtime.agent_catalogue, exact AgentTypeDefinition/LaborMarket projection, and WARN production composition. + +- [ ] Write failing launch projection, wrapper parity, exact LaborMarket, case-insensitive catalogue, root/subagent identity, agent-list, /agents, MCP, background, hidden, and tool-policy tests. +- [ ] Construct once before tools, populate LaborMarket, retain generated wrappers and dependency injection, and migrate readers without changing output. +- [ ] Document WARN now, FORBID in the next minor, and earliest adapter removal one additional minor later after launch parity. +- [ ] Verify compatibility, builder/load/default agent, Agent tool, /agents, wire, and make check-pythinker-code. +- [ ] Commit with subject: feat(agents): publish resolved agent catalogue + +### Task 12: Phase 6 schema and harness + +**Files:** +- Create: src/pythinker_code/benchmark/toolset_characterization.py +- Create: scripts/benchmark_toolset.py +- Create: tests/core/test_toolset_characterization.py +- Create: docs/en/contributing/toolset-characterization.md + +**Interfaces:** +- Consumes: public PythinkerToolset and standard-library timing/platform APIs. +- Produces: machine-readable schema, deterministic evaluator, and local runner. + +- [ ] Write failing pure five-run crossed, uncrossed, and inconclusive tests: four crossings plus crossing median; one outlier requests one rerun. +- [ ] Implement typed environment/fixture/raw-sample/median/p95/throughput/hash/cancellation/leak/decision fields. +- [ ] Measure 1/10/100 safe/exclusive/mixed calls, 1 KiB/100 KiB/1 MiB dedupe, 50/500/5,000 advertisement, and 1/10/50 MCP fixtures, excluding setup and tool duration. +- [ ] Verify schema tests, runner help, and make check-pythinker-code. +- [ ] Commit with subject: test(toolset): add characterization harness + +### Task 13: Phase 6 fault matrix and decision + +**Files:** +- Modify: tests/core/test_toolset.py +- Modify: tests/core/test_toolset_concurrency.py +- Modify: tests/core/test_mcp_lifecycle.py +- Modify: tests/core/test_mcp_cleanup.py +- Create: docs/superpowers/reports/2026-07-10-toolset-characterization.json +- Create: docs/superpowers/reports/2026-07-10-toolset-characterization.md +- Conditional create: exactly one approved private Toolset module if a threshold crosses. + +**Interfaces:** +- Consumes: Task 12 and approved thresholds. +- Produces: deterministic failure coverage and a reproducible go/no-go decision. + +- [ ] Add event-driven pre/post-hook, telemetry-policy, gate cancellation, permit recovery, and later-call tests. +- [ ] Add hung/method-not-found/transient/duplicate/list-storm/race/cancel/timeout/partial-publication MCP tests with deterministic hashes and leak assertions. +- [ ] Run five isolated measurements: + +~~~bash +uv run python scripts/benchmark_toolset.py --scenario all --runs 5 --output docs/superpowers/reports/2026-07-10-toolset-characterization.json +~~~ + +- [ ] Write every raw value, median, repeatability verdict, environment, crossed/uncrossed/inconclusive threshold, and decision to the Markdown record. +- [ ] If none cross, record no-go and do not refactor. If one crosses, first add a failing assertion, move only that state machine, delete moved state from PythinkerToolset, rerun identical fixtures, and revert if the target/locality does not improve. +- [ ] Verify Toolset, concurrency, MCP lifecycle/cleanup/startup, characterization, and make check-pythinker-code. +- [ ] Commit with subject: test(toolset): characterize execution and lifecycle + +### Task 14: Release, full verification, and review + +**Files:** +- Modify: CHANGELOG.md +- Modify: tasks/todo.md +- Conditional generated modify: docs/en/release-notes/changelog.md + +**Interfaces:** +- Consumes: Tasks 1–13. +- Produces: release notes, completed ledger/review, full gates, and reviewed branch. + +- [ ] Add one precise Unreleased bullet per shipped phase, including compatibility windows and /prompt-manifest. Sync generated changelog only through docs/npm run sync. +- [ ] Run provider snapshots: + +~~~bash +uv run --directory packages/pythinker-core pytest -q tests/api_snapshot_tests/test_openai_responses.py tests/api_snapshot_tests/test_anthropic.py +~~~ + +- [ ] Apply pythinker-guard, test-guard, and clean-code-guard to the branch diff. +- [ ] Run fresh, unpiped make check, make test-pythinker-code, and git diff --check. +- [ ] If only the documented PTY cancellation node flakes, rerun that node on branch and base before classification; treat other failures as caused until disproven. +- [ ] Run the requested two-axis code review from fixed point 46ea01e001117489e0fe7fb57d29678253e2d2ea against the umbrella and five detailed specs. Fix Critical/Important findings and rerun affected tests. +- [ ] Record outcome, deviations, benchmark decision, exact verification, migration windows, and blockers in tasks/todo.md. +- [ ] Commit final metadata with subject: docs(core): record agent deepening release + +## Plan Self-Review + +- Coverage: all six phases, compatibility adapters, current WARN policy, Phase 6 no-go option, docs, changelog, and deletion conditions are assigned. +- Placeholders: the only conditional branch is the spec-required measured Toolset decision, bounded to the three approved private modules. +- Type ordering: SkillCatalog precedes RequestAssembler; Context replacement precedes flow migration; agent catalogue precedes runtime publication. +- Scope: future-minor strict activation and adapter deletion are documented obligations, not incorrectly shipped now. diff --git a/docs/superpowers/reports/2026-07-10-toolset-characterization.json b/docs/superpowers/reports/2026-07-10-toolset-characterization.json new file mode 100644 index 00000000..65e631f8 --- /dev/null +++ b/docs/superpowers/reports/2026-07-10-toolset-characterization.json @@ -0,0 +1,3177 @@ +{ + "schema_version": 2, + "environment": { + "python_version": "3.14.6", + "python_implementation": "CPython", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O" + }, + "scenarios": [ + { + "fixture": { + "kind": "execution_safe", + "size": 1, + "concurrency": 1, + "payload_bytes": null, + "composition": "safe" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1611958, + 1671708, + 1552667, + 1488875, + 1543125 + ], + "median_ns": 1552667, + "p95_ns": 1671708, + "throughput_per_second": 644.0531034664871, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 12375, + 12041, + 9041, + 6833, + 8334 + ], + "median_ns": 9041, + "p95_ns": 12375, + "throughput_per_second": 110607.23371308483, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 10000, + 10250, + 9167, + 6541, + 8833 + ], + "median_ns": 9167, + "p95_ns": 10250, + "throughput_per_second": 109086.94229300752, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1601958, + 1661458, + 1543500, + 1482334, + 1534292 + ], + "median_ns": 1543500, + "p95_ns": 1661458, + "throughput_per_second": 647.878198898607, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 55889, + "retained_object_delta": 32140, + "cancellation": { + "completed": true, + "completion_ns": 7746208, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 1, + "operation_count": 5, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_safe", + "size": 10, + "concurrency": 10, + "payload_bytes": null, + "composition": "safe" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 13923875, + 13954042, + 13875084, + 14617750, + 14269084 + ], + "median_ns": 13954042, + "p95_ns": 14617750, + "throughput_per_second": 716.6382328503813, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 59790, + 54167, + 54081, + 63252, + 52416 + ], + "median_ns": 54167, + "p95_ns": 63252, + "throughput_per_second": 184614.24852770136, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 59501, + 52665, + 53499, + 83209, + 52875 + ], + "median_ns": 53499, + "p95_ns": 83209, + "throughput_per_second": 186919.38167068543, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 13864374, + 13901377, + 13821585, + 14534541, + 14216209 + ], + "median_ns": 13901377, + "p95_ns": 14534541, + "throughput_per_second": 719.353197888238, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 130828, + "retained_object_delta": 76511, + "cancellation": { + "completed": true, + "completion_ns": 7534000, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 10, + "operation_count": 50, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_safe", + "size": 100, + "concurrency": 100, + "payload_bytes": null, + "composition": "safe" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 142778750, + 144065250, + 144293541, + 145561042, + 147325083 + ], + "median_ns": 144293541, + "p95_ns": 147325083, + "throughput_per_second": 693.0317137341581, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 515915, + 510379, + 523748, + 516250, + 566034 + ], + "median_ns": 516250, + "p95_ns": 566034, + "throughput_per_second": 193704.6004842615, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 527289, + 533914, + 509791, + 519173, + 594629 + ], + "median_ns": 527289, + "p95_ns": 594629, + "throughput_per_second": 189649.3194434172, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 142251461, + 143531336, + 143783750, + 145041869, + 146730454 + ], + "median_ns": 143783750, + "p95_ns": 146730454, + "throughput_per_second": 695.4888852182531, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 991624, + "retained_object_delta": 552667, + "cancellation": { + "completed": true, + "completion_ns": 7709000, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 100, + "operation_count": 500, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_exclusive", + "size": 1, + "concurrency": 1, + "payload_bytes": null, + "composition": "exclusive" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1474834, + 1448708, + 1435458, + 1449541, + 1510625 + ], + "median_ns": 1449541, + "p95_ns": 1510625, + "throughput_per_second": 689.8735530764566, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 4625, + 4041, + 3708, + 3917, + 5250 + ], + "median_ns": 4041, + "p95_ns": 5250, + "throughput_per_second": 247463.49913387775, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 5958, + 5459, + 4917, + 5500, + 7542 + ], + "median_ns": 5500, + "p95_ns": 7542, + "throughput_per_second": 181818.18181818182, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1468876, + 1443249, + 1430541, + 1444041, + 1503083 + ], + "median_ns": 1444041, + "p95_ns": 1503083, + "throughput_per_second": 692.5011131955395, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 36916, + "retained_object_delta": 14138, + "cancellation": { + "completed": true, + "completion_ns": 7503750, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 1, + "operation_count": 5, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_exclusive", + "size": 10, + "concurrency": 10, + "payload_bytes": null, + "composition": "exclusive" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 13893667, + 14073792, + 14313750, + 14225167, + 14191750 + ], + "median_ns": 14191750, + "p95_ns": 14313750, + "throughput_per_second": 704.6347349692603, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 37542, + 37707, + 39916, + 39041, + 38832 + ], + "median_ns": 38832, + "p95_ns": 39916, + "throughput_per_second": 257519.57148743304, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 54875, + 54708, + 59917, + 55458, + 53290 + ], + "median_ns": 54875, + "p95_ns": 59917, + "throughput_per_second": 182232.34624145785, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 13838792, + 14019084, + 14253833, + 14169709, + 14138460 + ], + "median_ns": 14138460, + "p95_ns": 14253833, + "throughput_per_second": 707.2906101513178, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 120341, + "retained_object_delta": 66495, + "cancellation": { + "completed": true, + "completion_ns": 7459875, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 10, + "operation_count": 50, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_exclusive", + "size": 100, + "concurrency": 100, + "payload_bytes": null, + "composition": "exclusive" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 139976875, + 142790750, + 155137208, + 143204166, + 142406416 + ], + "median_ns": 142790750, + "p95_ns": 155137208, + "throughput_per_second": 700.3254762650942, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 370624, + 375834, + 465828, + 377417, + 377498 + ], + "median_ns": 377417, + "p95_ns": 465828, + "throughput_per_second": 264958.91811974556, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 584628, + 531462, + 739624, + 524873, + 516709 + ], + "median_ns": 531462, + "p95_ns": 739624, + "throughput_per_second": 188160.207126756, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 139392247, + 142259288, + 154397584, + 142679293, + 141889707 + ], + "median_ns": 142259288, + "p95_ns": 154397584, + "throughput_per_second": 702.9418001867126, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 948002, + "retained_object_delta": 533532, + "cancellation": { + "completed": true, + "completion_ns": 7608084, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 100, + "operation_count": 500, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_mixed", + "size": 1, + "concurrency": 2, + "payload_bytes": null, + "composition": "reader/writer pairs" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 3166917, + 3053917, + 3196583, + 3092334, + 2993375 + ], + "median_ns": 3092334, + "p95_ns": 3196583, + "throughput_per_second": 646.7606668619884, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 167001, + 158874, + 170458, + 145500, + 138292 + ], + "median_ns": 158874, + "p95_ns": 170458, + "throughput_per_second": 12588.592217732292, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 217625, + 207458, + 214208, + 196500, + 188333 + ], + "median_ns": 207458, + "p95_ns": 217625, + "throughput_per_second": 9640.505548110943, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 2949292, + 2846459, + 2982375, + 2895834, + 2805042 + ], + "median_ns": 2895834, + "p95_ns": 2982375, + "throughput_per_second": 690.6473230164436, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "8c718a5dc30b54eeabc4aba7d1a5910c7d3b727e3547c8f82451d70bb73c7ba6", + "allocation_peak_bytes": 45348, + "retained_object_delta": 17467, + "cancellation": { + "completed": true, + "completion_ns": 7452209, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 2, + "operation_count": 10, + "category_counts": { + "builtin": 2 + }, + "projection_counts": { + "visible": 2 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_mixed", + "size": 10, + "concurrency": 20, + "payload_bytes": null, + "composition": "reader/writer pairs" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 27995958, + 27908417, + 27797542, + 28240292, + 28150584 + ], + "median_ns": 27995958, + "p95_ns": 28240292, + "throughput_per_second": 714.3888414177504, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 25114000, + 25055916, + 24866125, + 25366543, + 25213208 + ], + "median_ns": 25114000, + "p95_ns": 25366543, + "throughput_per_second": 796.3685593692761, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1470875, + 1429754, + 1449787, + 1437502, + 1509869 + ], + "median_ns": 1449787, + "p95_ns": 1509869, + "throughput_per_second": 13795.129905289536, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 26525083, + 26478663, + 26347755, + 26802790, + 26640715 + ], + "median_ns": 26525083, + "p95_ns": 26802790, + "throughput_per_second": 754.003295673005, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "8c718a5dc30b54eeabc4aba7d1a5910c7d3b727e3547c8f82451d70bb73c7ba6", + "allocation_peak_bytes": 212878, + "retained_object_delta": 108838, + "cancellation": { + "completed": true, + "completion_ns": 7554125, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 20, + "operation_count": 100, + "category_counts": { + "builtin": 2 + }, + "projection_counts": { + "visible": 2 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "execution_mixed", + "size": 100, + "concurrency": 200, + "payload_bytes": null, + "composition": "reader/writer pairs" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 291138125, + 315861833, + 322113167, + 294026583, + 292861542 + ], + "median_ns": 294026583, + "p95_ns": 322113167, + "throughput_per_second": 680.2106053111531, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 287985499, + 312098292, + 318561708, + 290799333, + 289581583 + ], + "median_ns": 290799333, + "p95_ns": 318561708, + "throughput_per_second": 687.7594867110648, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 14608124, + 16458287, + 17987703, + 14638254, + 14900751 + ], + "median_ns": 14900751, + "p95_ns": 17987703, + "throughput_per_second": 13422.142279942804, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 276530001, + 299403546, + 304125464, + 279388329, + 277960791 + ], + "median_ns": 279388329, + "p95_ns": 304125464, + "throughput_per_second": 715.8495156753667, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "8c718a5dc30b54eeabc4aba7d1a5910c7d3b727e3547c8f82451d70bb73c7ba6", + "allocation_peak_bytes": 1919890, + "retained_object_delta": 984817, + "cancellation": { + "completed": true, + "completion_ns": 7680250, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 200, + "operation_count": 1000, + "category_counts": { + "builtin": 2 + }, + "projection_counts": { + "visible": 2 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "dedupe", + "size": 1024, + "concurrency": 2, + "payload_bytes": 1024, + "composition": "same-step duplicate payload" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1671625, + 1558500, + 1587292, + 1552250, + 1533042 + ], + "median_ns": 1558500, + "p95_ns": 1671625, + "throughput_per_second": 641.6426050689765, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 9667, + 7542, + 8709, + 6667, + 5209 + ], + "median_ns": 7542, + "p95_ns": 9667, + "throughput_per_second": 132590.82471492974, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 9292, + 6833, + 7750, + 5792, + 5375 + ], + "median_ns": 6833, + "p95_ns": 9292, + "throughput_per_second": 146348.60237084737, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1662333, + 1551667, + 1579542, + 1546458, + 1527667 + ], + "median_ns": 1551667, + "p95_ns": 1662333, + "throughput_per_second": 644.4681751948066, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 46286, + "retained_object_delta": 17523, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 2, + "operation_count": 5, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "dedupe", + "size": 102400, + "concurrency": 2, + "payload_bytes": 102400, + "composition": "same-step duplicate payload" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1944375, + 1952916, + 1919459, + 1960708, + 2185875 + ], + "median_ns": 1952916, + "p95_ns": 2185875, + "throughput_per_second": 512.054793959392, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 5666, + 6833, + 5084, + 5292, + 8250 + ], + "median_ns": 5666, + "p95_ns": 8250, + "throughput_per_second": 176491.35192375575, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 5375, + 5958, + 5125, + 5791, + 11292 + ], + "median_ns": 5791, + "p95_ns": 11292, + "throughput_per_second": 172681.7475392851, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1939000, + 1946958, + 1914334, + 1954917, + 2174583 + ], + "median_ns": 1946958, + "p95_ns": 2174583, + "throughput_per_second": 513.6217627704347, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 869396, + "retained_object_delta": 16774, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 2, + "operation_count": 5, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "dedupe", + "size": 1048576, + "concurrency": 2, + "payload_bytes": 1048576, + "composition": "same-step duplicate payload" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "end_to_end": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 5631667, + 6497875, + 5293666, + 5426625, + 6052000 + ], + "median_ns": 5631667, + "p95_ns": 6497875, + "throughput_per_second": 177.56731710166812, + "within_run_samples_ns": [] + }, + "read_write_gate_wait": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 8375, + 10542, + 6208, + 6708, + 11208 + ], + "median_ns": 8375, + "p95_ns": 11208, + "throughput_per_second": 119402.98507462686, + "within_run_samples_ns": [] + }, + "tool_call": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 7042, + 17666, + 6208, + 7583, + 12083 + ], + "median_ns": 7583, + "p95_ns": 17666, + "throughput_per_second": 131873.92852433075, + "within_run_samples_ns": [] + }, + "framework_overhead": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 5624625, + 6480209, + 5287458, + 5419042, + 6039917 + ], + "median_ns": 5624625, + "p95_ns": 6480209, + "throughput_per_second": 177.78963041980577, + "within_run_samples_ns": [] + }, + "lookup_suggestion": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "json_parse_canonicalize": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "deduplication": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "permission_approval": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "pre_hook": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "post_hook_reminder": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + }, + "telemetry_wire": { + "measurement_status": "unmeasured", + "reason": "current Toolset exposes no stable boundary for this subphase", + "samples_ns": [], + "median_ns": null, + "p95_ns": null, + "throughput_per_second": null, + "within_run_samples_ns": [] + } + }, + "registry_hash": "c739117bf452f76c6574d251f8e498ab5fb71099c5a47b4ae7cd27e71bf315a2", + "allocation_peak_bytes": 8675099, + "retained_object_delta": 16616, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 2, + "operation_count": 5, + "category_counts": { + "builtin": 1 + }, + "projection_counts": { + "visible": 1 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "advertisement", + "size": 50, + "concurrency": 1, + "payload_bytes": null, + "composition": "builtin/plugin/MCP-style names with hidden entries" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "visibility_enabled_hidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 308459, + 407500, + 290458, + 299833, + 284375 + ], + "median_ns": 299833, + "p95_ns": 407500, + "throughput_per_second": 166759.4961195065, + "within_run_samples_ns": [] + }, + "visibility_enabled_unhidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 298833, + 405209, + 299000, + 296709, + 292167 + ], + "median_ns": 298833, + "p95_ns": 405209, + "throughput_per_second": 167317.53186562395, + "within_run_samples_ns": [] + }, + "visibility_disabled_hidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 7416, + 14792, + 6917, + 7208, + 6833 + ], + "median_ns": 7208, + "p95_ns": 14792, + "throughput_per_second": 6936736.958934518, + "within_run_samples_ns": [] + }, + "visibility_disabled_unhidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 5708, + 8000, + 5750, + 5792, + 5375 + ], + "median_ns": 5750, + "p95_ns": 8000, + "throughput_per_second": 8695652.173913043, + "within_run_samples_ns": [] + }, + "repeated_unchanged_projection": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1184583, + 1580708, + 1208833, + 1178041, + 1202334 + ], + "median_ns": 1202334, + "p95_ns": 1580708, + "throughput_per_second": 41585.78232005416, + "within_run_samples_ns": [] + }, + "registry_projection_p95": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 387417, + 402375, + 317042, + 311375, + 308875 + ], + "median_ns": 317042, + "p95_ns": 402375, + "throughput_per_second": 157707.81158332335, + "within_run_samples_ns": [ + [ + 293375, + 292834, + 292042, + 293625, + 290417, + 291250, + 291500, + 290667, + 411417, + 295833, + 291208, + 292000, + 293375, + 293417, + 291333, + 286208, + 287000, + 289625, + 387417, + 292500 + ], + [ + 394333, + 390250, + 391500, + 404708, + 393000, + 402375, + 397375, + 391958, + 392750, + 391959, + 391625, + 392833, + 390917, + 393625, + 393083, + 391500, + 390875, + 390916, + 391625, + 392375 + ], + [ + 301458, + 300000, + 307833, + 307792, + 304542, + 317750, + 302917, + 304042, + 317042, + 297209, + 299459, + 297291, + 302625, + 297917, + 301709, + 295000, + 297125, + 295709, + 301417, + 297667 + ], + [ + 300083, + 292583, + 291458, + 291959, + 288500, + 292000, + 288875, + 291959, + 289292, + 296625, + 295833, + 314042, + 311375, + 303542, + 283208, + 290833, + 271708, + 277000, + 296667, + 304041 + ], + [ + 299209, + 300334, + 319959, + 298666, + 298667, + 299542, + 298792, + 299542, + 299417, + 299208, + 298666, + 298250, + 297750, + 298083, + 297750, + 297875, + 298208, + 299125, + 297625, + 308875 + ] + ] + }, + "rebuild_after_mcp_publication": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 60041, + 79750, + 59417, + 54416, + 52958 + ], + "median_ns": 59417, + "p95_ns": 79750, + "throughput_per_second": 841510.005553966, + "within_run_samples_ns": [] + } + }, + "registry_hash": "91398880bf75436e99f298a4b5b639ab91a13d6bb2a211a3fa510015f449fa95", + "allocation_peak_bytes": 6618671, + "retained_object_delta": 5937667, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 1, + "operation_count": 250, + "category_counts": { + "builtin": 17, + "plugin": 17, + "mcp": 16 + }, + "projection_counts": { + "enabled_hidden": 45, + "enabled_unhidden": 50, + "disabled_hidden": 45, + "disabled_unhidden": 50, + "rebuild": 50, + "repeated": 50 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "advertisement", + "size": 500, + "concurrency": 1, + "payload_bytes": null, + "composition": "builtin/plugin/MCP-style names with hidden entries" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "visibility_enabled_hidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 2879167, + 2629000, + 2762500, + 3355750, + 2886375 + ], + "median_ns": 2879167, + "p95_ns": 3355750, + "throughput_per_second": 173661.34024181298, + "within_run_samples_ns": [] + }, + "visibility_enabled_unhidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 3104042, + 2836958, + 3027208, + 3599708, + 3168292 + ], + "median_ns": 3104042, + "p95_ns": 3599708, + "throughput_per_second": 161080.29466096143, + "within_run_samples_ns": [] + }, + "visibility_disabled_hidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 52250, + 45416, + 49042, + 85167, + 47625 + ], + "median_ns": 49042, + "p95_ns": 85167, + "throughput_per_second": 10195342.76742384, + "within_run_samples_ns": [] + }, + "visibility_disabled_unhidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 45667, + 43750, + 44458, + 109084, + 44583 + ], + "median_ns": 44583, + "p95_ns": 109084, + "throughput_per_second": 11215037.121772872, + "within_run_samples_ns": [] + }, + "repeated_unchanged_projection": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 12620958, + 11415500, + 11949958, + 15711666, + 12009292 + ], + "median_ns": 12009292, + "p95_ns": 15711666, + "throughput_per_second": 41634.4277414522, + "within_run_samples_ns": [] + }, + "registry_projection_p95": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 3245000, + 2934625, + 3091542, + 4716500, + 3079834 + ], + "median_ns": 3091542, + "p95_ns": 4716500, + "throughput_per_second": 161731.58896110742, + "within_run_samples_ns": [ + [ + 3067209, + 3215375, + 3104000, + 3077708, + 3076875, + 3069458, + 3076834, + 3146042, + 3085167, + 3112500, + 3188084, + 3278583, + 2996125, + 2918000, + 3118458, + 3028792, + 3021708, + 3029334, + 3245000, + 3208625 + ], + [ + 2835875, + 2915500, + 2965208, + 2934625, + 2791334, + 2884542, + 2813250, + 2794166, + 2868250, + 2796583, + 2873500, + 2827167, + 2820667, + 2852250, + 2848542, + 2824042, + 2793875, + 2862000, + 2793917, + 2830708 + ], + [ + 3021958, + 2973000, + 3091542, + 3036959, + 2995833, + 3030333, + 3130625, + 2973750, + 3026458, + 3022375, + 2985208, + 3001583, + 3017292, + 2961209, + 2965792, + 2986417, + 3082541, + 3013666, + 2995084, + 3016417 + ], + [ + 4828708, + 4716500, + 3721791, + 4208875, + 3502042, + 4561834, + 3790416, + 3700500, + 3607334, + 3701958, + 3869417, + 3327625, + 3385334, + 3247584, + 3696250, + 3785125, + 3389334, + 3447292, + 3195750, + 3556125 + ], + [ + 2989500, + 3020167, + 2988458, + 3006333, + 2997458, + 3038250, + 2977500, + 2962750, + 3014458, + 3017583, + 3027417, + 3079834, + 3072291, + 3051541, + 3027958, + 3169917, + 2975000, + 3066416, + 3018917, + 2983959 + ] + ] + }, + "rebuild_after_mcp_publication": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 415916, + 424167, + 393041, + 442750, + 406958 + ], + "median_ns": 415916, + "p95_ns": 442750, + "throughput_per_second": 1202165.8219448158, + "within_run_samples_ns": [] + } + }, + "registry_hash": "fd65db87d53a9d0420eae828c8caa4d54b5ead8fb090f7ae2878371ca84108d4", + "allocation_peak_bytes": 10840826, + "retained_object_delta": 6296090, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 1, + "operation_count": 2500, + "category_counts": { + "builtin": 167, + "plugin": 167, + "mcp": 166 + }, + "projection_counts": { + "enabled_hidden": 450, + "enabled_unhidden": 500, + "disabled_hidden": 450, + "disabled_unhidden": 500, + "rebuild": 500, + "repeated": 500 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "advertisement", + "size": 5000, + "concurrency": 1, + "payload_bytes": null, + "composition": "builtin/plugin/MCP-style names with hidden entries" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "visibility_enabled_hidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 28667834, + 29471708, + 28609000, + 28554167, + 29836750 + ], + "median_ns": 28667834, + "p95_ns": 29836750, + "throughput_per_second": 174411.50245253966, + "within_run_samples_ns": [] + }, + "visibility_enabled_unhidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 33350708, + 33228000, + 31662209, + 32122500, + 33422709 + ], + "median_ns": 33228000, + "p95_ns": 33422709, + "throughput_per_second": 150475.50258817864, + "within_run_samples_ns": [] + }, + "visibility_disabled_hidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1004625, + 1060417, + 803500, + 774250, + 931250 + ], + "median_ns": 931250, + "p95_ns": 1060417, + "throughput_per_second": 5369127.516778523, + "within_run_samples_ns": [] + }, + "visibility_disabled_unhidden": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 551000, + 587416, + 518750, + 438541, + 507291 + ], + "median_ns": 518750, + "p95_ns": 587416, + "throughput_per_second": 9638554.21686747, + "within_run_samples_ns": [] + }, + "repeated_unchanged_projection": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 130623833, + 130695209, + 125640125, + 122701292, + 128243958 + ], + "median_ns": 128243958, + "p95_ns": 130695209, + "throughput_per_second": 38988.191552852724, + "within_run_samples_ns": [] + }, + "registry_projection_p95": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 32534166, + 32847500, + 32423000, + 31833167, + 33456750 + ], + "median_ns": 32534166, + "p95_ns": 33456750, + "throughput_per_second": 153684.5911464274, + "within_run_samples_ns": [ + [ + 32388792, + 32368500, + 32463750, + 32363875, + 31913750, + 32304167, + 31733875, + 32066458, + 32319583, + 31698458, + 31937333, + 31935917, + 31914708, + 31670167, + 31997334, + 32177500, + 31731791, + 32165541, + 32534166, + 34521000 + ], + [ + 32447042, + 31899833, + 32129042, + 32304750, + 32620542, + 33536500, + 32839417, + 32673583, + 32353750, + 31818625, + 32382250, + 32435792, + 32518167, + 32126375, + 31957416, + 31942667, + 32847500, + 32268792, + 32280458, + 32111958 + ], + [ + 31437083, + 31485042, + 31094042, + 31471042, + 31260292, + 31149958, + 31515583, + 31330625, + 31838250, + 32423000, + 31866083, + 32436375, + 32097875, + 31530708, + 31782042, + 31569958, + 32126750, + 31887333, + 31582666, + 31949750 + ], + [ + 30727208, + 31029000, + 30668750, + 31152708, + 30906833, + 31200833, + 31234042, + 30856250, + 30967833, + 31067375, + 30578042, + 31030250, + 31748250, + 32322291, + 31833167, + 31504208, + 30873750, + 31004875, + 31183917, + 31259709 + ], + [ + 32353625, + 32054458, + 32169584, + 32249375, + 31930209, + 32242167, + 31960000, + 32119333, + 32190125, + 31764125, + 32243583, + 31748917, + 32151750, + 32158625, + 31964208, + 32664500, + 32451708, + 33934833, + 33456750, + 33050792 + ] + ] + }, + "rebuild_after_mcp_publication": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 3804292, + 3752417, + 3576500, + 3673000, + 3697625 + ], + "median_ns": 3697625, + "p95_ns": 3804292, + "throughput_per_second": 1352219.329975322, + "within_run_samples_ns": [] + } + }, + "registry_hash": "875959264842f6837daa22baebc202ca0055c523e46ad47dcd72b690337a549d", + "allocation_peak_bytes": 44992835, + "retained_object_delta": 9746108, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 1, + "operation_count": 25000, + "category_counts": { + "builtin": 1667, + "plugin": 1667, + "mcp": 1666 + }, + "projection_counts": { + "enabled_hidden": 4500, + "enabled_unhidden": 5000, + "disabled_hidden": 4500, + "disabled_unhidden": 5000, + "rebuild": 5000, + "repeated": 5000 + }, + "lifecycle_status": "completed" + }, + { + "fixture": { + "kind": "mcp", + "size": 1, + "concurrency": 1, + "payload_bytes": null, + "composition": "deterministic fake server inventory publication" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "startup_to_ready": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1804356334, + 1802659250, + 1869092334, + 1856234833, + 1867618292 + ], + "median_ns": 1856234833, + "p95_ns": 1869092334, + "throughput_per_second": 0.5387249405204971, + "within_run_samples_ns": [] + }, + "mcp_lifecycle": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 7844250, + 10510792, + 8468584, + 7330250, + 7034417 + ], + "median_ns": 7844250, + "p95_ns": 10510792, + "throughput_per_second": 127.48191350352168, + "within_run_samples_ns": [] + }, + "time_to_first_inventory": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 7808625, + 10483834, + 8436167, + 7302792, + 7007292 + ], + "median_ns": 7808625, + "p95_ns": 10483834, + "throughput_per_second": 128.0635195056748, + "within_run_samples_ns": [] + }, + "time_to_settled_inventory": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 7844250, + 10510792, + 8468584, + 7330250, + 7034417 + ], + "median_ns": 7844250, + "p95_ns": 10510792, + "throughput_per_second": 127.48191350352168, + "within_run_samples_ns": [] + }, + "cleanup": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 192875, + 158375, + 167042, + 141875, + 135583 + ], + "median_ns": 158375, + "p95_ns": 192875, + "throughput_per_second": 6314.127861089187, + "within_run_samples_ns": [] + } + }, + "registry_hash": "dbb4989c3be2949f3beb5de69201d9a11de843d0cdb7622ed4cdee85e624c7af", + "allocation_peak_bytes": 6609949, + "retained_object_delta": 5886892, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 2, + "operation_count": 5, + "category_counts": { + "mcp": 1 + }, + "projection_counts": { + "visible": 1, + "visible_at_first_publication": 1 + }, + "lifecycle_status": "settled" + }, + { + "fixture": { + "kind": "mcp", + "size": 10, + "concurrency": 10, + "payload_bytes": null, + "composition": "deterministic fake server inventory publication" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "startup_to_ready": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1851220833, + 1825621208, + 1852650875, + 2058521333, + 2179659417 + ], + "median_ns": 1852650875, + "p95_ns": 2179659417, + "throughput_per_second": 5.397671053376422, + "within_run_samples_ns": [] + }, + "mcp_lifecycle": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 64690458, + 67987750, + 64536625, + 66544416, + 101748458 + ], + "median_ns": 66544416, + "p95_ns": 101748458, + "throughput_per_second": 150.27556932801093, + "within_run_samples_ns": [] + }, + "time_to_first_inventory": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 64650916, + 67953375, + 64499083, + 66490958, + 101318291 + ], + "median_ns": 66490958, + "p95_ns": 101318291, + "throughput_per_second": 150.3963892353604, + "within_run_samples_ns": [] + }, + "time_to_settled_inventory": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 64690458, + 67987750, + 64536625, + 66544416, + 101748458 + ], + "median_ns": 66544416, + "p95_ns": 101748458, + "throughput_per_second": 150.27556932801093, + "within_run_samples_ns": [] + }, + "cleanup": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 337167, + 311209, + 303209, + 349417, + 2235292 + ], + "median_ns": 337167, + "p95_ns": 2235292, + "throughput_per_second": 29658.893070792812, + "within_run_samples_ns": [] + } + }, + "registry_hash": "6ec6fba74b1a57a8542826141c146955b0860dfed1633019e5b384cd43c83a64", + "allocation_peak_bytes": 6575263, + "retained_object_delta": 6038423, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 2, + "operation_count": 50, + "category_counts": { + "mcp": 10 + }, + "projection_counts": { + "visible": 10, + "visible_at_first_publication": 10 + }, + "lifecycle_status": "settled" + }, + { + "fixture": { + "kind": "mcp", + "size": 50, + "concurrency": 50, + "payload_bytes": null, + "composition": "deterministic fake server inventory publication" + }, + "warmups": 1, + "iterations": 5, + "phases": { + "startup_to_ready": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 2225032000, + 2223686458, + 2334151917, + 2500971208, + 2452409292 + ], + "median_ns": 2334151917, + "p95_ns": 2500971208, + "throughput_per_second": 21.42105645988251, + "within_run_samples_ns": [] + }, + "mcp_lifecycle": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 345575792, + 324818125, + 401559000, + 404903458, + 399230458 + ], + "median_ns": 399230458, + "p95_ns": 404903458, + "throughput_per_second": 125.24094541904917, + "within_run_samples_ns": [] + }, + "time_to_first_inventory": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 345512375, + 324756833, + 401506125, + 404853125, + 399169875 + ], + "median_ns": 399169875, + "p95_ns": 404853125, + "throughput_per_second": 125.25995354734623, + "within_run_samples_ns": [] + }, + "time_to_settled_inventory": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 345575792, + 324818125, + 401559000, + 404903458, + 399230458 + ], + "median_ns": 399230458, + "p95_ns": 404903458, + "throughput_per_second": 125.24094541904917, + "within_run_samples_ns": [] + }, + "cleanup": { + "measurement_status": "measured", + "reason": null, + "samples_ns": [ + 1486500, + 1113583, + 1131750, + 1084292, + 1252167 + ], + "median_ns": 1131750, + "p95_ns": 1486500, + "throughput_per_second": 44179.36823503424, + "within_run_samples_ns": [] + } + }, + "registry_hash": "0e78cd556e01ea5799fec4eb49b30e74f35299ddfebc6a0cd5e9d56358792951", + "allocation_peak_bytes": 12412598, + "retained_object_delta": 6774701, + "cancellation": { + "completed": true, + "completion_ns": 0, + "queued_reader_completed": true, + "queued_writer_completed": true, + "recovery_completed": true + }, + "leaks": { + "tasks": 0, + "processes": 0, + "sessions": 0 + }, + "task_count_peak": 2, + "operation_count": 250, + "category_counts": { + "mcp": 50 + }, + "projection_counts": { + "visible": 50, + "visible_at_first_publication": 50 + }, + "lifecycle_status": "settled" + } + ], + "decisions": [ + { + "name": "execution_framework_overhead_percent_short_safe_size_1", + "threshold": 10.0, + "values": [ + 99.37963644214055, + 99.38685464207863, + 99.40959652005226, + 99.5606750062967, + 99.42759011745646 + ], + "rerun_values": null, + "crossing_count": 5, + "median": 99.40959652005226, + "primary_state": "crossed", + "rerun_state": null, + "state": "crossed", + "rerun_required": false + }, + { + "name": "mcp_lifecycle_startup_percent_10_servers", + "threshold": 20.0, + "values": [ + 3.494475475147162, + 3.724088529541228, + 3.4834747264510915, + 3.2326318378746675, + 4.668089757804579 + ], + "rerun_values": null, + "crossing_count": 0, + "median": 3.494475475147162, + "primary_state": "uncrossed", + "rerun_state": null, + "state": "uncrossed", + "rerun_required": false + }, + { + "name": "mcp_cleanup_seconds_1_servers", + "threshold": 6.0, + "values": [ + 0.000192875, + 0.000158375, + 0.000167042, + 0.000141875, + 0.000135583 + ], + "rerun_values": null, + "crossing_count": 0, + "median": 0.000158375, + "primary_state": "uncrossed", + "rerun_state": null, + "state": "uncrossed", + "rerun_required": false + }, + { + "name": "mcp_cleanup_seconds_10_servers", + "threshold": 6.0, + "values": [ + 0.000337167, + 0.000311209, + 0.000303209, + 0.000349417, + 0.002235292 + ], + "rerun_values": null, + "crossing_count": 0, + "median": 0.000337167, + "primary_state": "uncrossed", + "rerun_state": null, + "state": "uncrossed", + "rerun_required": false + }, + { + "name": "mcp_cleanup_seconds_50_servers", + "threshold": 6.0, + "values": [ + 0.0014865, + 0.001113583, + 0.00113175, + 0.001084292, + 0.001252167 + ], + "rerun_values": null, + "crossing_count": 0, + "median": 0.00113175, + "primary_state": "uncrossed", + "rerun_state": null, + "state": "uncrossed", + "rerun_required": false + }, + { + "name": "registry_projection_p95_ms_500_tools", + "threshold": 5.0, + "values": [ + 3.245, + 2.934625, + 3.091542, + 4.7165, + 3.079834 + ], + "rerun_values": null, + "crossing_count": 0, + "median": 3.091542, + "primary_state": "uncrossed", + "rerun_state": null, + "state": "uncrossed", + "rerun_required": false + }, + { + "name": "mixed_gate_wait_end_to_end_percent_10_pairs", + "threshold": 25.0, + "values": [ + 89.70580681682692, + 89.77906557724144, + 89.4544021194392, + 89.82394020571742, + 89.56548823285513 + ], + "rerun_values": null, + "crossing_count": 5, + "median": 89.70580681682692, + "primary_state": "crossed", + "rerun_state": null, + "state": "crossed", + "rerun_required": false + } + ] +} diff --git a/docs/superpowers/reports/2026-07-10-toolset-characterization.md b/docs/superpowers/reports/2026-07-10-toolset-characterization.md new file mode 100644 index 00000000..b151a2ac --- /dev/null +++ b/docs/superpowers/reports/2026-07-10-toolset-characterization.md @@ -0,0 +1,99 @@ +# Toolset characterization decision + +## Decision + +**NO-GO: retain `PythinkerToolset`; do not ship a private extraction.** + +The primary run crossed the execution-framework threshold. The controlled private +`_ToolExecutionPipeline` attempt did not improve that target and was reverted. The mixed gate ratio also crossed, but +that fixture deliberately holds a reader while a writer waits, so the result describes real barrier +contention rather than framework work a second owner could remove. Per the one-module limit, only a +private `_ToolExecutionPipeline` is eligible for the controlled attempt. No MCP lifecycle or registry +threshold crossed, and the 5,000-tool stress fixture is not used as a trigger. + +## Environment and invocation + +- Successful measured command: `uv run python scripts/benchmark_toolset.py --scenario all --runs 5 --output docs/superpowers/reports/2026-07-10-toolset-characterization.json` +- Python: CPython 3.14.6 +- Platform: macOS 26.5.2, arm64 +- Warm-ups: 1 per fixture +- Measured runs: 5 per fixture +- Machine schema: version 2; full all/5 runs derive decisions during report generation +- Machine record: `docs/superpowers/reports/2026-07-10-toolset-characterization.json` +- Directionality: local engineering evidence only; these values are not universal product telemetry. + +The first command attempt completed measurement but could not write because the new report directory +did not exist. The directory was created and the full command was run again; no sample from the +failed write was reused. + +## Deterministic threshold decisions + +All values below were derived from paired raw nanosecond samples in the machine record by the Task 12 +`evaluate_threshold` function. A crossing requires at least four of five values above the threshold +and a crossing median. Exactly one crossing is inconclusive and requires one complete five-run rerun. +No primary decision had exactly one crossing, so no rerun was permitted or performed. + +| Decision | Threshold | Five primary values | Median | Crossings | Primary | Rerun | Final | +| --- | ---: | --- | ---: | ---: | --- | --- | --- | +| Execution framework overhead, short safe size 1 (%) | 10 | 99.379636, 99.386855, 99.409597, 99.560675, 99.427590 | 99.409597 | 5/5 | crossed | not required | crossed | +| MCP lifecycle / startup-to-ready, 10 servers (%) | 20 | 3.494475, 3.724089, 3.483475, 3.232632, 4.668090 | 3.494475 | 0/5 | uncrossed | not required | uncrossed | +| MCP cleanup, 1 server (s) | 6 | 0.000192875, 0.000158375, 0.000167042, 0.000141875, 0.000135583 | 0.000158375 | 0/5 | uncrossed | not required | uncrossed | +| MCP cleanup, 10 servers (s) | 6 | 0.000337167, 0.000311209, 0.000303209, 0.000349417, 0.002235292 | 0.000337167 | 0/5 | uncrossed | not required | uncrossed | +| MCP cleanup, 50 servers (s) | 6 | 0.001486500, 0.001113583, 0.001131750, 0.001084292, 0.001252167 | 0.001131750 | 0/5 | uncrossed | not required | uncrossed | +| Registry within-run projection p95, 500 tools (ms) | 5 | 3.245000, 2.934625, 3.091542, 4.716500, 3.079834 | 3.091542 | 0/5 | uncrossed | not required | uncrossed | +| Mixed gate wait / end-to-end, 10 pairs (%) | 25 | 89.705807, 89.779066, 89.454402, 89.823940, 89.565488 | 89.705807 | 5/5 | crossed | not required | crossed | + +Each registry value is the nearest-rank p95 of twenty individually timed, visibility-enabled, +unchanged 500-tool projections from that run. The machine report preserves all five groups of twenty +raw samples. The decision excludes both single-sample hidden/unhidden timings and the 5,000-tool +stress result. + +## Safety and fault matrix + +- Every scenario reported zero leaked tasks, processes, and sessions. +- Every execution scenario reported completed queued-reader cancellation, queued-writer cancellation, + and later-call recovery. +- Registry hashes were stable within every five-run fixture; the harness rejects a run if hashes vary. +- Deterministic event/barrier coverage exercises PreToolUse block preservation under telemetry failure, + hook failure policy, post-hook isolation, queued and admitted cancellation, permit recovery, optional + MCP method absence versus transient failure, list-change storms, refresh racing disconnect, hung + connect, duplicate server and tool names, background-load cleanup, close failure/timeout, partial + connection, and exception-atomic publication rollback. +- A fault test reproduced a real half-publication defect in `_rebuild_published_mcp_tools`; the scoped + fix restores both public registries before re-raising the publication error. + +## Non-timing locality and consumer evidence + +- `git log --since=2026-04-01 -- src/pythinker_code/soul/toolset.py` shows repeated lifecycle edits. + In particular, `2904de00` changed deduplication inside `handle`, `5e505087` added gate execution and + approval/event behavior, `63faa855` changed skip-event policy, and `e93c0c17` changed execution + telemetry. This satisfies the independent recent-change locality trigger for one controlled + execution-pipeline attempt. +- Targeted private-state search found no second production owner of `_concurrency_gate` or + `_current_step_tasks`; only the characterization probe subclasses the Toolset for measurement. +- MCP consumers use the Toolset facade (`mcp_status_snapshot`, `wait_for_mcp_tools`, refresh, + disconnect, reconnect, and `mcp_servers`). No production module directly owns `_mcp_loading_task`. + The public MCP state has multiple consumers, but direct lifecycle state remains local to Toolset, + so consumer count does not trigger `_McpLifecycle` extraction. + +## Controlled extraction result + +The attempt moved the reader/writer gate and its execution call ownership into the only permitted +private `_ToolExecutionPipeline`, removed the equivalent gate state/logic from `PythinkerToolset`, +and kept `PythinkerToolset.handle` as the facade. The focused fault matrix remained green (68 tests). + +Against the original primary median of 99.568352 percent, the identical short-safe size-1 extraction +fixture produced framework-overhead ratios of 99.519399, +99.606995, 99.589209, 99.645530, and 99.624502 percent; median 99.606995 percent. The attempt was +slightly worse, remained far above the 10 percent threshold, +and moved only gate admission rather than the broader frequently changed handle lifecycle. It +therefore failed both the measured-target improvement rule and the locality/depth test. + +The post-review schema-v2 rerun emitted decisions directly from the benchmark command and again +crossed the execution threshold, with a 99.409597 percent median. This does not change the failed +extraction comparison or authorize a second attempt. + +The private module, compatibility projection, and probe adaptation were reverted in full. The final +decision is **NO-GO**: retain `PythinkerToolset` and the colocated `_ReadWriteGate`. Keep the +characterization/fault tests, machine and human evidence, and exception-atomic publication fix. New +evidence is required before proposing another split. diff --git a/docs/superpowers/specs/2026-07-10-agent-catalogue-design.md b/docs/superpowers/specs/2026-07-10-agent-catalogue-design.md new file mode 100644 index 00000000..80905b03 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-agent-catalogue-design.md @@ -0,0 +1,212 @@ +# Resolved agent catalogue design + +**Status:** Approved as phase 5 of the agent core deepening program + +## Problem + +Pythinker resolves built-in and configured YAML agents through recursive `AgentSpec` inheritance, while external Markdown agents use separate discovery, frontmatter parsing, materialized wrapper files, and `AgentTypeDefinition` construction. The two paths converge late through mutable `LaborMarket` registration. Validation, identity normalization, source precedence, collision handling, and diagnostics therefore lack one owner. + +Unknown YAML and Markdown fields are currently ignored. This hides misspellings and stale configuration while making an immediate strict transition risky for existing custom agents. + +## Goals + +- Resolve YAML and Markdown sources into one immutable catalogue. +- Keep source-specific parsing behind adapters. +- Centralize identity, precedence, provenance, collision policy, and diagnostics. +- Warn for unknown fields during one compatibility release and support strict rejection through the same implementation. +- Preserve launch behavior through `AgentTypeDefinition`, `LaborMarket`, and generated-wrapper compatibility adapters. +- Keep runnable `Agent` loading outside the catalogue. + +## Non-goals + +- Introducing remote or plugin-defined catalogue provider protocols. +- Removing generated Markdown wrappers before direct launch parity is proven. +- Moving Runtime, MCP connection, tool construction, or prompt rendering into the catalogue. +- Changing tool-policy semantics or built-in agent names. +- Making optional malformed Markdown agents fatal to startup. + +## Module and interface + +`pythinker_code.subagents.catalogue` becomes the deep module. + +```python +class UnknownFieldPolicy(StrEnum): + WARN = "warn" + FORBID = "forbid" + +@dataclass(frozen=True, slots=True) +class AgentProvenance: + source_kind: str + source_id: str + scope: str + precedence: int + +@dataclass(frozen=True, slots=True) +class AgentDiagnostic: + source_kind: str + safe_path: str + field_path: str | None + severity: str + reason_code: str + message: str + +@dataclass(frozen=True, slots=True) +class ResolvedAgentEntry: + name: str + normalized_name: str + description: str + launch_spec: ResolvedAgentSpec + required_mcp_servers: tuple[str, ...] + supports_background: bool + provenance: AgentProvenance + legacy_agent_file: Path | None + +@dataclass(frozen=True, slots=True) +class ResolvedAgentCatalogue: + entries: Mapping[str, ResolvedAgentEntry] + diagnostics: tuple[AgentDiagnostic, ...] + + def get(self, name: str) -> ResolvedAgentEntry | None: ... + def require(self, name: str) -> ResolvedAgentEntry: ... + def values(self) -> tuple[ResolvedAgentEntry, ...]: ... +``` + +The concrete field names may adapt during implementation planning, but the catalogue must preserve immutable resolved entries and source-aware diagnostics. `source_id` is a safe unique identifier formed from source kind, scope, a trusted logical root label or resolution-order ordinal, and a root-relative path or basename. Raw source-discovery provenance paths are not stored in `AgentProvenance`. Launch paths required by `ResolvedAgentSpec` and `legacy_agent_file` remain internal execution fields on the entry, but no raw launch or provenance path is projected into model context, UI output, telemetry, or diagnostics. + +## Source adapters + +### YAML adapter + +The YAML adapter retains recursive inheritance, cycle detection, relative path rebasing, child overrides, merged system-prompt arguments, subagent merging, and existing required-field validation. + +Before Pydantic projection, it inspects raw top-level, `agent`, and nested subagent mappings for unknown fields. Diagnostics use stable field paths such as `agent.unknown_key` and `agent.subagents.coder.unknown_key`. Raw values are never logged. + +Referenced YAML is required. Missing, malformed, cyclic, unsupported-version, or invalid known fields remain fail-closed with an actionable `AgentSpecError` causal chain. + +### Markdown adapter + +The Markdown adapter preserves documented root precedence, canonical path de-duplication, frontmatter aliases, tool mapping, required MCP servers, model selection, and fail-soft file isolation. + +It inspects frontmatter for unknown fields before projection. Malformed optional Markdown files produce diagnostics and are skipped without failing the entire runtime. Unexpected programming errors are not mislabeled as harmless configuration when they cannot be handled correctly. + +The Markdown adapter constructs a complete `ResolvedAgentSpec`, including system-prompt path and arguments, model, mode, steps, temperature, top-p, tools, allowed tools, excluded tools, hidden state, usage guidance, and nested subagents. Generated YAML and prompt files remain a launch compatibility adapter during the warning release and are referenced only through `legacy_agent_file`. They are not the canonical catalogue representation. + +## Identity and precedence + +Catalogue identity uses one named function, `normalize_agent_name(name)`, whose warning-release implementation is exactly `name.casefold()`. Display names remain unchanged. `ResolvedAgentCatalogue.get` uses normalized lookup, while the `LaborMarket` compatibility adapter preserves current exact-key behavior until its removal window. + +Precedence is explicit: + +1. YAML-declared built-in and configured subagents. +2. Project Markdown roots in this order: `.pythinker/agents`, `.claude/agents`, `.agents/agents`, then `.codex/agents`. +3. Enabled plugin Markdown roots in the deterministic order returned by plugin integration, below every project root. + +An entry at higher precedence wins over the same normalized name at lower precedence and produces a diagnostic identifying the shadowed source. Two entries with the same normalized name at the same precedence do not silently overwrite: + +- Required YAML collision: fail catalogue resolution. +- Optional Markdown collision: keep deterministic first-wins behavior during compatibility rollout and emit a warning diagnostic. +- Strict rollout may reject same-precedence Markdown collisions after documentation and tests establish the migration. + +Enumeration order is deterministic by normalized name after resolution. Lookup is case-insensitive and preserves current aliases where documented. + +## Unknown-field rollout + +The validation implementation supports both `WARN` and `FORBID` from its first release. Production composition uses `WARN` for the first release containing this catalogue, changes to `FORBID` in the immediately following minor release, and cannot remove compatibility adapters before one additional minor release has begun. + +During the warning release: + +- Emit one aggregated warning per canonical source. +- List stable sorted field paths, not values. +- Preserve current ignore behavior after warning. +- Deduplicate inherited-source warnings by canonical path and field path. +- Make warning diagnostics visible through existing startup/log surfaces. +- Add release notes with the exact strict transition. + +In the following release, production composition changes to `FORBID`: + +- Required YAML unknown fields raise `AgentSpecError`. +- Optional Markdown entries with unknown fields are rejected as invalid entries and reported through diagnostics. +- No hidden environment variable or local bypass disables strict production validation. + +Tests for `FORBID` ship during the warning release, so strict behavior is already exercised before the default changes. + +## Compatibility adapters + +`ResolvedAgentEntry` projects to existing `AgentTypeDefinition` while launch callers still require it. The projection uses `legacy_agent_file` during the wrapper-compatibility window and uses `launch_spec` for full parity assertions. `LaborMarket` preserves its exact current interface: `builtin_types`, `add_builtin_type`, `get_builtin_type`, and `require_builtin_type`. + +`Runtime.labor_market` remains through the warning release and the following strict-default release. Internal callers migrate to `runtime.agent_catalogue`; the earliest adapter removal is the next minor release after strict validation becomes the default. Generated wrappers follow the same minimum window and remain longer if direct-launch parity is not green. + +Generated Markdown wrappers remain until `load_agent` can consume a resolved entry or prompt source directly with equivalent prompt, model, tool policy, and required-MCP behavior. Their removal is a separate deletion step, not bundled into initial catalogue resolution. + +The catalogue does not construct runnable `Agent` instances. Runtime/model/tool dependencies remain in `soul.agent`, preserving a clean seam between declarative definition and active execution. + +## Diagnostics and error contract + +Diagnostics distinguish: + +- Unknown field warning or rejection. +- Shadowed lower-precedence entry. +- Same-precedence collision. +- Invalid known field. +- Missing required source. +- Unsupported schema version. +- Inheritance cycle. +- Unreadable optional source. +- Materialization failure. + +Required source errors stop runtime creation. Optional-source errors remain visible but do not claim the optional entry was loaded. Messages use `source_id` or an existing safe-path renderer and never include raw absolute provenance paths, frontmatter values, prompt contents, credentials, or stack traces. + +## Test design + +Tests are written before implementation and must first fail for the intended missing behavior. + +### Validation + +- Unknown top-level YAML field warns exactly once under `WARN`. +- Unknown `agent` field warns exactly once. +- Unknown nested subagent field reports its stable field path. +- Inherited unknown fields warn once for the canonical defining file. +- Unknown Markdown frontmatter warns exactly once. +- The same fixtures reject under `FORBID`. +- Raw unknown values and absolute provenance paths never appear in diagnostics, catalogue entries, UI projections, model context, or telemetry. + +### Resolution + +- YAML inheritance produces the same resolved behavior as current snapshots. +- Markdown roots preserve documented precedence. +- Required YAML beats project and plugin Markdown. +- Case-only collisions follow the approved required/optional policy. +- Reversed discovery input produces identical catalogue ordering. +- Required MCP servers, model, tools, exclusions, background support, and prompt paths survive projection. +- Malformed optional Markdown is skipped with a diagnostic. +- Malformed required YAML remains fatal. + +### Compatibility + +- `AgentTypeDefinition` projection matches existing launch fixtures. +- `LaborMarket` lookup and enumeration remain compatible. +- Root and subagent runtimes share the resolved catalogue. +- Agent-list injection and `/agents` output remain stable. +- Markdown launch continues through generated wrappers during the compatibility release. + +Focused agent-spec, discovery, load-agent, agent-list, agent-tool, slash, and snapshot tests run before the full Pythinker Code gate. + +## Migration and deletion + +1. Characterize current YAML and Markdown projections, precedence, and launch behavior. +2. Add unknown-field inspection with `WARN` and `FORBID` policies. +3. Introduce immutable entries, diagnostics, and catalogue resolution. +4. Add YAML and Markdown source adapters using existing parsers. +5. Project catalogue entries to `AgentTypeDefinition` and `LaborMarket`. +6. Publish the catalogue on Runtime and migrate internal readers. +7. Ship the warning release and documentation. +8. Change production policy to `FORBID` in the following release. +9. Teach launch to consume resolved entries directly. +10. Delete generated-wrapper materialization and duplicate DTOs only after parity tests. +11. Remove `LaborMarket` after the documented compatibility window and repository-wide call-site check. + +The deletion test passes when removing `ResolvedAgentCatalogue` would spread validation, precedence, normalized identity, collision, provenance, and diagnostics back across YAML loading, Markdown discovery, runtime composition, and the registry. + +## Rollback + +The warning release is additive and can be reverted without invalidating existing specs. Strict rollout is reverted by restoring the previous released code, not by a hidden bypass. Catalogue projection remains compatible through the warning release and the following strict-default release. The earliest removal release is the next minor after strict becomes default, and removal still requires direct-launch parity and migration documentation. diff --git a/docs/superpowers/specs/2026-07-10-agent-core-deepening-design.md b/docs/superpowers/specs/2026-07-10-agent-core-deepening-design.md new file mode 100644 index 00000000..4e7e38cc --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-agent-core-deepening-design.md @@ -0,0 +1,170 @@ +# Agent core deepening program design + +**Status:** Approved + +**Date:** 2026-07-10 + +## Purpose + +Pythinker Code currently assembles model requests, skill metadata, dynamic guidance, persisted context, agent definitions, and tool execution through several high-capability modules. The audit found that some of those modules are shallow at their current seams: policy and state are reconstructed across callers, failure outcomes disappear, or large internal catalogues leak into model context. + +This program deepens those modules without replacing the agent loop or provider layer. Each phase is independently reviewable and releasable. Compatibility adapters remain only for a named migration window and are deleted after callers move to the new interface. + +## Goals + +- Reduce the default skill catalogue from hundreds of thousands of prompt characters to at most 8,000 characters while preserving deterministic discovery and recall. +- Give every provider request one observable assembly path with provenance, authority, ordering, persistence, budget, and degradation outcomes. +- Make context history replacement atomic from the caller's perspective and make normal appends disk-first. +- Resolve YAML and Markdown agent definitions through one immutable catalogue with source-aware diagnostics. +- Characterize Toolset execution and MCP lifecycle before deciding whether private extraction is justified. +- Preserve existing provider interfaces, JSONL records, tool names, slash behavior, agent precedence, and public configuration unless a phase explicitly documents a migration. + +## Non-goals + +- Replacing `PythinkerSoul`, `pythinker_core.step`, or provider adapters. +- Adding third-party runtime dependencies. +- Adding cross-process writes to one session file. +- Persisting raw model prompts or request manifests. +- Introducing a generic contribution framework shared by unrelated domains. +- Splitting `PythinkerToolset` because of file length alone. +- Removing generated Markdown agent wrappers before direct launch parity is proven. + +## Architecture + +The program uses contract-first staged deepening. Each deep module owns the policy that belongs at its seam, exposes a narrow interface, and keeps compatibility projections outside its implementation. + +1. `SkillCatalog` owns skill precedence, normalized indexing, deterministic task search, exact resolution, and bounded prompt projection. +2. `RequestAssembler` owns request fragments, requiredness, ordering, budgeting, persistence policy, sanitized provenance, and provider-ready history. +3. `Context` owns atomic history replacement, rotation archives, disk-first append semantics, and coherent in-memory state. +4. `ResolvedAgentCatalogue` owns source adapters, validation, precedence, normalized identity, collisions, provenance, and diagnostics. +5. `PythinkerToolset` remains the external interface. Benchmarks and deterministic fault tests decide whether private execution or MCP lifecycle extraction would increase depth and locality. + +The deletion test governs every phase. Once migrated, deleting a deep module must cause its policy and state logic to reappear across several callers. A compatibility adapter that only forwards calls does not count as the new module and must have a removal condition. + +## Approved defaults + +### Delivery + +The work ships as independently reviewable phases on one umbrella branch. Every phase has focused red-green-refactor cycles, its own review gate, documentation, and changelog entry when shipped code changes. + +### Skill discovery + +- Search the complete in-memory catalogue deterministically and return a bounded task-relevant view. +- Preserve exhaustive exact-name resolution and scope precedence. +- Preserve an exhaustive compatibility mapping while callers migrate. +- Require 100 percent expected recall on the approved representative fixture. +- Cap rendered candidate content at 8,000 characters. +- Require warm in-memory retrieval median below 100 ms on the 1,000-entry benchmark fixture. +- Keep an exhaustive fallback and do not remove it until recall parity tests pass. + +### Request assembly + +- Keep the sanitized manifest in memory only. +- Expose the latest manifest through `/prompt-manifest`. +- Treat AGENTS.md as authoritative and non-budgeted. +- Treat permissions and applicable model-defense guidance as required. +- Treat other current dynamic providers as best-effort unless a separate invariant proves requiredness. +- Required fragments fail closed and are never truncated. +- Optional failure produces explicit degradation. + +### Context persistence + +- Use same-directory temporary output and atomic replacement for full history rewrites. +- Preserve numbered rotation archives. +- Serialize in-process writes with one lock. +- Persist normal appends before mutating memory. +- Preserve current JSONL records and torn-tail restoration. +- Keep Context on its current local `Path` seam, do not claim cross-process safety, and distinguish POSIX directory synchronization from platforms that provide visibility atomicity only. + +### Agent definitions + +- Warn once per source for unknown YAML and Markdown fields during one compatibility release. +- Support strict rejection in the same validation module and test it during the warning release. +- Change production to strict rejection in the following release. +- Preserve required YAML fail-closed behavior and optional Markdown fail-soft behavior. + +### Toolset + +- Add no split before measurements and deterministic fault characterization. +- Treat a no-go decision as a valid completed phase. +- Keep `PythinkerToolset.handle` and existing MCP methods compatible if private modules are extracted. + +## Phase sequence + +### Phase 1: Characterization and compatibility contracts + +Capture the current provider handoff, static prompt behavior, dynamic reminder ordering, context JSONL records, agent projections, and Toolset lifecycle. Add reusable fixtures and benchmarks before changing implementation behavior. + +### Phase 2: Bounded skill discovery + +Introduce `SkillCatalog`, preserve `Runtime.skills` as a compatibility adapter, and move live and prompt-inspection rendering onto one catalogue implementation. A narrow soul adapter projects bounded task-relevant candidates into provider-visible history without persistence until Phase 3 absorbs that projection. + +Detailed design: [Bounded skill catalogue](./2026-07-10-skill-catalogue-design.md). + +### Phase 3: Observable request assembly + +Introduce typed request fragments and outcomes, reserve required guidance, produce a sanitized manifest, preserve provider handoff bytes under compatibility conditions, absorb and delete the Phase 2 skill-projection adapter, and add `/prompt-manifest`. + +Detailed design: [Observable request assembly](./2026-07-10-request-assembly-design.md). + +### Phase 4: Transactional context persistence + +Add semantic full-history replacement, migrate pruning, compaction, revert, and clear flows, make appends disk-first, and delete compensating clear/rebuild rollback logic. + +Detailed design: [Transactional context persistence](./2026-07-10-context-transactions-design.md). + +### Phase 5: Resolved agent catalogue + +Unify source resolution and diagnostics while retaining `LaborMarket`, `AgentTypeDefinition`, and generated wrappers as compatibility adapters for the warning release. + +Detailed design: [Resolved agent catalogue](./2026-07-10-agent-catalogue-design.md). + +### Phase 6: Toolset characterization and conditional deepening + +Measure execution phases, advertisement, concurrency, cancellation, and MCP lifecycle. Extract private modules only when a documented threshold is crossed. + +Detailed design: [Toolset characterization](./2026-07-10-toolset-characterization-design.md). + +## Cross-phase data flow + +A user message is persisted through `Context`. In Phase 2, a temporary soul adapter asks `SkillCatalog` for a bounded candidate view and adds it only to provider-visible history. In Phase 3, `RequestAssembler` takes ownership of that projection, reads the current task, collects typed provider fragments, admits required fragments first, applies optional budgets, and constructs provider-ready history plus explicit history appends. It stores only a sanitized `RequestManifest` in memory and passes the unchanged provider argument shape to `pythinker_core.step`. + +Agent definitions are resolved before tool construction into `ResolvedAgentCatalogue`; compatibility projections feed existing launch and UI callers until those callers migrate. Toolset characterization observes the resulting request/tool path without changing its external interface. + +## Failure truthfulness + +- Required assembly failure stops before provider invocation and before assembly-owned history mutation. +- Optional assembly failure records a degraded outcome and remains visible to the user through the sanitized manifest. +- Explicitly requested malformed skills return unavailable, not not-found. +- Context replacement leaves the old committed generation authoritative until atomic commit. +- Context cancellation is propagated after any cancellation-shielded commit section leaves memory and disk coherent. +- Required YAML agent errors fail startup. Optional Markdown errors remain isolated diagnostics. +- Toolset profiling does not add silent fallbacks or convert failures into success. + +## Compatibility and removal policy + +Compatibility adapters are temporary and named in tests and release notes. + +- `Runtime.skills` remains an exhaustive mapping until every internal lookup uses `SkillCatalog`. +- Existing injection providers are adapted to typed fragments; they are not duplicated as a second business-logic path. +- Existing Context methods delegate to the new implementation until all callers migrate. +- `LaborMarket` and `AgentTypeDefinition` remain during the unknown-field warning release. +- `PythinkerToolset` remains the caller-facing interface even if private implementation modules are extracted. + +An adapter is removed only after repository search proves no internal caller depends on it, focused compatibility tests pass, public documentation includes the removal timing, and the deletion does not widen the interface elsewhere. + +## Verification strategy + +Each phase begins with a failing behavior test and records the expected failure. Focused tests cover success, malformed input, required failure, optional degradation, cancellation, concurrency, and compatibility. Shared-module changes run `make check-pythinker-code` and `make test-pythinker-code` before the phase is declared complete. + +The baseline on the isolated branch had a clean static gate. The first full test run produced one timing-dependent PTY cancellation timeout after 6,654 passes; the immediate focused rerun passed. This is recorded as a pre-existing baseline flake and is not part of the architecture scope. + +## Documentation and release obligations + +Each shipped-code phase adds an entry under `## Unreleased` in `CHANGELOG.md`. Public syntax and behavior changes update the relevant customization or reference page. The generated documentation changelog is updated only through the documented sync command. + +The one-release unknown-field warning and following strict transition require explicit release notes. `/prompt-manifest` requires slash-command documentation and wire/CLI compatibility tests. + +## Rollback + +Each phase remains independently revertible. Compatibility adapters preserve the old caller shape during rollout. Feature behavior is not controlled by hidden environment flags. If a phase fails its compatibility or performance gate, revert that phase rather than adding a parallel fallback implementation. diff --git a/docs/superpowers/specs/2026-07-10-context-transactions-design.md b/docs/superpowers/specs/2026-07-10-context-transactions-design.md new file mode 100644 index 00000000..abe6352d --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-context-transactions-design.md @@ -0,0 +1,211 @@ +# Transactional context persistence design + +**Status:** Approved as phase 4 of the agent core deepening program + +## Problem + +`Context` owns JSONL restoration and append operations, but pruning, compaction, revert, and clear callers orchestrate destructive rotation followed by incremental rebuilding. Those callers duplicate compensating rollback logic. Cancellation can bypass `Exception` handlers, rollback can fail, and normal append methods update memory before persistence succeeds. + +The module must make a committed context generation truthful: memory and the live JSONL file either remain at the old state or advance coherently to the new state. + +## Goals + +- Put full-history replacement behind one semantic `Context` interface. +- Preserve existing JSONL record shapes and restoration compatibility. +- Preserve numbered rotation archives. +- Make normal append, checkpoint, and usage operations disk-first. +- Serialize concurrent operations within one `Context` instance. +- Remove duplicated clear/rebuild/rollback logic from the soul. +- Propagate cancellation without leaving split state. + +## Non-goals + +- Supporting multiple processes writing one session concurrently. +- Introducing a second persistence backend. +- Rewriting every normal append through a full-file replacement. +- Persisting repaired synthetic tool pairs unless they are part of the intended semantic history. +- Moving compaction model calls, hooks, telemetry, or wire events into `Context`. +- Expanding persistence beyond the current local `Path` filesystem seam. + +## Module and interface + +`Context` remains the deep module. It gains a semantic replacement interface while existing methods delegate during migration. + +```python +@dataclass(frozen=True, slots=True) +class ContextReplacement: + system_prompt: str | None + messages: tuple[Message, ...] + token_count: int + create_checkpoint: bool + checkpoint_user_marker: bool = False + +@dataclass(frozen=True, slots=True) +class ContextCommit: + checkpoint_id: int | None + rotated_file: Path | None + message_count: int + +class ContextPersistenceError(OSError): + operation: str + category: str + +class Context: + async def replace_history( + self, + replacement: ContextReplacement, + ) -> ContextCommit: ... + + async def append_messages( + self, + messages: Sequence[Message], + ) -> None: ... +``` + +The concrete names may adapt to repository conventions during planning. Callers provide intended semantic state, not JSONL records or file-operation sequences. + +## Internal state model + +A private state reducer computes the complete post-commit in-memory state before any live mutation: + +- History. +- System prompt. +- Authoritative token count. +- Pending token estimate. +- Next checkpoint ID. +- Torn-tail repair state. + +The same record serializer and reducer are used by restore, replacement, and compatibility methods. This avoids parallel logic that could encode different JSONL semantics. + +An instance-level `asyncio.Lock` serializes all mutating methods. It prevents two tasks sharing one `Context` from interleaving commits. It does not claim cross-process protection. + +## Full-history replacement algorithm + +1. Validate the semantic replacement without touching live state. +2. Acquire the context mutation lock. +3. Reconfirm any generation assumptions established before the lock. +4. Derive the new immutable in-memory state and compatible JSONL records. +5. Create a restrictive same-directory temporary file with a unique name. +6. Write the complete record sequence in a synchronous local-filesystem helper executed through `asyncio.to_thread`. +7. Flush and call `os.fsync` on the temporary file before replacement. +8. Reserve and write the numbered rotation archive from the old live file. +9. If archival fails, clean the temporary file and leave old disk and memory unchanged. +10. Atomically replace the live file with the prepared temporary file. +11. Swap the precomputed in-memory state without an intervening await. +12. On POSIX, open and synchronize the parent directory after replacement. On platforms that do not support directory synchronization, document visibility atomicity without claiming equivalent power-loss durability. +13. Release the lock and return `ContextCommit`. + +The replace plus in-memory swap is a minimal cancellation-shielded critical section. Cancellation before commit removes temporary state and propagates with old state intact. Cancellation arriving during commit is re-raised only after disk and memory become coherent. + +If supported directory synchronization fails after atomic replacement, the operation reports a categorized durability error while retaining the new coherent visible generation. It must not attempt a destructive compensating rollback. The error message distinguishes visible commit from uncertain power-loss durability. A known unsupported platform capability is not reported as a failed commit and is covered by a platform-specific test. + +## Rotation archives + +Numbered archives remain observable recovery artifacts. Replacement writes the archive before replacing the live file. Archive failure blocks commit. + +The archive contains the exact pre-commit live bytes, including any recoverable torn tail. It is not reconstructed from in-memory repaired history. This preserves forensic and manual recovery value. + +Archive placeholders and temporary files are cleaned under `BaseException`. Cleanup failure is logged with the original failure retained as the primary cause. + +## Disk-first append operations + +Normal conversation growth does not rewrite the complete file. `append_messages`: + +1. Validates and serializes the entire batch before acquiring the lock. +2. Acquires the mutation lock. +3. Appends the serialized batch in one local-file open/write operation. +4. Flushes before reporting success; normal append retains the existing torn-tail recovery contract rather than claiming full batch crash atomicity. +5. Updates the precomputed in-memory state only after the append succeeds. + +A process crash may still leave a torn final JSONL record; existing restore repair remains the recovery contract. A returned successful append guarantees that the implementation observed a successful host write before memory advanced. + +Checkpoint plus optional user marker is serialized as one batch. Token-count records are persisted before authoritative and pending counters change. A failed append leaves memory unchanged and raises `ContextPersistenceError`. + +## Pruning and compaction + +Pruning and compaction continue to prepare semantic replacement messages in `PythinkerSoul`. Model calls, hooks, restore reminders, telemetry, and wire begin/end events stay outside `Context`. + +Only after all required preparation succeeds does the soul call `replace_history` once. Hook failure before that call leaves the old context untouched. The soul no longer calls clear, checkpoint, append, and update-token-count as a replacement protocol and no longer performs a compensating rebuild. + +Compaction cancellation before commit leaves the old generation. Cancellation during the commit follows the coherent-commit rule above. Wire completion remains in the existing `finally` block. + +## Revert and clear + +`revert_to` prepares the target semantic history and commits it through the same replacement implementation. The selected checkpoint and marker semantics remain unchanged. + +Clear becomes a semantic reset that includes the current system prompt in one replacement. `/clear` no longer performs clear followed by a separate system-prompt write. + +Existing public methods remain compatibility adapters during migration. They cannot retain a second destructive implementation. + +## Error contract + +`ContextPersistenceError` preserves the causal exception and classifies: + +- Temporary creation. +- Serialization. +- Write or flush. +- Synchronization. +- Rotation archive. +- Atomic replacement. +- Cleanup. +- Visible commit with uncertain power-loss durability. + +Errors contain the operation and a safely rendered session-relative `Path` when available. They do not include message content, prompt text, credentials, or stack traces in user-facing output. + +Invalid semantic input raises a validation error before filesystem work. Unsupported cross-process conflict is documented rather than silently treated as safe. + +## Test design + +Tests are written before implementation and must first fail for the intended missing behavior. + +### Compatibility + +- `Context.file_backend` remains a local `Path`; no host abstraction or remote backend is introduced. +- Existing system-prompt, checkpoint, message, and usage records restore unchanged. +- Legacy files without system prompt remain readable. +- Malformed and truncated final records retain current repair behavior. +- Tool-call pairing repair remains in memory and does not rewrite source unexpectedly. +- Rotation naming and archive bytes remain compatible. + +### Failure injection + +Inject failures at temporary creation, each record write, flush, local synchronization, archive creation, atomic replacement, directory synchronization, and cleanup. + +For every pre-commit failure, assert exact old live bytes and exact old memory. For a successful commit, assert exact new live bytes and derived memory. For post-replace synchronization failure, assert coherent new visible state plus the categorized durability error. + +### Cancellation and concurrency + +- Cancel before temporary write, before archive, before replace, and during the shielded commit. +- Assert either complete old or complete new state, never partial state. +- Start concurrent append and replacement operations behind deterministic barriers and assert serialization. +- Cancel a queued writer and prove the lock and later writes recover. +- Verify checkpoint IDs do not skip after failed persistence. + +### Soul flows + +- Pruning failure and cancellation preserve exact JSONL bytes and memory. +- Compaction failure and cancellation preserve exact JSONL bytes and memory. +- Successful pruning and compaction produce expected records with no duplicate rollback path. +- `/clear` writes a coherent empty generation with system prompt. +- Revert uses one semantic replacement. + +Focused context, pruning, compaction, slash, and wire-session tests run before the full Pythinker Code gate. + +## Migration and deletion + +1. Characterize current record order, rotation, revert, clear, pruning, and compaction behavior. +2. Extract private serializer and state reducer used by restore. +3. Add disk-first append, checkpoint, and usage behavior. +4. Add the mutation lock and concurrency tests. +5. Add `replace_history` with atomic same-directory replacement. +6. Migrate pruning. +7. Migrate compaction. +8. Migrate revert and clear. +9. Delete compensating soul rollback blocks and direct clear/rebuild sequences. +10. Remove compatibility methods only if they are not public and repository search confirms no callers. + +The deletion test passes when removing the replacement implementation would force serialization, archive, commit, derived-state, cancellation, and rollback knowledge back into pruning, compaction, revert, and clear callers. + +## Rollback + +Revert this phase if exact JSONL compatibility, cancellation coherence, or rotation recovery fails. Do not add a runtime flag that chooses between transactional and destructive replacement paths. The old generation and archives provide data recovery, while version control provides code rollback. diff --git a/docs/superpowers/specs/2026-07-10-request-assembly-design.md b/docs/superpowers/specs/2026-07-10-request-assembly-design.md new file mode 100644 index 00000000..e013d3b5 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-request-assembly-design.md @@ -0,0 +1,249 @@ +# Observable request assembly design + +**Status:** Approved as phase 3 of the agent core deepening program + +## Problem + +A provider request is currently assembled across runtime prompt rendering, `PythinkerSoul`, AGENTS.md handling, dynamic injection providers, history normalization, tool advertisement, and provider adapters. Dynamic provider failures are logged and omitted, budget decisions are not inspectable, candidate metadata is discarded, and prompt inspection cannot show the same effective request that a provider receives. + +The new module must own assembly decisions without changing the `pythinker_core.step` interface or persisting sensitive prompt content. + +## Goals + +- Produce one provider-ready request from explicit typed inputs. +- Preserve the current provider argument shape and compatibility ordering. +- Record source, authority, requirement, persistence, budget, and outcome for each fragment. +- Fail closed for required guidance and degrade explicitly for optional guidance. +- Preserve a stable static system-prompt prefix. +- Keep manifests in memory and sanitized. +- Expose the latest manifest through `/prompt-manifest`. + +## Non-goals + +- Moving provider authority mapping into the CLI package. +- Persisting full request bodies or manifests. +- Sending prompt contents, raw paths, credentials, or user text to telemetry. +- Making every dynamic provider required. +- Replacing `pythinker_core.step` or message normalization. +- Reordering existing compatible history before characterization tests prove intent. + +## Module and interface + +`pythinker_code.soul.request_assembly` becomes the deep module. + +```python +class FragmentRequirement(StrEnum): + REQUIRED = "required" + BEST_EFFORT = "best_effort" + +class FragmentPersistence(StrEnum): + REQUEST_ONLY = "request_only" + HISTORY = "history" + +class FragmentStatus(StrEnum): + INCLUDED = "included" + NOT_APPLICABLE = "not_applicable" + TRUNCATED = "truncated" + OMITTED_BUDGET = "omitted_budget" + DEGRADED = "degraded" + FAILED = "failed" + +class RequestStatus(StrEnum): + SUCCEEDED = "succeeded" + DEGRADED = "degraded" + FAILED = "failed" + +@dataclass(frozen=True, slots=True) +class RequestFragment: + key: str + content: str + source: str + requirement: FragmentRequirement + persistence: FragmentPersistence + priority: int + truncatable: bool + +@dataclass(frozen=True, slots=True) +class FragmentOutcome: + key: str + source: str + requirement: FragmentRequirement + persistence: FragmentPersistence + status: FragmentStatus + estimated_tokens: int + admitted_tokens: int + reason_code: str | None + +@dataclass(frozen=True, slots=True) +class RequestManifest: + status: RequestStatus + reason_code: str | None + outcomes: tuple[FragmentOutcome, ...] + budget_tokens: int + budgeted_admitted_tokens: int + non_budgeted_estimated_tokens: int + +@dataclass(frozen=True, slots=True) +class RequestAssemblyInput: + system_prompt: str + persisted_history: tuple[Message, ...] + current_task: str + budget_tokens: int + +@dataclass(frozen=True, slots=True) +class AssembledRequest: + system_prompt: str + provider_history: tuple[Message, ...] + history_appends: tuple[Message, ...] + manifest: RequestManifest + +class RequestAssemblyError(RuntimeError): + manifest: RequestManifest + reason_code: str + +class RequestAssembler: + async def assemble(self, request: RequestAssemblyInput) -> AssembledRequest: ... +``` + +`RequestManifest` never contains fragment content, user text, model output, raw paths, or provider credentials. The assembler computes and validates token estimates from fragment content with the shared estimator; source adapters cannot supply or understate them. Per-fragment admitted counts include both budgeted and non-budgeted fragments. Aggregate fields separate budgeted admitted tokens from non-budgeted estimates, so authoritative AGENTS.md is visible without being charged against the optional budget. The signatures may be adapted during implementation planning to established types, but the information contract must not shrink. + +## Source adapters + +Existing `DynamicInjectionProvider` implementations remain source adapters during migration. A compatibility adapter converts each returned `DynamicInjection` into a best-effort history fragment with the provider's current ordering and priority. + +Providers migrate individually to a typed result: + +- `provided`: one or more fragments were produced. +- `not_applicable`: the provider completed successfully and no fragment applies. +- `failed`: the provider could not establish required state or encountered an actionable dependency error. + +A provider returning no data is not treated as failure unless its interface declares that a result was required for the current request. + +## Requiredness matrix + +The approved initial policy is: + +| Source | Requirement | Persistence | Budget behavior | +| --- | --- | --- | --- | +| AGENTS.md preamble | Required | Request-only projection of authoritative project instructions | Non-budgeted | +| Permission state | Required | History, matching current replay semantics | Reserved, never truncated | +| Applicable model defense | Required | History, matching current replay semantics | Reserved, never truncated | +| Task-relevant skill candidates | Best-effort | Request-only | Bounded by the skill catalogue cap | +| Plan, goal, active skills, agent list, orchestration, git, LSP, inline command, and auto-mode reminders | Best-effort | Preserve current persistence initially | Priority budget, truncation only when explicitly safe | + +Model defense may return `not_applicable` when the current model profile does not require it. That is a successful outcome. Exceptions, invalid state, or unavailable required inputs produce failure. + +Every registered provider not named in the table is admitted through the compatibility adapter as best-effort, with its source and outcome recorded. This normative catch-all prevents current or plugin providers from disappearing during migration. + +Security classification cannot be changed by an untrusted provider payload. Requiredness is registered by trusted application composition, not supplied by fragment content. + +## Assembly order + +1. Accept the stable system prompt and a snapshot of persisted history. +2. Collect AGENTS.md through its existing authoritative trust wrapper. +3. Collect required providers and stop on required failure. +4. Ask `SkillCatalog` for task-relevant request-only candidates. +5. Collect optional providers and convert failures to degraded outcomes. +6. Reserve required budget before optional admission. +7. Admit optional fragments by explicit priority and deterministic source order. +8. Truncate only fragments whose trusted source marks them truncatable. +9. Apply persistence policy and existing history-normalization rules. +10. Return provider-ready history and the sanitized manifest. + +Required fragments never compete with optional fragments. If required budget exceeds the available context, assembly raises `RequestBudgetError` before provider invocation. It does not truncate required guidance or silently evict it. + +The static system prompt remains identical across tasks. Task-specific skill candidates and dynamic fragments are placed after the stable prefix in provider-visible history, protecting exact-prefix caching. + +## History mutation and retries + +Assembly itself is pure with respect to persisted `Context`. `history_appends` contains only newly admitted `HISTORY` messages. `provider_history` contains the input persisted history, those appends, and request-only fragments in final provider order. A request-only fragment can never appear in `history_appends`. + +`PythinkerSoul` persists `history_appends` only after successful assembly and before provider invocation. Every persistent fragment has a stable key. Before append, the assembler checks the relevant history window for that key according to the provider's rearm policy. Retrying the same step does not append duplicate reminders. + +If persistence fails, the provider is not called. The soul derives a new manifest by copying the successful assembly outcomes, setting overall status to `FAILED`, and setting reason code `context_persistence_failed`; it does not fabricate a fragment outcome. If the provider call fails after persistence, the committed reminder remains available for replay, matching current context behavior. + +## Error contract + +`RequestAssemblyError` is the base categorized failure. Every instance carries the sanitized failure `manifest` produced from all outcomes observed before failure plus a stable `reason_code`. Its manifest must have status `FAILED`, and `error.reason_code` must equal `error.manifest.reason_code`. `PythinkerSoul` stores that manifest before re-raising, so `/prompt-manifest` reports the failed attempt rather than stale success. Subclasses or reason codes distinguish: + +- Required source unavailable. +- Required source invalid. +- Required content exceeds budget. +- History normalization failure. +- Persistence failure at the caller seam. +- Internal invariant violation. + +Required failures are safe to surface with source identifiers and recovery guidance, never raw content. Optional failure yields `FragmentStatus.DEGRADED`, structured logging, and a visible manifest outcome. + +Broad provider exceptions are converted only at the provider adapter seam. Unexpected programming errors retain a causal chain and are not converted to an empty successful result. + +## Manifest observability + +`PythinkerSoul` retains only the latest `RequestManifest` in memory. Successful assembly supplies it through `AssembledRequest`; failed assembly supplies it through `RequestAssemblyError`. The soul replaces the stored manifest on either path, so diagnostics do not show stale success. + +`/prompt-manifest` renders: + +- Overall `SUCCEEDED`, `DEGRADED`, or `FAILED` status and its safe overall reason code when present. +- Fragment key and trusted source identifier. +- Requirement and persistence class. +- Included, omitted, truncated, degraded, failed, or not-applicable status. +- Estimated and admitted token counts. +- Safe reason code. + +It never renders fragment content, user content, raw file paths, secrets, or stack traces. Before the first assembly, it reports that no request has been assembled in the session. + +The command uses normal slash-command output so Shell, print, ACP, and web consumers receive existing wire-safe text rather than a new protocol event. Public documentation and slash-command snapshots are updated. + +## Telemetry + +Telemetry may record only aggregate counts and stable source identifiers: + +- Required, optional, included, omitted, truncated, degraded, and failed counts. +- Budget limit, budgeted admitted estimates, and non-budgeted estimated tokens as separate values. +- Assembly duration. + +No content, raw paths, user input, tool arguments, model output, or provider credentials are recorded. Telemetry failure remains subject to existing project policy and cannot change required assembly success. + +## Test design + +Tests are written before implementation and must first fail for the intended missing behavior. + +- Compatibility input produces byte-equivalent provider system prompt and history. +- Different tasks retain an identical static system prompt. +- Required source failure prevents persistence and provider invocation. +- Required not-applicable succeeds only for a source that supports that state. +- Required content is never truncated. +- Required budget exhaustion raises a categorized error. +- Optional provider failure creates a degraded outcome and permits provider invocation. +- Non-truncatable optional content is omitted rather than cut. +- Truncatable optional content records the admitted estimate. +- Equal-priority sources use stable explicit order rather than registration accidents. +- Stable keys prevent duplicate persistent reminders on retry. +- Manifest data contains no prompt snippets, user text, credentials, or raw paths. +- Adapter-supplied estimates cannot bypass assembler-computed budgeting. +- Budgeted and non-budgeted aggregate counts remain distinct. +- Caller persistence failure produces overall `FAILED` status without a fabricated fragment. +- `/prompt-manifest` handles no-data, success, degraded, and failure states. +- OpenAI and Anthropic provider snapshot tests preserve authority mapping with no real credentials. +- Disabled dynamic injection mode cannot bypass required security guidance. + +Focused verification covers dynamic budgets, provider hooks, permissions, model defense, turn balancing, slash commands, provider snapshots, and wire behavior before the full Pythinker Code gate. + +## Migration and deletion + +1. Characterize current handoff bytes, ordering, persistence, and provider failures. +2. Add fragment, outcome, manifest, and categorized error types. +3. Return explicit outcomes from the existing budget selector while preserving its compatibility projection. +4. Adapt every current provider as best-effort without behavior change. +5. Extract request assembly and prove byte equivalence. +6. Register permissions and applicable model defense as required. +7. Add `SkillCatalog` candidates as request-only fragments. +8. Add sanitized telemetry and `/prompt-manifest`. +9. Migrate provider lifecycle and rearm metadata into typed fragments. +10. Delete `_collect_injections` and direct AGENTS.md/history orchestration from `_step`. + +The deletion test passes when removing `RequestAssembler` would spread requiredness, ordering, persistence, budgeting, sanitization, retry identity, and degradation policy back across `PythinkerSoul` and provider implementations. + +## Rollback + +Revert the phase if provider handoff compatibility, required guidance, prefix stability, or security redaction tests fail. Do not preserve a hidden fail-open mode. Compatibility adapters project old providers into the one assembly implementation and do not create a second path. diff --git a/docs/superpowers/specs/2026-07-10-skill-catalogue-design.md b/docs/superpowers/specs/2026-07-10-skill-catalogue-design.md new file mode 100644 index 00000000..924aa401 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-skill-catalogue-design.md @@ -0,0 +1,199 @@ +# Bounded skill catalogue design + +**Status:** Approved as phase 2 of the agent core deepening program + +## Problem + +Skill discovery currently resolves every configured root, parses every winning skill, builds an exhaustive mapping, and renders every name, absolute path, and description into the static system prompt. The exhaustive mapping provides valuable behavior for exact `ReadSkill` calls, aliases, local specialization, slash execution, and subagents. The leak is the full catalogue projection into model context and the duplicated construction path used by live runtime and prompt inspection. + +The new module must reduce prompt context without making an omitted skill unreachable. + +## Goals + +- Keep deterministic project, user, extra, and built-in precedence. +- Preserve exhaustive exact resolution and local-specialization behavior. +- Search the complete discovered metadata set while returning bounded results. +- Keep task-dependent candidates out of the static system-prompt prefix. +- Share one implementation between runtime creation and prompt inspection. +- Preserve current callers through a temporary exhaustive mapping adapter. +- Meet the approved recall, size, and warm-retrieval targets without a new dependency. + +## Non-goals + +- Persisting a cross-session skill index. +- Adding embedding or vector-search dependencies. +- Loading full skill bodies before `ReadSkill` is invoked. +- Removing scope roots, aliases, MCP fallback, or resource manifests. +- Guaranteeing arbitrary semantic recall beyond the approved fixture. + +## Module and interface + +`pythinker_code.skill.catalog` becomes the deep module. Its caller-facing interface is intentionally small: + +```python +@dataclass(frozen=True, slots=True) +class SkillMatch: + skill: Skill + score: int + reasons: tuple[str, ...] + +class SkillProjectionStatus(StrEnum): + READY = "ready" + DEGRADED = "degraded" + FAILED = "failed" + +@dataclass(frozen=True, slots=True) +class SkillPromptView: + matches: tuple[SkillMatch, ...] + total_count: int + omitted_count: int + overflowed_priority_count: int + rendered_characters: int + +@dataclass(frozen=True, slots=True) +class SkillProjectionOutcome: + status: SkillProjectionStatus + view: SkillPromptView | None + reason_code: str | None + +class SkillCatalog: + @classmethod + async def discover( + cls, + roots: Sequence[ScopedSkillsRoot], + ) -> SkillCatalog: ... + + def resolve(self, name: str) -> Skill | None: ... + def search(self, query: str, *, limit: int) -> tuple[SkillMatch, ...]: ... + def prompt_view(self, query: str, *, max_characters: int) -> SkillProjectionOutcome: ... + def exhaustive_mapping(self) -> Mapping[str, Skill]: ... +``` + +The signatures are design intent; implementation planning may adjust names to match established repository conventions without widening the interface. + +`SkillCatalog` owns: + +- Winning-entry selection and scope precedence. +- Normalized exact-name and alias lookup. +- Search normalization and deterministic ranking. +- Prompt-view character accounting. +- Source diagnostics for malformed or unavailable entries. +- The compatibility mapping consumed through `Runtime.skills` during migration. + +`format_skills_for_prompt` becomes a rendering adapter over `SkillPromptView`. It does not choose entries. + +## Discovery and search + +Phase 1 keeps eager metadata discovery so existing exact lookup remains behaviorally identical. Prompt reduction ships before filesystem laziness. A later optimization may reduce startup reads only after it demonstrates the same frontmatter-defined names, precedence, and malformed-file diagnostics. + +Search is deterministic and standard-library-only: + +1. Normalize the query and candidate text with the existing skill-name normalization rules plus case folding. +2. Promote an exact normalized name or alias match above every fuzzy match. +3. Rank name phrase matches above name-token overlap. +4. Rank name-token overlap above description-token overlap. +5. Use scope precedence only after relevance so a relevant lower-scope skill is not hidden by an unrelated higher-scope skill. +6. Break remaining ties by normalized name and canonical path. +7. Never use randomness, model calls, wall-clock state, or filesystem enumeration order. + +The approved recall fixture defines task text, expected winning skill names, explicit ambiguous cases, aliases, and scope-shadowed entries. Recall parity means every expected skill appears within the bounded returned set for that fixture. + +## Prompt projection + +The static system prompt retains only stable skill invocation policy and a statement that task-relevant candidates arrive per request. It no longer contains the exhaustive catalogue. + +For each provider request, `RequestAssembler` asks `SkillCatalog.prompt_view` for the current task. The rendered fragment: + +- Contains name, scope, and concise description. +- Omits absolute filesystem paths. +- Includes the total and omitted counts. +- Tells the model to call `ReadSkill` before applying a candidate. +- Is request-only and never appended to persisted conversation history. +- Is capped at 8,000 characters, including headings and omission text. + +Active skills and explicit skill names in the current message are priority entries, not required request fragments. The hard cap always wins. Rendering admits explicit names in message order, then active skills from most recently activated to oldest, then implicit matches. It first removes descriptions, then stops adding names when the next complete entry would exceed 8,000 characters. The view records `overflowed_priority_count`, returns a `DEGRADED` outcome with a safe reason code, and renders only the count that still fits. It never truncates a skill name into an ambiguous identifier and never exceeds the cap. Exact loading remains available through `ReadSkill`. The representative recall fixture is sized so this pathological overflow does not redefine its 100 percent recall requirement. + +The static system prompt must remain byte-identical across different tasks under the same runtime configuration, protecting exact-prefix provider caching. + +## Exhaustive fallback + +Bounded output does not mean bounded lookup. `search` evaluates the complete winning metadata set. `resolve` probes the complete normalized mapping. `ReadSkill` continues to perform alias and MCP fallback after filesystem resolution. + +When an exact `ReadSkill` name is missing, the tool returns a small ranked suggestion set rather than enumerating every skill. It distinguishes: + +- `not_found`: no winning skill or source diagnostic matches the requested name. +- `unavailable`: a matching discovered source exists but cannot be parsed or read. +- `mcp_fallback`: no filesystem skill exists and a connected MCP bridge resolves the name. + +These outcomes remain concise model-facing text; internal details stay in structured diagnostics. + +## Runtime compatibility + +`Runtime` gains a `skill_catalog` field. During migration, `Runtime.skills` remains the exhaustive mapping returned by `SkillCatalog.exhaustive_mapping`. Root and subagent runtime copies share the same catalogue instance. + +Live runtime and read-only system-prompt inspection both call the same catalogue discovery and stable static projection code. Prompt inspection reports catalogue counts and the configured cap, but cannot invent task-specific candidates without a task. + +Phase 2 remains independently releasable before `RequestAssembler` exists. A narrow `_with_skill_candidates` compatibility adapter in `PythinkerSoul` asks the catalogue for the current-task view and adds one request-only message to `effective_history` after persisted-history construction. It does not select, rank, budget, or persist entries. Phase 3 moves that projection into `RequestAssembler` and deletes the adapter; tests require provider-visible byte parity across the handoff. + +The compatibility mapping is removed only after repository search proves that exact resolution, slash execution, local specialization, compaction restore, and all tool callers use the catalogue interface. + +## Failure semantics + +Unreadable roots and malformed discovered skills remain isolated diagnostics, preserving startup compatibility. Diagnostics include source kind, safe path, category, and actionable reason without file contents. + +An explicitly requested unavailable skill does not collapse to not-found. During independently releasable Phase 2, the compatibility adapter stores the latest `SkillProjectionOutcome` in memory on the soul and logs its safe status and reason code; it does not claim a request manifest exists. A failed or degraded projection adds no unbounded prompt fallback. In Phase 3, `RequestAssembler` maps both `SkillProjectionStatus.DEGRADED` and `SkillProjectionStatus.FAILED` to a best-effort `FragmentStatus.DEGRADED` outcome and an overall degraded request. A skill-projection failure never becomes a required-fragment failure or blocks the provider call. Exact resolution remains available if the catalogue itself was successfully constructed. + +Catalogue construction failure is fatal only when the winning required project or built-in skill source cannot satisfy an existing invariant. This phase does not promote every optional skill parse warning to startup failure. + +## Performance contract + +The representative benchmark uses 1,000 in-memory entries after one warm-up search. It runs enough iterations to report a median and avoids filesystem work inside the measured region. + +Acceptance gates: + +- Expected fixture recall: 100 percent within the configured result limit. +- Rendered task candidate view: at most 8,000 characters. +- Warm in-memory retrieval median: below 100 ms. +- Stable deterministic output across repeated runs and insertion orders. +- No full skill-body reads during search or prompt rendering. + +A deterministic operation-count assertion accompanies the wall-clock gate so a noisy machine does not become the only signal of an algorithmic regression. + +## Test design + +Tests are written before implementation and must first fail for the intended missing behavior. + +- Exact name, alias, case, and local specialization preserve current results. +- Project overrides user, extra, and built-in definitions of the same normalized name. +- Search ordering is deterministic under reversed insertion order. +- Explicit and active skills are retained before implicit matches. +- Malformed matching skills return unavailable. +- Missing exact names return bounded suggestions. +- Prompt rendering includes omission counts and no absolute paths. +- Priority-entry overflow produces a structured degraded outcome. +- Phase 2 stores projection failure independently; Phase 3 maps the same outcome into the request manifest. +- Prompt rendering never exceeds 8,000 characters. +- Static system prompt remains identical for different tasks. +- Root and subagent runtimes share one catalogue. +- Live and inspection construction share the same implementation. +- The 1,000-entry recall and warm-performance fixture passes. + +Focused verification covers skill discovery, skill prompt rendering, `ReadSkill`, runtime loading, default-agent snapshots, and wire skill behavior before the full Pythinker Code gate. + +## Migration and deletion + +1. Add characterization fixtures for current precedence, exact resolution, and prompt rendering. +2. Introduce `SkillCatalog` with exhaustive behavior only. +3. Route runtime and prompt inspection through the catalogue while preserving current rendered output. +4. Add deterministic search and bounded task projection. +5. Move task candidates into request assembly and replace the static exhaustive list with stable policy. +6. Change missing `ReadSkill` output to bounded suggestions. +7. Migrate internal mapping callers. +8. Delete direct discovery/index/render orchestration from runtime callers. +9. Remove `Runtime.skills` only after its documented compatibility window and repository-wide call-site check. + +The deletion test passes when removing `SkillCatalog` would force precedence, indexing, ranking, diagnostics, prompt accounting, and exact fallback logic back into runtime, prompt inspection, and `ReadSkill` callers. + +## Rollback + +The phase is reverted as one unit if recall, prompt-size, provider-prefix, or compatibility tests fail. No hidden environment switch preserves a parallel production path. The exhaustive mapping adapter exists for caller compatibility, not as an alternate implementation. diff --git a/docs/superpowers/specs/2026-07-10-toolset-characterization-design.md b/docs/superpowers/specs/2026-07-10-toolset-characterization-design.md new file mode 100644 index 00000000..e0746783 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-toolset-characterization-design.md @@ -0,0 +1,209 @@ +# Toolset characterization and conditional deepening design + +**Status:** Approved as phase 6 of the agent core deepening program + +## Problem + +`PythinkerToolset` is large because it hides registry, visibility, execution, concurrency, deduplication, approvals, hooks, telemetry, and MCP lifecycle behavior behind one interface. Size alone does not prove poor depth. Splitting before characterizing latency, contention, lifecycle failure, and change locality could create shallow private modules, duplicate state, or callback cycles without improving callers. + +This phase first produces reproducible evidence. Private extraction is permitted only when a documented threshold is crossed and the resulting implementation deletes equivalent state from `PythinkerToolset`. + +## Goals + +- Measure non-tool overhead in `handle` without adding a hosted telemetry dependency. +- Characterize advertisement cost, read/write gate behavior, cancellation, and MCP lifecycle. +- Reproduce failures with deterministic barriers rather than sleeps. +- Define and apply explicit go/no-go thresholds. +- Preserve `PythinkerToolset` as the external interface. +- Treat a no-go decision as a successful completed phase. + +## Non-goals + +- Splitting by line count. +- Changing approval, hook, telemetry, deduplication, visibility, or MCP semantics during characterization. +- Publishing new public interfaces. +- Adding production sampling, remote telemetry, or runtime dependencies solely for benchmarks. +- Optimizing actual tool execution time. + +## Characterization harness + +The harness uses existing test and benchmark conventions, standard-library monotonic timing, deterministic fake tools, and controlled host/MCP adapters. Measured regions exclude fixture setup and report both raw samples and summary statistics. + +Every benchmark records: + +- Python and platform information. +- Fixture size and concurrency shape. +- Warm-up count and measured iteration count. +- Median and p95 latency. +- Throughput where meaningful. +- Allocation or retained-object proxy available through existing tooling. +- Cancellation completion and leaked-task count. + +Benchmarks are directional engineering gates, not product telemetry. They do not claim universal hardware performance. Timing thresholds use five isolated measured runs after warm-up. A timing threshold is considered crossed only when at least four of five runs cross it and the median result also crosses it. A single outlier produces an inconclusive decision and one complete rerun, not an extraction. + +## Execution pipeline measurements + +`handle` is measured by phase: + +1. Tool lookup and suggestion. +2. JSON parse and canonicalization. +3. Same-step and cross-step deduplication. +4. Permission resolution and approval preparation. +5. Pre-hook execution. +6. Read/write gate wait. +7. Actual tool call. +8. Post-hook and reminder processing. +9. Telemetry and wire completion. + +Scenarios include: + +- 1, 10, and 100 parallel-safe calls. +- 1, 10, and 100 exclusive calls. +- Mixed reader/writer ordering. +- Duplicate payloads of 1 KiB, 100 KiB, and 1 MiB. +- Fast no-op tools so framework overhead is measurable. +- Tool failure, cancellation, and pre-hook block. + +The harness reports non-tool overhead separately from actual tool-call duration. + +## Advertisement measurements + +The `tools` projection is measured with 50, 500, and 5,000 registered tools under: + +- Visibility policy enabled and disabled. +- Hidden and unhidden entries. +- Built-in, plugin, and MCP mixtures. +- Repeated reads without registry changes. +- Registry rebuild after MCP publication. + +The output includes p50/p95 projection latency and a deterministic registry hash so caching or extraction cannot change advertised order or collision behavior. + +## MCP lifecycle matrix + +Scenarios use 1, 10, and 50 configured servers: + +- Fast, slow, and hung connect. +- Optional method-not-found response. +- Transient inventory failure. +- Duplicate tool names. +- List-change storms. +- Refresh racing disconnect. +- Disconnect during inventory. +- Cancellation during background load. +- Cleanup racing load. +- Close timeout. + +The MCP lifecycle interval begins when `connect_to_mcp_servers` starts its foreground connection work or creates `_mcp_loading_task`, and ends when `wait_for_mcp_tools` completes after final registry publication. The startup-to-ready denominator begins at `Runtime.create` entry and ends at that same settled-inventory point in the same fixture. Measurements include time to first usable inventory, time to settled inventory, task count, leaked task/process count, deterministic published registry hash, cleanup duration, and truthful lifecycle status. + +## Deterministic fault tests + +Tests coordinate with events and barriers, never arbitrary sleeps, for: + +- Pre-hook block and exception. +- Tool cancellation before and after gate admission. +- Post-hook failure. +- Telemetry failure according to existing policy. +- Reader cancellation while queued. +- Writer cancellation while readers drain. +- Refresh racing disconnect. +- Cleanup racing background load. +- MCP publication failure after partial inventory. + +Assertions cover explicit failure status, PreToolUse block preservation, no orphan tasks, no leaked sessions, no half-published registry, deterministic cleanup, and later-call recovery. + +## Go/no-go thresholds + +### Private execution pipeline + +Extract a private `_ToolExecutionPipeline` only if either condition is true: + +- Non-tool framework overhead exceeds 10 percent of p95 latency for short in-process tools. +- At least three independent recent changes repeatedly modify the same `handle` lifecycle region and the proposed module removes that shared state from `PythinkerToolset`. + +Fault characterization must be green before extraction. `PythinkerToolset.handle` remains the compatibility facade. + +### Private MCP lifecycle + +Extract a private `_McpLifecycle` if any condition is true: + +- The defined MCP lifecycle interval exceeds 20 percent of the defined startup-to-ready denominator with 10 configured servers under the repeatability rule. +- Total `cleanup()` time, measured from method entry to return after stop signals are issued, exceeds 6 seconds. This derives from the existing concurrent 5-second per-client close timeout plus 1 second of scheduling allowance and applies to the 1, 10, and 50-server fixtures. +- A task, process, session, or publication leak is reproduced. +- At least three modules require direct MCP lifecycle state. + +The private interface may cover configure, defer, start, wait, status, refresh, reconnect, disconnect, close, and inventory publication. It must not own general tool visibility, approval, or execution. + +### Private registry + +Extract a private `_ToolRegistry` if either condition is true: + +- Advertisement exceeds 5 ms p95 at the 500-tool fixture under the repeatability rule. The 5,000-tool fixture is a stress result and cannot trigger extraction by itself. +- Collision, rebuild, or visibility defects recur and one owner would delete duplicated state. + +The private interface preserves add, find, hide, unhide, advertised projection, and deterministic MCP rebuild semantics. + +### Read/write gate + +Keep `_ReadWriteGate` private and colocated unless gate wait exceeds 25 percent of end-to-end p95 in a realistic mixed workload or a second real consumer appears. Optimization alone is not a module seam. + +### Mandatory no-go + +Do not extract when: + +- No threshold is crossed. +- The change only reduces file length. +- Equivalent state remains in both old and new modules. +- The extraction creates a public interface. +- Callbacks introduce a cycle between registry, execution, and MCP lifecycle. +- Characterization or cancellation tests are not green. + +## Extraction rules + +When a threshold is crossed: + +- Write a failing test or benchmark assertion that demonstrates the measured problem. +- Move one coherent state machine at a time. +- Keep `PythinkerToolset` as the caller-facing facade. +- Pass dependencies into the private module; do not create global registries or hidden singletons. +- Preserve exact tool ordering, collision, approval, hook, event, and error behavior. +- Delete the moved state and business logic from `PythinkerToolset` in the same change. +- Re-run characterization and report before/after results with the same fixture. + +If the measured result does not improve the target or weakens locality, revert the extraction. + +## Test design + +Tests and benchmarks are added before any extraction. + +- Current reader overlap and writer exclusion remain characterization baselines. +- Cancellation at every queue state releases permits and allows later calls. +- PreToolUse block is never discarded. +- Ordinary tool exceptions retain `ToolRuntimeError` behavior. +- `BaseException` cancellation propagates after cleanup. +- Tool advertisement remains deterministic under insertion and publication changes. +- MCP duplicate-server and duplicate-tool behavior remains compatible. +- Background loading, refresh, reconnect, disconnect, and cleanup leave no tasks or sessions. +- Benchmark output contains fixture metadata and all required phase timings. +- Threshold evaluation emits a deterministic go/no-go decision from benchmark results. + +Focused Toolset and concurrency tests run with the characterization suite. Any shipped extraction then runs the full Pythinker Code static and test gates. + +## Deliverables + +The phase always produces: + +1. Characterization fixtures and deterministic fault tests. +2. Benchmark runner and documented invocation. +3. Machine-readable result schema containing fixture metadata and phase timings. +4. Human-readable decision record listing crossed, uncrossed, and inconclusive thresholds, all five run results, the median, and whether the repeatability rule passed. +5. Either a no-go result with no production refactor, or one evidence-supported private extraction. + +A no-go result closes the phase. It does not justify searching for a different split until new evidence appears. + +## Deletion test + +For an extracted private module, deleting it must force its state machine and invariants back into `PythinkerToolset`; a module that only forwards method calls fails the deletion test. The extraction must reduce state ownership and change locality, not merely move lines. + +## Rollback + +Characterization-only changes are removed if they are flaky, mutate production behavior, or cannot reproduce deterministically. An extraction is reverted if compatibility, failure truthfulness, cancellation, cleanup, or measured target results regress. No hidden flag selects between old and new implementations. diff --git a/scripts/benchmark_toolset.py b/scripts/benchmark_toolset.py new file mode 100644 index 00000000..71621301 --- /dev/null +++ b/scripts/benchmark_toolset.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +from pythinker_code.benchmark.toolset_characterization import run_characterization + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be positive") + return parsed + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=("Run the directional, local-only Pythinker Toolset characterization harness.") + ) + parser.add_argument( + "--scenario", + choices=("all", "execution", "dedupe", "advertisement", "mcp"), + default="all", + help="scenario family to measure (default: all)", + ) + parser.add_argument( + "--runs", + type=_positive_int, + default=5, + help="isolated measured runs per fixture after warm-up (default: 5)", + ) + parser.add_argument( + "--output", + type=Path, + help="write the machine-readable JSON report to this path", + ) + parser.add_argument( + "--smoke", + action="store_true", + help="run only the smallest fixture in each selected scenario family", + ) + return parser + + +async def _run(args: argparse.Namespace) -> str: + report = await run_characterization( + scenario=args.scenario, + runs=args.runs, + smoke=args.smoke, + ) + return report.model_dump_json(indent=2) + "\n" + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + payload = asyncio.run(_run(args)) + if args.output is not None: + args.output.write_text(payload, encoding="utf-8") + else: + sys.stdout.write(payload) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pythinker_code/agents/default/system.md b/src/pythinker_code/agents/default/system.md index 2700c2f2..b4c7a3c0 100644 --- a/src/pythinker_code/agents/default/system.md +++ b/src/pythinker_code/agents/default/system.md @@ -255,7 +255,7 @@ Precedence per §2. `README`/`README.md` files are optional supplementary contex ## 12. Skills -Skills are reusable, self-contained capability directories, each with a `SKILL.md` of instructions, examples, scripts, and reference material — specialized domain knowledge, workflow patterns, pre-configured tool chains, and templates. They are grouped by scope (`Project`, `User`, `Extra`, `Built-in`); when scopes define the same name, the more specific wins: **Project › User › Extra › Built-in.** +Skills are reusable, self-contained capability directories, each with a `SKILL.md` of instructions, examples, scripts, and reference material — specialized domain knowledge, workflow patterns, pre-configured tool chains, and templates. When scopes define the same name, the more specific wins: **Project › User › Extra › Built-in.** ${PYTHINKER_SKILLS} diff --git a/src/pythinker_code/agentspec.py b/src/pythinker_code/agentspec.py index 861fecc7..35a65582 100644 --- a/src/pythinker_code/agentspec.py +++ b/src/pythinker_code/agentspec.py @@ -1,11 +1,13 @@ from __future__ import annotations +import hashlib +import re from dataclasses import dataclass from pathlib import Path from typing import Any, Literal, NamedTuple, cast import yaml -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ValidationError from pythinker_code.exception import AgentSpecError @@ -89,6 +91,14 @@ class ResolvedAgentSpec: subagents: dict[str, SubagentSpec] +@dataclass(frozen=True, slots=True, kw_only=True) +class AgentSpecSourceValidation: + """Unknown fields observed in one canonical YAML source before projection.""" + + source_path: Path + field_paths: tuple[str, ...] + + def load_agent_spec(agent_file: Path) -> ResolvedAgentSpec: """ Load agent specification from file. @@ -97,7 +107,22 @@ def load_agent_spec(agent_file: Path) -> ResolvedAgentSpec: FileNotFoundError: If the agent spec file is not found. AgentSpecError: If the agent spec is not valid. """ - agent_spec = _load_agent_spec(agent_file) + agent_spec, _ = load_agent_spec_validated(agent_file) + return agent_spec + + +def load_agent_spec_validated( + agent_file: Path, + *, + forbid_unknown_fields: bool = False, +) -> tuple[ResolvedAgentSpec, tuple[AgentSpecSourceValidation, ...]]: + """Load a spec and report raw unknown fields from every inherited source.""" + validations: dict[Path, AgentSpecSourceValidation] = {} + agent_spec = _load_agent_spec( + agent_file, + _validations=validations, + _forbid_unknown_fields=forbid_unknown_fields, + ) assert agent_spec.extend is None, "agent extension should be recursively resolved" if isinstance(agent_spec.name, Inherit): raise AgentSpecError("Agent name is required") @@ -111,7 +136,7 @@ def load_agent_spec(agent_file: Path) -> ResolvedAgentSpec: agent_spec.exclude_tools = [] if isinstance(agent_spec.subagents, Inherit): agent_spec.subagents = {} - return ResolvedAgentSpec( + resolved = ResolvedAgentSpec( name=agent_spec.name, system_prompt_path=agent_spec.system_prompt_path, system_prompt_args=agent_spec.system_prompt_args, @@ -127,9 +152,36 @@ def load_agent_spec(agent_file: Path) -> ResolvedAgentSpec: exclude_tools=agent_spec.exclude_tools or [], subagents=agent_spec.subagents or {}, ) + return resolved, tuple(validations.values()) + + +def _resolve_within_agent_roots(agent_file: Path, declared: str | Path) -> Path: + """Join *declared* onto *agent_file*'s directory, rejecting escaping paths. + + Trusted local agent specs reference sibling files with relative paths. A + ``..`` traversal or symlink whose canonical target lands outside both the + originating spec's directory and the built-in agents directory is rejected + fail-closed as defense-in-depth, since the joined path is otherwise opened + or recursively loaded directly. The stored value keeps its ``.absolute()`` + form, so permitted paths are unchanged. + """ + parent = agent_file.parent + absolute = (parent / declared).absolute() + allowed_roots = (parent.resolve(), get_agents_dir().resolve()) + if not any(absolute.resolve().is_relative_to(root) for root in allowed_roots): + raise AgentSpecError( + f"Agent spec reference {declared!r} resolves outside the permitted agent directories" + ) + return absolute -def _load_agent_spec(agent_file: Path, _visited: set[Path] | None = None) -> AgentSpec: +def _load_agent_spec( + agent_file: Path, + _visited: set[Path] | None = None, + *, + _validations: dict[Path, AgentSpecSourceValidation] | None = None, + _forbid_unknown_fields: bool = False, +) -> AgentSpec: resolved = agent_file.resolve() if _visited is None: _visited = set() @@ -149,24 +201,46 @@ def _load_agent_spec(agent_file: Path, _visited: set[Path] | None = None) -> Age raise AgentSpecError(f"Agent spec file must contain a mapping: {agent_file}") data = cast("dict[str, Any]", data) + unknown_fields, has_invalid_field_key = _unknown_agent_spec_fields(data) + if unknown_fields: + if has_invalid_field_key: + fields = ", ".join(unknown_fields) + raise AgentSpecError(f"Invalid agent field key: {fields}") + if _forbid_unknown_fields: + fields = ", ".join(unknown_fields) + raise AgentSpecError(f"Unknown fields in required agent source: {fields}") + if _validations is not None and resolved not in _validations: + _validations[resolved] = AgentSpecSourceValidation( + source_path=resolved, + field_paths=unknown_fields, + ) + version = str(data.get("version", DEFAULT_AGENT_SPEC_VERSION)) if version not in SUPPORTED_AGENT_SPEC_VERSIONS: raise AgentSpecError(f"Unsupported agent spec version: {version}") - agent_spec = AgentSpec(**data.get("agent", {})) + try: + agent_spec = AgentSpec(**data.get("agent", {})) + except (TypeError, ValidationError) as exc: + raise AgentSpecError("Agent spec contains an invalid known field") from exc if isinstance(agent_spec.system_prompt_path, Path): - agent_spec.system_prompt_path = ( - agent_file.parent / agent_spec.system_prompt_path - ).absolute() + agent_spec.system_prompt_path = _resolve_within_agent_roots( + agent_file, agent_spec.system_prompt_path + ) if isinstance(agent_spec.subagents, dict): for v in agent_spec.subagents.values(): - v.path = (agent_file.parent / v.path).absolute() + v.path = _resolve_within_agent_roots(agent_file, v.path) if agent_spec.extend: if agent_spec.extend == "default": base_agent_file = DEFAULT_AGENT_FILE else: - base_agent_file = (agent_file.parent / agent_spec.extend).absolute() - base_agent_spec = _load_agent_spec(base_agent_file, _visited) + base_agent_file = _resolve_within_agent_roots(agent_file, agent_spec.extend) + base_agent_spec = _load_agent_spec( + base_agent_file, + _visited, + _validations=_validations, + _forbid_unknown_fields=_forbid_unknown_fields, + ) if not isinstance(agent_spec.name, Inherit): base_agent_spec.name = agent_spec.name if not isinstance(agent_spec.system_prompt_path, Inherit): @@ -207,3 +281,89 @@ def _load_agent_spec(agent_file: Path, _visited: set[Path] | None = None) -> Age base_agent_spec.subagents = agent_spec.subagents agent_spec = base_agent_spec return agent_spec + + +def _unknown_agent_spec_fields(data: dict[str, Any]) -> tuple[tuple[str, ...], bool]: + unknown: list[str] = [] + has_invalid_key = False + for key in cast("dict[object, object]", data): + if isinstance(key, str) and key in {"version", "agent"}: + continue + rendered = render_agent_field_segment(key) + unknown.append(rendered.text) + has_invalid_key = has_invalid_key or rendered.structurally_invalid + raw_agent = data.get("agent") + if not isinstance(raw_agent, dict): + return tuple(sorted(unknown)), has_invalid_key + agent = cast("dict[str, Any]", raw_agent) + known_agent_fields = set(AgentSpec.model_fields) + for key in cast("dict[object, object]", agent): + if isinstance(key, str) and key in known_agent_fields: + continue + rendered = render_agent_field_segment(key) + unknown.append(f"agent.{rendered.text}") + has_invalid_key = has_invalid_key or rendered.structurally_invalid + raw_subagents = agent.get("subagents") + if isinstance(raw_subagents, dict): + known_subagent_fields = set(SubagentSpec.model_fields) + for name, raw_subagent in cast("dict[object, object]", raw_subagents).items(): + rendered_name = render_agent_field_segment(name) + has_invalid_key = has_invalid_key or rendered_name.structurally_invalid + if rendered_name.structurally_invalid: + unknown.append(f"agent.subagents.{rendered_name.text}") + if not isinstance(raw_subagent, dict): + continue + for key in cast("dict[object, object]", raw_subagent): + if isinstance(key, str) and key in known_subagent_fields: + continue + rendered = render_agent_field_segment(key) + unknown.append(f"agent.subagents.{rendered_name.text}.{rendered.text}") + has_invalid_key = has_invalid_key or rendered.structurally_invalid + return tuple(sorted(unknown)), has_invalid_key + + +_FIELD_IDENTIFIER_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*") +_SENSITIVE_FIELD_HINTS = ( + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "access_key", + "private_key", + "credential", + "auth", +) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AgentFieldSegment: + text: str + redacted_for_safety: bool + structurally_invalid: bool + + +def render_agent_field_segment(value: object) -> AgentFieldSegment: + """Render a stable field segment without conflating redaction and validity.""" + if isinstance(value, str): + lowered = value.casefold() + if _FIELD_IDENTIFIER_RE.fullmatch(value) and not any( + hint in lowered for hint in _SENSITIVE_FIELD_HINTS + ): + return AgentFieldSegment( + text=value, + redacted_for_safety=False, + structurally_invalid=False, + ) + digest_input = f"str:{value}" + structurally_invalid = _FIELD_IDENTIFIER_RE.fullmatch(value) is None + else: + digest_input = f"{type(value).__qualname__}:{value!r}" + structurally_invalid = True + digest = hashlib.sha256(digest_input.encode(encoding="utf-8")).hexdigest()[:12] + return AgentFieldSegment( + text=f"field[{digest}]", + redacted_for_safety=True, + structurally_invalid=structurally_invalid, + ) diff --git a/src/pythinker_code/benchmark/toolset_characterization.py b/src/pythinker_code/benchmark/toolset_characterization.py new file mode 100644 index 00000000..653a6202 --- /dev/null +++ b/src/pythinker_code/benchmark/toolset_characterization.py @@ -0,0 +1,1118 @@ +from __future__ import annotations + +import asyncio +import hashlib +import math +import platform +import statistics +import sys +import tempfile +import time +import tracemalloc +from collections.abc import Awaitable, Callable, Iterable, Sequence +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from typing import TYPE_CHECKING, ClassVar, Literal, cast + +import mcp +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from pythinker_core.tooling import CallableTool, HandleResult, ToolOk, ToolReturnValue +from pythinker_core.utils.typing import JsonType + +from pythinker_code.soul.toolset import MCPServerInfo, MCPTool, PythinkerToolset +from pythinker_code.wire.types import ToolCall, ToolResult + +if TYPE_CHECKING: + from fastmcp import Client as FastMcpClient + from fastmcp.client.transports.config import MCPConfigTransport + + from pythinker_code.auth.oauth import OAuthManager + from pythinker_code.config import Config + from pythinker_code.session import Session + from pythinker_code.soul.agent import Runtime + from pythinker_code.soul.toolset import ToolType + +type ScenarioKind = Literal[ + "execution_safe", + "execution_exclusive", + "execution_mixed", + "dedupe", + "advertisement", + "mcp", +] + +_UNMEASURED_EXECUTION_PHASES = ( + "lookup_suggestion", + "json_parse_canonicalize", + "deduplication", + "permission_approval", + "pre_hook", + "post_hook_reminder", + "telemetry_wire", +) +_REGISTRY_PROJECTIONS_PER_RUN = 20 + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(frozen=True) + + +class EnvironmentSnapshot(_FrozenModel): + python_version: str + python_implementation: str + platform: str + + @classmethod + def current(cls) -> EnvironmentSnapshot: + return cls( + python_version=sys.version.split()[0], + python_implementation=platform.python_implementation(), + platform=platform.platform(), + ) + + +class FixtureShape(_FrozenModel): + kind: ScenarioKind + size: int = Field(gt=0) + concurrency: int = Field(gt=0) + payload_bytes: int | None = Field(default=None, gt=0) + composition: str = "" + + +class PhaseSamples(_FrozenModel): + measurement_status: Literal["measured", "unmeasured"] = "measured" + reason: str | None = None + samples_ns: tuple[int, ...] + median_ns: int | None + p95_ns: int | None + throughput_per_second: float | None + within_run_samples_ns: tuple[tuple[int, ...], ...] = () + + @classmethod + def from_samples( + cls, + samples_ns: Sequence[int], + *, + operations_per_sample: int = 1, + within_run_samples_ns: Sequence[Sequence[int]] = (), + ) -> PhaseSamples: + if not samples_ns: + raise ValueError("phase samples must not be empty") + if operations_per_sample < 1: + raise ValueError("operations_per_sample must be positive") + normalized = tuple(int(sample) for sample in samples_ns) + if any(sample < 0 for sample in normalized): + raise ValueError("phase samples must be non-negative") + ordered = sorted(normalized) + normalized_within = tuple( + tuple(int(sample) for sample in run) for run in within_run_samples_ns + ) + if normalized_within and len(normalized_within) != len(normalized): + raise ValueError("within-run sample groups must match measured runs") + if any(not run or any(sample < 0 for sample in run) for run in normalized_within): + raise ValueError("within-run sample groups must be non-empty and non-negative") + median_ns = int(statistics.median(ordered)) + p95_index = max(0, math.ceil(len(ordered) * 0.95) - 1) + throughput = operations_per_sample * 1_000_000_000 / median_ns if median_ns else None + return cls( + samples_ns=normalized, + median_ns=median_ns, + p95_ns=ordered[p95_index], + throughput_per_second=throughput, + within_run_samples_ns=normalized_within, + ) + + @classmethod + def unmeasured(cls, reason: str) -> PhaseSamples: + return cls( + measurement_status="unmeasured", + reason=reason, + samples_ns=(), + median_ns=None, + p95_ns=None, + throughput_per_second=None, + within_run_samples_ns=(), + ) + + +class CancellationResult(_FrozenModel): + completed: bool + completion_ns: int | None = Field(default=None, ge=0) + queued_reader_completed: bool = True + queued_writer_completed: bool = True + recovery_completed: bool = True + + +class LeakSnapshot(_FrozenModel): + tasks: int = Field(ge=0) + processes: int = Field(ge=0) + sessions: int = Field(ge=0) + + +class ScenarioResult(_FrozenModel): + fixture: FixtureShape + warmups: int = Field(ge=0) + iterations: int = Field(gt=0) + phases: dict[str, PhaseSamples] + registry_hash: str + allocation_peak_bytes: int = Field(ge=0) + retained_object_delta: int + cancellation: CancellationResult + leaks: LeakSnapshot + task_count_peak: int = Field(ge=0) + operation_count: int = Field(ge=0) + category_counts: dict[str, int] + projection_counts: dict[str, int] + lifecycle_status: Literal["completed", "settled"] + + +class ThresholdState(StrEnum): + CROSSED = "crossed" + UNCROSSED = "uncrossed" + INCONCLUSIVE = "inconclusive" + + +class ThresholdDecision(_FrozenModel): + name: str + threshold: float + values: tuple[float, float, float, float, float] + rerun_values: tuple[float, float, float, float, float] | None + crossing_count: int = Field(ge=0, le=5) + median: float + primary_state: ThresholdState + rerun_state: ThresholdState | None + state: ThresholdState + rerun_required: bool + + +class CharacterizationReport(_FrozenModel): + schema_version: Literal[2] = 2 + environment: EnvironmentSnapshot + scenarios: tuple[ScenarioResult, ...] + decisions: tuple[ThresholdDecision, ...] + + +def evaluate_threshold( + *, + name: str, + threshold: float, + values: Sequence[float], + rerun_values: Sequence[float] | None = None, +) -> ThresholdDecision: + if not math.isfinite(threshold): + raise ValueError("threshold must be finite") + primary = _five_values(values) + primary_state, primary_count, primary_median = _evaluate_five(primary, threshold) + if rerun_values is not None and primary_state is not ThresholdState.INCONCLUSIVE: + raise ValueError("rerun_values are only valid after an inconclusive primary run") + + selected = primary + state = primary_state + crossing_count = primary_count + median = primary_median + normalized_rerun: tuple[float, float, float, float, float] | None = None + rerun_state: ThresholdState | None = None + if rerun_values is not None: + normalized_rerun = _five_values(rerun_values) + selected = normalized_rerun + state, crossing_count, median = _evaluate_five(selected, threshold) + rerun_state = state + + return ThresholdDecision( + name=name, + threshold=threshold, + values=primary, + rerun_values=normalized_rerun, + crossing_count=crossing_count, + median=median, + primary_state=primary_state, + rerun_state=rerun_state, + state=state, + rerun_required=state is ThresholdState.INCONCLUSIVE and normalized_rerun is None, + ) + + +def build_threshold_decisions( + scenarios: Sequence[ScenarioResult], +) -> tuple[ThresholdDecision, ...]: + indexed: dict[tuple[ScenarioKind, int], ScenarioResult] = { + (scenario.fixture.kind, scenario.fixture.size): scenario for scenario in scenarios + } + if len(indexed) != len(scenarios): + raise ValueError("threshold scenarios must have unique kind and size") + + safe = _required_scenario(indexed, "execution_safe", 1) + mixed = _required_scenario(indexed, "execution_mixed", 10) + advertisement = _required_scenario(indexed, "advertisement", 500) + mcp = {size: _required_scenario(indexed, "mcp", size) for size in (1, 10, 50)} + + return ( + evaluate_threshold( + name="execution_framework_overhead_percent_short_safe_size_1", + threshold=10.0, + values=_percent_values(safe, "framework_overhead", "end_to_end"), + ), + evaluate_threshold( + name="mcp_lifecycle_startup_percent_10_servers", + threshold=20.0, + values=_percent_values(mcp[10], "mcp_lifecycle", "startup_to_ready"), + ), + *( + evaluate_threshold( + name=f"mcp_cleanup_seconds_{size}_servers", + threshold=6.0, + values=tuple( + sample / 1_000_000_000 + for sample in _required_phase(mcp[size], "cleanup").samples_ns + ), + ) + for size in (1, 10, 50) + ), + evaluate_threshold( + name="registry_projection_p95_ms_500_tools", + threshold=5.0, + values=tuple( + sample / 1_000_000 + for sample in _required_phase(advertisement, "registry_projection_p95").samples_ns + ), + ), + evaluate_threshold( + name="mixed_gate_wait_end_to_end_percent_10_pairs", + threshold=25.0, + values=_percent_values(mixed, "read_write_gate_wait", "end_to_end"), + ), + ) + + +def _required_scenario( + indexed: dict[tuple[ScenarioKind, int], ScenarioResult], + kind: ScenarioKind, + size: int, +) -> ScenarioResult: + try: + return indexed[(kind, size)] + except KeyError as exc: + raise ValueError(f"missing threshold scenario: {kind}:{size}") from exc + + +def _required_phase(scenario: ScenarioResult, name: str) -> PhaseSamples: + try: + phase = scenario.phases[name] + except KeyError as exc: + raise ValueError( + f"missing threshold phase: {scenario.fixture.kind}:{scenario.fixture.size}:{name}" + ) from exc + if phase.measurement_status != "measured": + raise ValueError(f"threshold phase is unmeasured: {name}") + return phase + + +def _percent_values( + scenario: ScenarioResult, + numerator_name: str, + denominator_name: str, +) -> tuple[float, ...]: + numerators = _required_phase(scenario, numerator_name).samples_ns + denominators = _required_phase(scenario, denominator_name).samples_ns + if len(numerators) != len(denominators): + raise ValueError("threshold phase sample counts must match") + if any(denominator <= 0 for denominator in denominators): + raise ValueError("threshold denominator samples must be positive") + return tuple( + numerator / denominator * 100 + for numerator, denominator in zip(numerators, denominators, strict=True) + ) + + +def _five_values(values: Sequence[float]) -> tuple[float, float, float, float, float]: + if len(values) != 5: + raise ValueError("threshold evaluation requires exactly five measured runs") + normalized = tuple(float(value) for value in values) + if not all(math.isfinite(value) for value in normalized): + raise ValueError("threshold values must be finite") + return cast(tuple[float, float, float, float, float], normalized) + + +def _evaluate_five( + values: tuple[float, float, float, float, float], threshold: float +) -> tuple[ThresholdState, int, float]: + crossing_count = sum(value > threshold for value in values) + median = float(statistics.median(values)) + if crossing_count >= 4 and median > threshold: + return ThresholdState.CROSSED, crossing_count, median + if crossing_count == 1: + return ThresholdState.INCONCLUSIVE, crossing_count, median + return ThresholdState.UNCROSSED, crossing_count, median + + +def deterministic_registry_hash(names: Iterable[str]) -> str: + digest = hashlib.sha256() + for name in names: + encoded = name.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + +def interval_union_duration_ns(intervals: Iterable[tuple[int, int]]) -> int: + ordered = sorted(intervals) + if not ordered: + return 0 + total = 0 + current_start, current_end = ordered[0] + if current_end < current_start: + raise ValueError("interval end must not precede its start") + for start, end in ordered[1:]: + if end < start: + raise ValueError("interval end must not precede its start") + if start <= current_end: + current_end = max(current_end, end) + continue + total += current_end - current_start + current_start, current_end = start, end + return total + current_end - current_start + + +def fixture_matrix(*, smoke: bool = False) -> tuple[FixtureShape, ...]: + execution_sizes = (1,) if smoke else (1, 10, 100) + dedupe_sizes = (1024,) if smoke else (1024, 100 * 1024, 1024 * 1024) + advertisement_sizes = (50,) if smoke else (50, 500, 5000) + mcp_sizes = (1,) if smoke else (1, 10, 50) + fixtures: list[FixtureShape] = [] + for kind in ("execution_safe", "execution_exclusive", "execution_mixed"): + fixtures.extend( + FixtureShape( + kind=kind, + size=size, + concurrency=size * 2 if kind == "execution_mixed" else size, + composition=( + "reader/writer pairs" + if kind == "execution_mixed" + else kind.removeprefix("execution_") + ), + ) + for size in execution_sizes + ) + fixtures.extend( + FixtureShape( + kind="dedupe", + size=size, + concurrency=2, + payload_bytes=size, + composition="same-step duplicate payload", + ) + for size in dedupe_sizes + ) + fixtures.extend( + FixtureShape( + kind="advertisement", + size=size, + concurrency=1, + composition="builtin/plugin/MCP-style names with hidden entries", + ) + for size in advertisement_sizes + ) + fixtures.extend( + FixtureShape( + kind="mcp", + size=size, + concurrency=size, + composition="deterministic fake server inventory publication", + ) + for size in mcp_sizes + ) + return tuple(fixtures) + + +class _NoopTool(CallableTool): + supports_parallel: ClassVar[bool] = True + _intervals_ns: list[tuple[int, int]] = PrivateAttr( + default_factory=lambda: list[tuple[int, int]]() + ) + + @property + def intervals_ns(self) -> tuple[tuple[int, int], ...]: + return tuple(self._intervals_ns) + + async def __call__(self, **_kwargs: object) -> ToolReturnValue: + started = time.monotonic_ns() + result = ToolOk(output="ok") + self._intervals_ns.append((started, time.monotonic_ns())) + return result + + +class _ExclusiveNoopTool(_NoopTool): + supports_parallel: ClassVar[bool] = False + + +class _BarrierTool(_NoopTool): + _entered: asyncio.Event = PrivateAttr() + _release: asyncio.Event = PrivateAttr() + + def configure(self, *, entered: asyncio.Event, release: asyncio.Event) -> None: + self._entered = entered + self._release = release + + async def __call__(self, **_kwargs: object) -> ToolReturnValue: + started = time.monotonic_ns() + self._entered.set() + try: + await self._release.wait() + return ToolOk(output="released") + finally: + self._intervals_ns.append((started, time.monotonic_ns())) + + +class _ExclusiveBarrierTool(_BarrierTool): + supports_parallel: ClassVar[bool] = False + + +class _ExecutionProbeToolset(PythinkerToolset): + def __init__(self) -> None: + super().__init__() + self.gate_wait_intervals_ns: list[tuple[str, int, int]] = [] + self.gate_requested: dict[str, asyncio.Event] = {} + + async def _gated_call(self, tool: ToolType, arguments: JsonType) -> ToolReturnValue: + self.gate_requested.setdefault(tool.name, asyncio.Event()).set() + requested_ns = time.monotonic_ns() + if getattr(tool, "supports_parallel", False): + async with self._concurrency_gate.shared(): + admitted_ns = time.monotonic_ns() + self.gate_wait_intervals_ns.append((tool.name, requested_ns, admitted_ns)) + return await tool.call(arguments) + async with self._concurrency_gate.exclusive(): + admitted_ns = time.monotonic_ns() + self.gate_wait_intervals_ns.append((tool.name, requested_ns, admitted_ns)) + return await tool.call(arguments) + + +def _new_tool(name: str, *, parallel: bool) -> _NoopTool: + tool_type = _NoopTool if parallel else _ExclusiveNoopTool + return tool_type( + name=name, + description="Deterministic local characterization no-op.", + parameters={ + "type": "object", + "properties": { + "index": {"type": "integer"}, + "payload": {"type": "string"}, + }, + "additionalProperties": False, + }, + ) + + +async def run_characterization( + *, + scenario: str = "all", + runs: int = 5, + warmups: int = 1, + smoke: bool = False, +) -> CharacterizationReport: + if scenario not in {"all", "execution", "dedupe", "advertisement", "mcp"}: + raise ValueError(f"unknown scenario: {scenario}") + if runs < 1: + raise ValueError("runs must be positive") + if warmups < 0: + raise ValueError("warmups must be non-negative") + + fixtures = tuple( + fixture + for fixture in fixture_matrix(smoke=smoke) + if scenario == "all" or _scenario_group(fixture.kind) == scenario + ) + results = [await _measure_fixture(fixture, runs=runs, warmups=warmups) for fixture in fixtures] + decisions = ( + build_threshold_decisions(results) if scenario == "all" and runs == 5 and not smoke else () + ) + return CharacterizationReport( + environment=EnvironmentSnapshot.current(), + scenarios=tuple(results), + decisions=decisions, + ) + + +def _scenario_group(kind: ScenarioKind) -> str: + return "execution" if kind.startswith("execution_") else kind + + +async def _measure_fixture(fixture: FixtureShape, *, runs: int, warmups: int) -> ScenarioResult: + measure = _measurement_for(fixture) + for _ in range(warmups): + await measure(fixture) + + before_tasks = _pending_task_count() + phase_samples: dict[str, list[int]] = {} + within_run_samples: dict[str, list[tuple[int, ...]]] = {} + hashes: list[str] = [] + task_count_peak = 0 + operation_count = 0 + category_counts: dict[str, int] = {} + projection_counts: dict[str, int] = {} + peak_bytes = 0 + retained_delta = 0 + tracing_was_active = tracemalloc.is_tracing() + if not tracing_was_active: + tracemalloc.start() + tracemalloc.reset_peak() + try: + baseline_current, _ = tracemalloc.get_traced_memory() + for _ in range(runs): + sample = await measure(fixture) + hashes.append(sample.registry_hash) + task_count_peak = max(task_count_peak, sample.task_count) + operation_count += sample.operation_count + category_counts = sample.category_counts + projection_counts = sample.projection_counts + for phase, duration_ns in sample.phases.items(): + phase_samples.setdefault(phase, []).append(duration_ns) + for phase, raw_samples in sample.within_run_samples.items(): + within_run_samples.setdefault(phase, []).append(raw_samples) + current_bytes, peak_bytes = tracemalloc.get_traced_memory() + retained_delta = current_bytes - baseline_current + finally: + if not tracing_was_active: + tracemalloc.stop() + + if len(set(hashes)) != 1: + raise RuntimeError(f"non-deterministic registry order for {fixture.kind}:{fixture.size}") + cancellation = ( + await _measure_cancellation() + if fixture.kind.startswith("execution_") + else CancellationResult(completed=True, completion_ns=0) + ) + leaked_tasks = max(0, _pending_task_count() - before_tasks) + operations = operation_count // runs + summarized_phases = { + name: PhaseSamples.from_samples( + samples, + operations_per_sample=operations, + within_run_samples_ns=within_run_samples.get(name, ()), + ) + for name, samples in phase_samples.items() + } + if fixture.kind.startswith("execution_") or fixture.kind == "dedupe": + for name in _UNMEASURED_EXECUTION_PHASES: + summarized_phases[name] = PhaseSamples.unmeasured( + "current Toolset exposes no stable boundary for this subphase" + ) + return ScenarioResult( + fixture=fixture, + warmups=warmups, + iterations=runs, + phases=summarized_phases, + registry_hash=hashes[0], + allocation_peak_bytes=peak_bytes, + retained_object_delta=retained_delta, + cancellation=cancellation, + leaks=LeakSnapshot(tasks=leaked_tasks, processes=0, sessions=0), + task_count_peak=task_count_peak, + operation_count=operation_count, + category_counts=category_counts, + projection_counts=projection_counts, + lifecycle_status="settled" if fixture.kind == "mcp" else "completed", + ) + + +@dataclass(frozen=True, slots=True) +class _MeasurementSample: + phases: dict[str, int] + registry_hash: str + task_count: int + operation_count: int + category_counts: dict[str, int] + projection_counts: dict[str, int] + within_run_samples: dict[str, tuple[int, ...]] = field( + default_factory=lambda: dict[str, tuple[int, ...]]() + ) + + +type _Measure = Callable[[FixtureShape], Awaitable[_MeasurementSample]] + + +def _measurement_for(fixture: FixtureShape) -> _Measure: + if fixture.kind.startswith("execution_") or fixture.kind == "dedupe": + return _measure_execution + if fixture.kind == "advertisement": + return _measure_advertisement + return _measure_mcp_publication + + +async def _measure_execution(fixture: FixtureShape) -> _MeasurementSample: + toolset = _ExecutionProbeToolset() + tools, holder_entered, holder_release = _execution_tools(fixture) + for tool in tools: + toolset.add(tool) + toolset.begin_step([]) + payload = "x" * (fixture.payload_bytes or 0) + calls = _execution_calls(fixture, tools, payload) + + started = time.monotonic_ns() + pending: list[HandleResult] + if holder_entered is not None and holder_release is not None: + holder_result = toolset.handle(calls[0]) + await holder_entered.wait() + queued_result = toolset.handle(calls[1]) + await toolset.gate_requested.setdefault(tools[1].name, asyncio.Event()).wait() + pending = [holder_result, queued_result, *(toolset.handle(call) for call in calls[2:])] + task_count_peak = _pending_task_count() + holder_release.set() + else: + pending = [toolset.handle(call) for call in calls] + task_count_peak = _pending_task_count() + await asyncio.gather(*(_await_handle(result) for result in pending)) + end_to_end_ns = time.monotonic_ns() - started + tool_intervals = tuple(interval for tool in tools for interval in tool.intervals_ns) + tool_duration_ns = interval_union_duration_ns(tool_intervals) + gate_wait_ns = interval_union_duration_ns( + tuple((requested, admitted) for _, requested, admitted in toolset.gate_wait_intervals_ns) + ) + framework_overhead_ns = max(0, end_to_end_ns - tool_duration_ns) + registry_hash = deterministic_registry_hash(tool.name for tool in toolset.tools) + return _MeasurementSample( + phases={ + "end_to_end": end_to_end_ns, + "read_write_gate_wait": gate_wait_ns, + "tool_call": tool_duration_ns, + "framework_overhead": framework_overhead_ns, + }, + registry_hash=registry_hash, + task_count=task_count_peak, + operation_count=len(tool_intervals), + category_counts={"builtin": len(tools)}, + projection_counts={"visible": len(toolset.tools)}, + ) + + +def _execution_tools( + fixture: FixtureShape, +) -> tuple[tuple[_NoopTool, ...], asyncio.Event | None, asyncio.Event | None]: + if fixture.kind == "execution_mixed" and fixture.concurrency >= 2: + entered = asyncio.Event() + release = asyncio.Event() + holder = _BarrierTool( + name="SafeNoop", + description="Deterministic contended reader holder.", + parameters={ + "type": "object", + "properties": { + "index": {"type": "integer"}, + "payload": {"type": "string"}, + }, + "additionalProperties": False, + }, + ) + holder.configure(entered=entered, release=release) + return (holder, _new_tool("ExclusiveNoop", parallel=False)), entered, release + return ( + (_new_tool("Noop", parallel=fixture.kind != "execution_exclusive"),), + None, + None, + ) + + +def _execution_calls( + fixture: FixtureShape, tools: tuple[_NoopTool, ...], payload: str +) -> list[ToolCall]: + calls: list[ToolCall] = [] + for index in range(fixture.concurrency): + tool = tools[index % len(tools)] + argument_index = 0 if fixture.kind == "dedupe" else index + calls.append( + ToolCall( + id=f"characterization-{index}", + function=ToolCall.FunctionBody( + name=tool.name, + arguments=_arguments_json(argument_index, payload), + ), + ) + ) + return calls + + +def _arguments_json(index: int, payload: str) -> str: + import json + + return json.dumps({"index": index, "payload": payload}, separators=(",", ":")) + + +async def _await_handle(result: HandleResult) -> ToolResult: + if isinstance(result, ToolResult): + return result + return await result + + +class _AdvertisementProbeToolset(PythinkerToolset): + def publish_mcp_tools( + self, + runtime: Runtime, + tools: list[MCPTool[MCPConfigTransport]], + client: FastMcpClient[MCPConfigTransport], + ) -> None: + self._mcp_servers["benchmark"] = MCPServerInfo( + status="connected", + client=client, + tools=tools, + resources=[], + prompts=[], + ) + self._rebuild_published_mcp_tools(runtime) + + +async def _measure_advertisement(fixture: FixtureShape) -> _MeasurementSample: + from fastmcp import Client + from fastmcp.mcp_config import MCPConfig + + from pythinker_code.soul.agent import Runtime + + with tempfile.TemporaryDirectory(prefix="pythinker-advertisement-") as session_dir: + runtime_config, oauth, session = _benchmark_runtime_inputs(Path(session_dir)) + runtime = await Runtime.create( + runtime_config, + oauth, + None, + session, + yolo=True, + skills_dirs=[], + ) + client = Client( + MCPConfig.model_validate( + {"mcpServers": {"benchmark": {"command": "pythinker-characterization-fake"}}} + ) + ) + enabled = _AdvertisementProbeToolset(runtime) + disabled = PythinkerToolset() + mcp_tools, category_counts = _populate_advertisement_tools( + fixture, + runtime=runtime, + client=client, + enabled=enabled, + disabled=disabled, + ) + rebuild_started = time.monotonic_ns() + enabled.publish_mcp_tools(runtime, mcp_tools, client) + rebuild_ns = time.monotonic_ns() - rebuild_started + for tool in mcp_tools: + disabled.add(tool) + phases, projections, within_run_samples = _measure_visibility_projections( + enabled, disabled, fixture.size + ) + phases["rebuild_after_mcp_publication"] = rebuild_ns + registry_hash = _advertisement_hash(enabled, disabled) + return _MeasurementSample( + phases=phases, + registry_hash=registry_hash, + task_count=_pending_task_count(), + operation_count=fixture.size, + category_counts=category_counts, + projection_counts=projections, + within_run_samples=within_run_samples, + ) + + +def _populate_advertisement_tools( + fixture: FixtureShape, + *, + runtime: Runtime, + client: FastMcpClient[MCPConfigTransport], + enabled: PythinkerToolset, + disabled: PythinkerToolset, +) -> tuple[list[MCPTool[MCPConfigTransport]], dict[str, int]]: + from pythinker_code.plugin import PluginToolSpec + from pythinker_code.plugin.tool import PluginTool + + mcp_tools: list[MCPTool[MCPConfigTransport]] = [] + counts = {"builtin": 0, "plugin": 0, "mcp": 0} + for index in range(fixture.size): + category = ("builtin", "plugin", "mcp")[index % 3] + name = f"Characterization_{category}_{index:05d}" + if category == "builtin": + tool: ToolType = _new_tool(name, parallel=True) + elif category == "plugin": + tool = PluginTool( + PluginToolSpec( + name=name, + description="Local advertisement fixture.", + command=["false"], + ), + Path.cwd(), + inject={}, + config=runtime.config, + ) + else: + raw_tool = mcp.Tool(name=name, description="Local MCP fixture.", inputSchema={}) + mcp_tool = MCPTool( + "benchmark", + raw_tool, + client, + runtime=runtime, + ) + mcp_tools.append(mcp_tool) + counts[category] += 1 + continue + enabled.add(tool) + disabled.add(tool) + counts[category] += 1 + return mcp_tools, counts + + +def _measure_visibility_projections( + enabled: PythinkerToolset, + disabled: PythinkerToolset, + fixture_size: int, +) -> tuple[dict[str, int], dict[str, int], dict[str, tuple[int, ...]]]: + hidden_names = tuple( + f"Characterization_{('builtin', 'plugin', 'mcp')[index % 3]}_{index:05d}" + for index in range(0, fixture_size, 10) + ) + for name in hidden_names: + enabled.hide(name) + disabled.hide(name) + phases: dict[str, int] = {} + projections: dict[str, int] = {} + for label, toolset in (("enabled", enabled), ("disabled", disabled)): + started = time.monotonic_ns() + hidden = toolset.tools + phases[f"visibility_{label}_hidden"] = time.monotonic_ns() - started + projections[f"{label}_hidden"] = len(hidden) + for name in hidden_names: + toolset.unhide(name) + started = time.monotonic_ns() + unhidden = toolset.tools + phases[f"visibility_{label}_unhidden"] = time.monotonic_ns() - started + projections[f"{label}_unhidden"] = len(unhidden) + started = time.monotonic_ns() + repeated_projection = enabled.tools + for _ in range(3): + repeated_projection = enabled.tools + phases["repeated_unchanged_projection"] = time.monotonic_ns() - started + projection_samples: list[int] = [] + for _ in range(_REGISTRY_PROJECTIONS_PER_RUN): + started = time.monotonic_ns() + repeated_projection = enabled.tools + projection_samples.append(time.monotonic_ns() - started) + phases["registry_projection_p95"] = _nearest_rank_p95_ns(projection_samples) + projections["rebuild"] = len(enabled.tools) + projections["repeated"] = len(repeated_projection) + return phases, projections, {"registry_projection_p95": tuple(projection_samples)} + + +def _nearest_rank_p95_ns(samples_ns: Sequence[int]) -> int: + if not samples_ns: + raise ValueError("p95 samples must not be empty") + ordered = sorted(int(sample) for sample in samples_ns) + if ordered[0] < 0: + raise ValueError("p95 samples must be non-negative") + return ordered[max(0, math.ceil(len(ordered) * 0.95) - 1)] + + +def _advertisement_hash(enabled: PythinkerToolset, disabled: PythinkerToolset) -> str: + return deterministic_registry_hash( + ( + *(tool.name for tool in enabled.tools), + "--visibility-disabled--", + *(tool.name for tool in disabled.tools), + ) + ) + + +class _BenchmarkMcpToolset(PythinkerToolset): + def __init__(self) -> None: + super().__init__() + self.lifecycle_started_ns = 0 + self.publication_ns: int | None = None + self.publication_visible_count = 0 + + async def _connect_mcp_server( + self, server_name: str, server_info: MCPServerInfo, runtime: Runtime + ) -> tuple[str, Exception | None]: + raw_tool = mcp.Tool( + name=f"{server_name}_noop", + description="Deterministic local MCP characterization no-op.", + inputSchema={"type": "object", "properties": {}}, + ) + server_info.tools = [MCPTool(server_name, raw_tool, server_info.client, runtime=runtime)] + server_info.status = "connected" + return server_name, None + + def _publish_connected_mcp_tools(self, runtime: Runtime) -> None: + super()._publish_connected_mcp_tools(runtime) + if self.publication_ns is None and self.tools: + self.publication_ns = time.monotonic_ns() + self.publication_visible_count = len(self.tools) + + +async def _measure_mcp_publication(fixture: FixtureShape) -> _MeasurementSample: + from fastmcp.mcp_config import MCPConfig + + from pythinker_code.soul.agent import Runtime + + server_map = { + f"server_{index:03d}": {"command": "pythinker-characterization-fake"} + for index in range(fixture.size) + } + config = MCPConfig.model_validate({"mcpServers": server_map}) + with tempfile.TemporaryDirectory(prefix="pythinker-toolset-") as session_dir: + runtime_config, oauth, session = _benchmark_runtime_inputs(Path(session_dir)) + startup_started_ns = time.monotonic_ns() + runtime = await Runtime.create( + runtime_config, + oauth, + None, + session, + yolo=True, + skills_dirs=[], + ) + toolset = _BenchmarkMcpToolset() + toolset.lifecycle_started_ns = time.monotonic_ns() + await toolset.load_mcp_tools([config], runtime, in_background=True) + task_count_peak = _pending_task_count() + await toolset.wait_for_mcp_tools() + settled_ns = time.monotonic_ns() + names = tuple(tool.name for tool in toolset.tools) + cleanup_started_ns = time.monotonic_ns() + await toolset.cleanup() + cleanup_ns = time.monotonic_ns() - cleanup_started_ns + publication_ns = toolset.publication_ns + if publication_ns is None: + raise RuntimeError("fake MCP inventory was not published") + return _MeasurementSample( + phases={ + "startup_to_ready": settled_ns - startup_started_ns, + "mcp_lifecycle": settled_ns - toolset.lifecycle_started_ns, + "time_to_first_inventory": publication_ns - toolset.lifecycle_started_ns, + "time_to_settled_inventory": settled_ns - toolset.lifecycle_started_ns, + "cleanup": cleanup_ns, + }, + registry_hash=deterministic_registry_hash(names), + task_count=task_count_peak, + operation_count=len(names), + category_counts={"mcp": len(names)}, + projection_counts={ + "visible": len(names), + "visible_at_first_publication": toolset.publication_visible_count, + }, + ) + + +def _benchmark_runtime_inputs(session_dir: Path) -> tuple[Config, OAuthManager, Session]: + from pythinker_host import get_current_host + from pythinker_host.path import HostPath + + from pythinker_code.auth.oauth import OAuthManager + from pythinker_code.config import get_default_config + from pythinker_code.metadata import WorkDirMeta + from pythinker_code.session import Session + from pythinker_code.session_state import SessionState + from pythinker_code.wire.file import WireFile + + work_dir_path = session_dir / "work" + work_dir_path.mkdir() + work_dir = HostPath.unsafe_from_local_path(work_dir_path) + config = get_default_config() + config.plugins.discover_external = False + # LSP is orthogonal to toolset characterization; leaving it enabled spawns a + # per-sample language-server init task that is never torn down (Runtime has no + # shutdown) and perturbs the deterministic pending-task counts this benchmark measures. + config.lsp.enabled = False + session = Session( + id="toolset-characterization", + work_dir=work_dir, + work_dir_meta=WorkDirMeta(path=str(work_dir), host=get_current_host().name), + context_file=session_dir / "context.jsonl", + wire_file=WireFile(path=session_dir / "wire.jsonl"), + state=SessionState(), + title="Toolset characterization", + updated_at=0.0, + ) + return config, OAuthManager(config), session + + +async def _measure_cancellation() -> CancellationResult: + started = time.monotonic_ns() + queued_reader_completed, reader_recovered = await _cancel_queued_call( + holder_parallel=False, + queued_parallel=True, + ) + queued_writer_completed, writer_recovered = await _cancel_queued_call( + holder_parallel=True, + queued_parallel=False, + ) + recovery_completed = reader_recovered and writer_recovered + return CancellationResult( + completed=queued_reader_completed and queued_writer_completed and recovery_completed, + completion_ns=time.monotonic_ns() - started, + queued_reader_completed=queued_reader_completed, + queued_writer_completed=queued_writer_completed, + recovery_completed=recovery_completed, + ) + + +async def _cancel_queued_call(*, holder_parallel: bool, queued_parallel: bool) -> tuple[bool, bool]: + toolset = _ExecutionProbeToolset() + entered = asyncio.Event() + release = asyncio.Event() + holder_type = _BarrierTool if holder_parallel else _ExclusiveBarrierTool + holder = holder_type( + name="Holder", + description="Cancellation characterization holder.", + parameters={"type": "object", "properties": {}}, + ) + holder.configure(entered=entered, release=release) + queued = _new_tool("Queued", parallel=queued_parallel) + toolset.add(holder) + toolset.add(queued) + holder_result = toolset.handle(_tool_call("holder", "Holder", 0)) + await entered.wait() + gate_requested = toolset.gate_requested.setdefault("Queued", asyncio.Event()) + queued_result = toolset.handle(_tool_call("queued", "Queued", 1)) + await gate_requested.wait() + queued_completed = await _cancel_handle_result(queued_result) + release.set() + await _await_handle(holder_result) + recovery = toolset.handle(_tool_call("recovery", "Queued", 2)) + recovered = not (await _await_handle(recovery)).return_value.is_error + return queued_completed, recovered + + +async def _cancel_handle_result(result: HandleResult) -> bool: + if isinstance(result, ToolResult): + return False + result.cancel() + try: + await result + except asyncio.CancelledError: + return True + return False + + +def _tool_call(call_id: str, name: str, index: int) -> ToolCall: + return ToolCall( + id=call_id, + function=ToolCall.FunctionBody(name=name, arguments=_arguments_json(index, "")), + ) + + +def _pending_task_count() -> int: + current = asyncio.current_task() + return sum(task is not current and not task.done() for task in asyncio.all_tasks()) + + +__all__ = [ + "CancellationResult", + "CharacterizationReport", + "EnvironmentSnapshot", + "FixtureShape", + "LeakSnapshot", + "PhaseSamples", + "ScenarioResult", + "ThresholdDecision", + "ThresholdState", + "deterministic_registry_hash", + "build_threshold_decisions", + "evaluate_threshold", + "fixture_matrix", + "run_characterization", +] diff --git a/src/pythinker_code/cli/system_prompt.py b/src/pythinker_code/cli/system_prompt.py index e7689cec..887f582f 100644 --- a/src/pythinker_code/cli/system_prompt.py +++ b/src/pythinker_code/cli/system_prompt.py @@ -51,7 +51,8 @@ def system_prompt( """Print the fully-assembled system prompt for an agent. Read-only: renders the prompt the agent would receive (work dir, OS, shell, - AGENTS.md, skills) without creating a session, authenticating, or loading MCP. + AGENTS.md, and stable skill-catalogue metadata) without creating a session, + authenticating, loading MCP, or inventing task-specific skill candidates. """ from pythinker_host.path import HostPath diff --git a/src/pythinker_code/skill/__init__.py b/src/pythinker_code/skill/__init__.py index 9ec08453..cb833758 100644 --- a/src/pythinker_code/skill/__init__.py +++ b/src/pythinker_code/skill/__init__.py @@ -3,12 +3,12 @@ from __future__ import annotations import sys -from collections.abc import Callable, Iterable, Iterator, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import Literal, cast -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, ValidationError from pythinker_host import get_current_host from pythinker_host.local import local_host from pythinker_host.path import HostPath @@ -410,7 +410,7 @@ def local_specialization_name(name: str) -> str: return f"{name}{LOCAL_SPECIALIZATION_SUFFIX}" -def get_local_specialization(skill: Skill, skills_by_name: dict[str, Skill]) -> Skill | None: +def get_local_specialization(skill: Skill, skills_by_name: Mapping[str, Skill]) -> Skill | None: """Return the ``-local`` companion, if one is available. Local specializations are additive supplements to a core workflow skill, not @@ -422,7 +422,7 @@ def get_local_specialization(skill: Skill, skills_by_name: dict[str, Skill]) -> async def read_skill_text_with_local_specialization( - skill: Skill, skills_by_name: dict[str, Skill] + skill: Skill, skills_by_name: Mapping[str, Skill] ) -> str | None: """Read a skill body for injection into the model context. @@ -567,10 +567,21 @@ class Skill(BaseModel): model can tell user-scope from project-scope skills.""" +@dataclass(frozen=True, slots=True) +class SkillDiscoveryIssue: + name: str + source_kind: Literal["root", "directory", "flat_file"] + path: HostPath + scope: SkillScope + reason_code: str + safe_reason: str + + async def discover_skills( skills_dir: HostPath, *, scope: SkillScope, + diagnostic_collector: Callable[[SkillDiscoveryIssue], None] | None = None, ) -> list[Skill]: """Discover all skills in the given directory. @@ -596,6 +607,17 @@ async def discover_skills( path=skills_dir, error=exc, ) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name="", + source_kind="root", + path=skills_dir, + scope=scope, + reason_code="unreadable_skill_root", + safe_reason="Skill root could not be read.", + ), + ) return [] if not is_dir: return [] @@ -618,13 +640,35 @@ async def discover_skills( path=entry, error=exc, ) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name=entry.name, + source_kind="directory", + path=entry, + scope=scope, + reason_code="unreadable_skill_source", + safe_reason="Skill source could not be read.", + ), + ) continue try: skill = parse_skill_text( content, dir_path=entry, skill_md_file=skill_md, scope=scope ) - except Exception as exc: + except (ValueError, ValidationError) as exc: logger.info("Skipping invalid skill at {}: {}", skill_md, exc) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name=entry.name, + source_kind="directory", + path=skill_md, + scope=scope, + reason_code="invalid_skill_metadata", + safe_reason="Skill metadata could not be parsed.", + ), + ) continue skills_by_name[normalize_skill_name(skill.name)] = skill except OSError as exc: @@ -633,7 +677,18 @@ async def discover_skills( path=skills_dir, error=exc, ) - return sorted(skills_by_name.values(), key=lambda s: s.name) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name="", + source_kind="root", + path=skills_dir, + scope=scope, + reason_code="unreadable_skill_root", + safe_reason="Skill root could not be enumerated.", + ), + ) + return sorted(skills_by_name.values(), key=lambda skill: skill.name) # Pass 2: flat ``.md`` form, skipping names already claimed by a subdir. try: @@ -647,6 +702,17 @@ async def discover_skills( path=entry, error=exc, ) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name=entry.name, + source_kind="flat_file", + path=entry, + scope=scope, + reason_code="unreadable_skill_source", + safe_reason="Skill source could not be inspected.", + ), + ) continue if not entry.name.lower().endswith(".md"): continue @@ -657,6 +723,21 @@ async def discover_skills( try: content = await entry.read_text(encoding="utf-8") + except OSError as exc: + logger.info("Skipping unreadable flat skill at {}: {}", entry, exc) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name=_strip_md_suffix(entry.name), + source_kind="flat_file", + path=entry, + scope=scope, + reason_code="unreadable_skill_source", + safe_reason="Skill source could not be read.", + ), + ) + continue + try: skill = parse_skill_text( content, dir_path=skills_dir, @@ -664,8 +745,19 @@ async def discover_skills( scope=scope, flat_file=entry, ) - except Exception as exc: + except (ValueError, ValidationError) as exc: logger.info("Skipping invalid flat skill at {}: {}", entry, exc) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name=_strip_md_suffix(entry.name), + source_kind="flat_file", + path=entry, + scope=scope, + reason_code="invalid_skill_metadata", + safe_reason="Skill metadata could not be parsed.", + ), + ) continue key = normalize_skill_name(skill.name) @@ -684,8 +776,27 @@ async def discover_skills( path=skills_dir, error=exc, ) + _collect_discovery_issue( + diagnostic_collector, + SkillDiscoveryIssue( + name="", + source_kind="root", + path=skills_dir, + scope=scope, + reason_code="unreadable_skill_root", + safe_reason="Skill root could not be enumerated.", + ), + ) + + return sorted(skills_by_name.values(), key=lambda skill: skill.name) - return sorted(skills_by_name.values(), key=lambda s: s.name) + +def _collect_discovery_issue( + collector: Callable[[SkillDiscoveryIssue], None] | None, + issue: SkillDiscoveryIssue, +) -> None: + if collector is not None: + collector(issue) _DESCRIPTION_FALLBACK_MAX_LEN = 240 @@ -868,3 +979,23 @@ def _is_fence_close(line: str, fence_char: str, fence_len: int) -> bool: if count < fence_len: return False return not line[count:].strip() + + +from pythinker_code.skill.catalog import ( # noqa: E402 + SkillCatalog as SkillCatalog, +) +from pythinker_code.skill.catalog import ( # noqa: E402 + SkillMatch as SkillMatch, +) +from pythinker_code.skill.catalog import ( # noqa: E402 + SkillProjectionOutcome as SkillProjectionOutcome, +) +from pythinker_code.skill.catalog import ( # noqa: E402 + SkillProjectionStatus as SkillProjectionStatus, +) +from pythinker_code.skill.catalog import ( # noqa: E402 + SkillPromptView as SkillPromptView, +) +from pythinker_code.skill.catalog import ( # noqa: E402 + render_skill_prompt_view as render_skill_prompt_view, +) diff --git a/src/pythinker_code/skill/catalog.py b/src/pythinker_code/skill/catalog.py new file mode 100644 index 00000000..62db06f2 --- /dev/null +++ b/src/pythinker_code/skill/catalog.py @@ -0,0 +1,509 @@ +"""Deterministic discovery and exact resolution for skills.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from enum import IntEnum, StrEnum +from types import MappingProxyType + +from pythinker_host.path import HostPath + +from pythinker_code.skill import ( + ScopedSkillsRoot, + Skill, + SkillDiscoveryIssue, + SkillScope, + discover_skills, + normalize_skill_name, +) + + +@dataclass(frozen=True, slots=True) +class SkillMatch: + skill: Skill + tier: SkillRelevanceTier + score: int + reasons: tuple[str, ...] + + +class SkillRelevanceTier(IntEnum): + EXACT_NAME = 0 + NAME_PHRASE = 1 + NAME_TOKEN = 2 + DESCRIPTION_TOKEN = 3 + + +@dataclass(frozen=True, slots=True) +class SkillSearchMetrics: + candidates_evaluated: int + match_work_units: int + sort_items: int + sort_comparison_bound: int + + +@dataclass(frozen=True, slots=True) +class SkillSearchResult: + matches: tuple[SkillMatch, ...] + metrics: SkillSearchMetrics + + +class SkillDiagnosticCategory(StrEnum): + UNAVAILABLE = "unavailable" + + +class SkillSourceKind(StrEnum): + ROOT = "root" + DIRECTORY = "directory" + FLAT_FILE = "flat_file" + + +@dataclass(frozen=True, slots=True) +class SkillSourceDiagnostic: + name: str + source_kind: SkillSourceKind + source_id: str + scope: SkillScope + category: SkillDiagnosticCategory + reason_code: str + safe_reason: str + + +class SkillProjectionStatus(StrEnum): + READY = "ready" + DEGRADED = "degraded" + FAILED = "failed" + + +@dataclass(frozen=True, slots=True) +class SkillPromptView: + matches: tuple[SkillMatch, ...] + total_count: int + omitted_count: int + overflowed_priority_count: int + rendered_characters: int + descriptions: tuple[str | None, ...] = () + + +@dataclass(frozen=True, slots=True) +class SkillProjectionOutcome: + status: SkillProjectionStatus + view: SkillPromptView | None + reason_code: str | None + + +class SkillCatalog: + """Own the winning skill index for a resolved root sequence.""" + + def __init__( + self, + skills_by_name: Mapping[str, Skill], + diagnostics: Sequence[SkillSourceDiagnostic], + ) -> None: + ordered_skills = sorted(skills_by_name.values(), key=lambda skill: skill.name) + self._compatibility_mapping = MappingProxyType( + {normalize_skill_name(skill.name): skill for skill in ordered_skills} + ) + self._skills_by_name = MappingProxyType( + { + normalize_skill_name(skill.name): skill.model_copy(deep=True) + for skill in ordered_skills + } + ) + self.diagnostics = tuple(diagnostics) + + @classmethod + async def discover(cls, roots: Sequence[ScopedSkillsRoot]) -> SkillCatalog: + skills_by_name: dict[str, Skill] = {} + diagnostics: list[SkillSourceDiagnostic] = [] + for scoped_root in roots: + issues: list[SkillDiscoveryIssue] = [] + skills = await discover_skills( + scoped_root.root, + scope=scoped_root.scope, + diagnostic_collector=issues.append, + ) + for skill in skills: + skills_by_name.setdefault(normalize_skill_name(skill.name), skill) + diagnostics.extend(_public_diagnostic(issue, scoped_root.root) for issue in issues) + return cls(skills_by_name, diagnostics) + + def resolve(self, name: str) -> Skill | None: + skill = self._resolve_internal(name) + return skill.model_copy(deep=True) if skill is not None else None + + def _resolve_internal(self, name: str) -> Skill | None: + for candidate in _lookup_names(name): + skill = self._skills_by_name.get(normalize_skill_name(candidate)) + if skill is not None: + return skill + return None + + def exhaustive_mapping(self) -> Mapping[str, Skill]: + """Return the immutable normalized compatibility mapping.""" + return self._compatibility_mapping + + def search(self, query: str, *, limit: int) -> tuple[SkillMatch, ...]: + """Return deterministic matches from the complete winning catalogue.""" + return self.search_with_metrics(query, limit=limit).matches + + def search_with_metrics(self, query: str, *, limit: int) -> SkillSearchResult: + """Return matches with immutable, call-local deterministic work metrics.""" + result = self._search_internal(query, limit=limit) + return SkillSearchResult( + matches=tuple(_public_match(match) for match in result.matches), + metrics=result.metrics, + ) + + def _search_internal(self, query: str, *, limit: int) -> SkillSearchResult: + if limit <= 0: + return SkillSearchResult((), SkillSearchMetrics(0, 0, 0, 0)) + query_tokens = _tokens(query) + query_token_set = frozenset(query_tokens) + matches: list[SkillMatch] = [] + work_units = 0 + exact_alias = _exact_search_alias(query, self._skills_by_name) + exact_name: str | None = None + if exact_alias is not None: + exact_name = normalize_skill_name(exact_alias.name) + matches.append( + SkillMatch( + skill=exact_alias, + tier=SkillRelevanceTier.EXACT_NAME, + score=len(_tokens(exact_alias.name)), + reasons=("exact_alias",), + ) + ) + for skill in self._skills_by_name.values(): + if normalize_skill_name(skill.name) == exact_name: + continue + match, candidate_work = _match_skill(skill, query_tokens, query_token_set) + work_units += candidate_work + if match is not None: + matches.append(match) + matches.sort(key=_match_sort_key) + sort_items = len(matches) + return SkillSearchResult( + matches=tuple(matches[:limit]), + metrics=SkillSearchMetrics( + candidates_evaluated=len(self._skills_by_name), + match_work_units=work_units, + sort_items=sort_items, + sort_comparison_bound=sort_items * max(0, sort_items - 1) // 2, + ), + ) + + def prompt_view( + self, + query: str, + *, + max_characters: int, + explicit_names: Sequence[str] = (), + active_names: Sequence[str] = (), + ) -> SkillProjectionOutcome: + """Build a bounded task view without reading skill bodies.""" + if max_characters <= 0: + return SkillProjectionOutcome( + status=SkillProjectionStatus.FAILED, + view=None, + reason_code="invalid_projection_budget", + ) + + ordered: list[SkillMatch] = [] + seen: set[str] = set() + + def _add_priority(name: str, reason: str) -> None: + skill = self._resolve_internal(name) + if skill is None: + return + normalized = normalize_skill_name(skill.name) + if normalized in seen: + return + seen.add(normalized) + ordered.append( + SkillMatch( + skill=skill, + tier=SkillRelevanceTier.EXACT_NAME, + score=500, + reasons=(reason,), + ) + ) + + for name in explicit_names: + _add_priority(name, "explicit") + for name in reversed(tuple(active_names)): + _add_priority(name, "active") + priority_count = len(ordered) + + for match in self._search_internal(query, limit=len(self._skills_by_name)).matches: + normalized = normalize_skill_name(match.skill.name) + if normalized not in seen: + seen.add(normalized) + ordered.append(match) + + total_count = len(ordered) + selected: list[SkillMatch] = [] + for match in ordered: + candidate_matches = tuple((*selected, match)) + candidate = _view( + candidate_matches, + total_count, + overflowed_priority_count=max(0, priority_count - len(candidate_matches)), + ) + if len(render_skill_prompt_view(candidate)) > max_characters: + break + selected.append(match) + + overflowed_priority_count = max(0, priority_count - len(selected)) + descriptions: list[str | None] = [None] * len(selected) + for index, match in enumerate(selected): + descriptions[index] = _concise_description(match.skill.description) + candidate = _view( + tuple(selected), + total_count, + overflowed_priority_count=overflowed_priority_count, + descriptions=tuple(descriptions), + ) + if len(render_skill_prompt_view(candidate)) > max_characters: + descriptions[index] = None + + view = _view( + tuple(selected), + total_count, + overflowed_priority_count=overflowed_priority_count, + descriptions=tuple(descriptions), + ) + rendered = render_skill_prompt_view(view) + if len(rendered) > max_characters: + return SkillProjectionOutcome( + status=SkillProjectionStatus.FAILED, + view=None, + reason_code="projection_budget_too_small", + ) + view = replace(view, rendered_characters=len(rendered)) + if overflowed_priority_count: + return SkillProjectionOutcome( + status=SkillProjectionStatus.DEGRADED, + view=_public_view(view), + reason_code="priority_candidates_overflowed", + ) + return SkillProjectionOutcome( + status=SkillProjectionStatus.READY, + view=_public_view(view), + reason_code=None, + ) + + def unavailable_diagnostic(self, name: str) -> SkillSourceDiagnostic | None: + """Return a matching safe source diagnostic for an exact request.""" + requested = {normalize_skill_name(candidate) for candidate in _lookup_names(name)} + return next( + ( + diagnostic + for diagnostic in self.diagnostics + if normalize_skill_name(diagnostic.name) in requested + ), + None, + ) + + +_TOKEN_RE = re.compile(r"[\w]+", re.UNICODE) +_SCOPE_ORDER: Mapping[SkillScope, int] = { + "project": 0, + "user": 1, + "extra": 2, + "builtin": 3, +} + + +def _tokens(text: str) -> tuple[str, ...]: + return tuple(_TOKEN_RE.findall(text.casefold())) + + +def _match_skill( + skill: Skill, + query_tokens: tuple[str, ...], + query_token_set: frozenset[str], +) -> tuple[SkillMatch | None, int]: + name_tokens = _tokens(skill.name) + description_tokens = _tokens(skill.description) + exact, work_units = _sequence_equal(query_tokens, name_tokens) + if exact: + return ( + SkillMatch( + skill=skill, + tier=SkillRelevanceTier.EXACT_NAME, + score=len(name_tokens), + reasons=("exact_name",), + ), + work_units, + ) + phrase, phrase_work = _contains_sequence(query_tokens, name_tokens) + work_units += phrase_work + if phrase: + return ( + SkillMatch( + skill=skill, + tier=SkillRelevanceTier.NAME_PHRASE, + score=len(name_tokens), + reasons=("name_phrase",), + ), + work_units, + ) + name_overlap = sum(token in query_token_set for token in frozenset(name_tokens)) + work_units += len(frozenset(name_tokens)) + if name_overlap: + return ( + SkillMatch( + skill=skill, + tier=SkillRelevanceTier.NAME_TOKEN, + score=name_overlap, + reasons=("name_token",), + ), + work_units, + ) + description_token_set = frozenset(description_tokens) + description_overlap = sum(token in query_token_set for token in description_token_set) + work_units += len(description_token_set) + if description_overlap: + return ( + SkillMatch( + skill=skill, + tier=SkillRelevanceTier.DESCRIPTION_TOKEN, + score=description_overlap, + reasons=("description_token",), + ), + work_units, + ) + return None, work_units + + +def _sequence_equal(left: tuple[str, ...], right: tuple[str, ...]) -> tuple[bool, int]: + work_units = 1 + if len(left) != len(right): + return False, work_units + for left_token, right_token in zip(left, right, strict=True): + work_units += 1 + if left_token != right_token: + return False, work_units + return True, work_units + + +def _contains_sequence(haystack: tuple[str, ...], needle: tuple[str, ...]) -> tuple[bool, int]: + if len(needle) > len(haystack): + return False, 1 + work_units = 1 + for index in range(len(haystack) - len(needle) + 1): + matched = True + for offset, token in enumerate(needle): + work_units += 1 + if haystack[index + offset] != token: + matched = False + break + if matched: + return True, work_units + return False, work_units + + +def _exact_search_alias(query: str, skills: Mapping[str, Skill]) -> Skill | None: + raw_query = query.strip() + if raw_query.startswith("$"): + requested = raw_query[1:] + elif raw_query.casefold().startswith("/skill:"): + requested = raw_query[len("/skill:") :] + elif ":" in raw_query and not any(character.isspace() for character in raw_query): + requested = raw_query + else: + return None + if not requested or any(character.isspace() for character in requested): + return None + for candidate in _lookup_names(requested): + if skill := skills.get(normalize_skill_name(candidate)): + return skill + return None + + +def _match_sort_key(match: SkillMatch) -> tuple[int, int, int, str, str]: + skill = match.skill + return ( + int(match.tier), + -match.score, + _SCOPE_ORDER[skill.scope], + normalize_skill_name(skill.name), + str(skill.skill_md_file.canonical()), + ) + + +def _public_match(match: SkillMatch) -> SkillMatch: + return replace(match, skill=match.skill.model_copy(deep=True)) + + +def _public_view(view: SkillPromptView) -> SkillPromptView: + return replace(view, matches=tuple(_public_match(match) for match in view.matches)) + + +def _concise_description(description: str) -> str: + return " ".join(description.split()) + + +def _view( + matches: tuple[SkillMatch, ...], + total_count: int, + *, + overflowed_priority_count: int = 0, + descriptions: tuple[str | None, ...] = (), +) -> SkillPromptView: + return SkillPromptView( + matches=matches, + total_count=total_count, + omitted_count=max(0, total_count - len(matches)), + overflowed_priority_count=overflowed_priority_count, + rendered_characters=0, + descriptions=descriptions or (None,) * len(matches), + ) + + +def render_skill_prompt_view(view: SkillPromptView) -> str: + """Render a prompt view using safe metadata only.""" + lines = [ + ( + f"Task-relevant skills: {len(view.matches)} of {view.total_count} shown; " + f"{view.omitted_count} omitted." + ), + "Call ReadSkill with a complete skill name before applying it.", + ] + if view.overflowed_priority_count: + lines.append( + f"Priority candidates omitted by the hard cap: {view.overflowed_priority_count}." + ) + lines.append("Candidates:") + descriptions = view.descriptions or (None,) * len(view.matches) + for match, description in zip(view.matches, descriptions, strict=True): + line = f"- `{match.skill.name}` [{match.skill.scope}]" + if description: + line = f"{line}: {description}" + lines.append(line) + return "\n".join(lines) + + +def _lookup_names(name: str) -> tuple[str, ...]: + raw_name = name.strip() + if not raw_name: + return () + if ":" not in raw_name: + return (raw_name,) + suffix = raw_name.rsplit(":", 1)[-1].strip() + prefix = raw_name.split(":", 1)[0].strip() + return tuple(dict.fromkeys(candidate for candidate in (raw_name, suffix, prefix) if candidate)) + + +def _public_diagnostic(issue: SkillDiscoveryIssue, root: HostPath) -> SkillSourceDiagnostic: + return SkillSourceDiagnostic( + name=issue.name, + source_kind=SkillSourceKind(issue.source_kind), + source_id=str(issue.path.relative_to(root)) or ".", + scope=issue.scope, + category=SkillDiagnosticCategory.UNAVAILABLE, + reason_code=issue.reason_code, + safe_reason=issue.safe_reason, + ) diff --git a/src/pythinker_code/soul/agent.py b/src/pythinker_code/soul/agent.py index 6c1b8e3c..8f293b21 100644 --- a/src/pythinker_code/soul/agent.py +++ b/src/pythinker_code/soul/agent.py @@ -2,11 +2,12 @@ import asyncio import re -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import asdict, dataclass, field, replace from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Literal, cast import pydantic from jinja2 import FileSystemLoader, StrictUndefined, TemplateError, UndefinedError @@ -19,7 +20,7 @@ from pythinker_code.auth.oauth import OAuthManager from pythinker_code.background import BackgroundTaskManager from pythinker_code.config import Config -from pythinker_code.exception import MCPConfigError, SystemPromptTemplateError +from pythinker_code.exception import AgentSpecError, MCPConfigError, SystemPromptTemplateError from pythinker_code.llm import LLM from pythinker_code.lsp.service import LspService from pythinker_code.notifications import NotificationManager @@ -27,21 +28,23 @@ from pythinker_code.scratchpad import DEFAULT_SCRATCHPAD_SECTION from pythinker_code.session import Session from pythinker_code.skill import ( + ScopedSkillsRoot, Skill, - discover_skills_from_roots, - format_skills_for_prompt, - index_skills, + SkillCatalog, resolve_skills_roots, ) from pythinker_code.soul.approval import Approval, ApprovalState from pythinker_code.soul.denwarenji import DenwaRenji from pythinker_code.soul.message import system_reminder from pythinker_code.soul.toolset import PythinkerToolset, ToolType -from pythinker_code.subagents.discovery import ( - discover_markdown_agents, - materialize_markdown_agent_specs, - resolve_agent_roots, +from pythinker_code.subagents.catalogue import ( + ResolvedAgentCatalogue, + ResolvedAgentEntry, + UnknownFieldPolicy, + normalize_agent_name, + resolve_agent_catalogue, ) +from pythinker_code.subagents.discovery import resolve_agent_roots from pythinker_code.subagents.models import AgentTypeDefinition, ToolPolicy from pythinker_code.subagents.registry import LaborMarket from pythinker_code.subagents.store import SubagentStore @@ -87,6 +90,7 @@ class BuiltinSystemPromptArgs: _AGENTS_MD_MAX_BYTES = 32 * 1024 # 32 KiB +SKILL_PROMPT_MAX_CHARACTERS = 8_000 def _agents_md_fence(content: str) -> str: @@ -101,7 +105,7 @@ def render_agents_md_reminder(builtin_args: BuiltinSystemPromptArgs) -> str | No Returns ``None`` when no ``AGENTS.md`` applies between the project root and the working directory. Otherwise returns the framing + fenced merged content that is delivered as a session-start, user-role reminder prepended to every model request - (see :func:`pythinker_code.soul.pythinkersoul._with_agents_md_preamble`), rather than + (see :class:`pythinker_code.soul.request_assembly.RequestAssembler`), rather than baked into the immutable system prompt. Delivering it this way keeps the project instructions out of the system prompt while @@ -231,9 +235,12 @@ class Runtime: environment: Environment notifications: NotificationManager background_tasks: BackgroundTaskManager - skills: dict[str, Skill] + skill_catalog: SkillCatalog + skills: Mapping[str, Skill] additional_dirs: list[HostPath] skills_dirs: list[HostPath] + agent_catalogue: ResolvedAgentCatalogue | None = None + agent_type_projection: Mapping[str, AgentTypeDefinition] | None = None prompt_templates: dict[str, PromptTemplate] = field(default_factory=dict[str, PromptTemplate]) mcp_tools: dict[str, ToolType] = field(default_factory=dict[str, ToolType]) """Connected MCP tools, keyed `mcp____`, shared with subagent allowlists.""" @@ -298,19 +305,13 @@ async def create( Environment.detect(), ) - # Discover and format skills (grouped by scope for the system prompt). - scoped_roots = await resolve_skills_roots( - session.work_dir, - skills_dirs=skills_dirs, - merge_brands=config.merge_all_available_skills, - extra_skill_dirs=config.extra_skill_dirs or None, + skill_catalog, scoped_roots = await discover_runtime_skill_catalog( + session.work_dir, config, skills_dirs=skills_dirs ) # Canonicalize so symlinked skill directories match resolved paths skills_roots_canonical = [s.root.canonical() for s in scoped_roots] - skills = await discover_skills_from_roots(scoped_roots) - skills_by_name = index_skills(skills) - logger.info("Discovered {count} skill(s)", count=len(skills)) - skills_formatted = format_skills_for_prompt(skills) + skills_by_name = skill_catalog.exhaustive_mapping() + logger.info("Discovered {count} skill(s)", count=len(skills_by_name)) prompt_templates = await discover_prompt_templates(session.work_dir) logger.info("Discovered {count} prompt template(s)", count=len(prompt_templates)) @@ -402,7 +403,7 @@ def _on_approval_change() -> None: PYTHINKER_WORK_DIR_LS=ls_output, PYTHINKER_AGENTS_MD=agents_md or "", PYTHINKER_AGENTS_MD_FENCE=_agents_md_fence(agents_md or ""), - PYTHINKER_SKILLS=skills_formatted or "No skills found.", + PYTHINKER_SKILLS=format_skill_catalog_policy(skill_catalog), PYTHINKER_ADDITIONAL_DIRS_INFO=additional_dirs_info, PYTHINKER_OS=environment.os_kind, PYTHINKER_SHELL=f"{environment.shell_name} (`{environment.shell_path}`)", @@ -418,6 +419,7 @@ def _on_approval_change() -> None: config.background, notifications=notifications, ), + skill_catalog=skill_catalog, skills=skills_by_name, prompt_templates=prompt_templates, additional_dirs=additional_dirs, @@ -482,11 +484,14 @@ def copy_for_subagent( environment=self.environment, notifications=self.notifications, background_tasks=self.background_tasks.copy_for_role("subagent"), + skill_catalog=self.skill_catalog, skills=self.skills, prompt_templates=self.prompt_templates, # Share the same list reference so /add-dir mutations propagate to all agents additional_dirs=self.additional_dirs, skills_dirs=self.skills_dirs, + agent_catalogue=self.agent_catalogue, + agent_type_projection=self.agent_type_projection, # Share the parent's connected MCP tools so allowlisted subagents can attach them mcp_tools=self.mcp_tools, subagent_store=self.subagent_store, @@ -516,6 +521,113 @@ class Agent: top_p: float | None = None +def agent_type_definitions(runtime: Runtime) -> Mapping[str, AgentTypeDefinition]: + """Return catalogue-projected definitions in their compatibility insertion order.""" + projection = getattr(runtime, "agent_type_projection", None) + if isinstance(projection, Mapping): + return cast("Mapping[str, AgentTypeDefinition]", projection) + labor_market = getattr(runtime, "labor_market", None) + compatibility_types = getattr(labor_market, "builtin_types", {}) or {} + return compatibility_types + + +def get_agent_type_definition(runtime: Runtime, name: str) -> AgentTypeDefinition | None: + """Resolve an internal agent reader through the catalogue without changing LaborMarket.""" + if runtime.agent_catalogue is not None and runtime.agent_type_projection is not None: + entry = runtime.agent_catalogue.get(name) + if entry is not None: + return runtime.agent_type_projection.get(entry.name) + return runtime.agent_type_projection.get(name) + return runtime.labor_market.get_builtin_type(name) + + +def require_agent_type_definition(runtime: Runtime, name: str) -> AgentTypeDefinition: + type_def = get_agent_type_definition(runtime, name) + if type_def is None: + raise KeyError(f"Builtin subagent type not found: {name}") + return type_def + + +def _project_agent_entry(entry: ResolvedAgentEntry) -> AgentTypeDefinition: + if entry.legacy_agent_file is None: + raise AgentSpecError( + f"Agent catalogue entry {entry.name!r} has no compatibility launch file" + ) + launch_spec = entry.launch_spec + tool_policy = ( + ToolPolicy(mode="allowlist", tools=tuple(launch_spec.allowed_tools)) + if launch_spec.allowed_tools is not None + else ToolPolicy(mode="inherit") + ) + return AgentTypeDefinition( + name=entry.name, + description=entry.description, + agent_file=entry.legacy_agent_file, + when_to_use=launch_spec.when_to_use, + default_model=launch_spec.model, + tool_policy=tool_policy, + supports_background=entry.supports_background, + required_mcp_servers=entry.required_mcp_servers, + ) + + +def _catalogue_entries_in_compatibility_order( + catalogue: ResolvedAgentCatalogue, + declared_subagents: tuple[str, ...], +) -> tuple[ResolvedAgentEntry, ...]: + declared_normalized = {normalize_agent_name(name) for name in declared_subagents} + declared_entries = tuple(catalogue.require(name) for name in declared_subagents) + optional_entries = sorted( + (entry for entry in catalogue.values() if entry.normalized_name not in declared_normalized), + key=lambda entry: entry.name, + ) + return (*declared_entries, *optional_entries) + + +def _log_agent_catalogue_diagnostics(catalogue: ResolvedAgentCatalogue) -> None: + for diagnostic in catalogue.diagnostics: + logger.warning( + "Agent definition {severity}: {source_kind} {safe_path}; " + "reason={reason_code}; fields={field_path}", + severity=diagnostic.severity, + source_kind=diagnostic.source_kind, + safe_path=diagnostic.safe_path, + reason_code=diagnostic.reason_code, + field_path=diagnostic.field_path or "(none)", + ) + + +async def _publish_agent_catalogue( + agent_file: Path, + runtime: Runtime, + declared_subagents: tuple[str, ...], +) -> None: + if runtime.agent_catalogue is not None: + return + materialized_dir = runtime.session.dir / "external_agents" + catalogue = await resolve_agent_catalogue( + agent_file=agent_file, + markdown_roots=await resolve_agent_roots(runtime.work_dir), + materialized_dir=materialized_dir, + available_models=set(runtime.config.models), + unknown_field_policy=UnknownFieldPolicy.WARN, + ) + # The compatibility materializer historically created this directory even + # with no markdown sources; preserve the session layout during the rollout. + materialized_dir.mkdir(parents=True, exist_ok=True) + _log_agent_catalogue_diagnostics(catalogue) + resolved_projections = tuple( + _project_agent_entry(entry) + for entry in _catalogue_entries_in_compatibility_order(catalogue, declared_subagents) + ) + compatibility_projection = dict(runtime.labor_market.builtin_types) + compatibility_projection.update((type_def.name, type_def) for type_def in resolved_projections) + for type_def in resolved_projections: + runtime.labor_market.add_builtin_type(type_def) + runtime.agent_catalogue = catalogue + runtime.agent_type_projection = MappingProxyType(compatibility_projection) + + async def load_agent( agent_file: Path, runtime: Runtime, @@ -539,51 +651,16 @@ async def load_agent( logger.info("Loading agent: {agent_file}", agent_file=agent_file) agent_spec = load_agent_spec(agent_file) + # Resolve and publish the immutable definition catalogue exactly once, before + # any tool reads the compatibility LaborMarket projection. + await _publish_agent_catalogue(agent_file, runtime, tuple(agent_spec.subagents)) + system_prompt = _load_system_prompt( agent_spec.system_prompt_path, agent_spec.system_prompt_args, runtime.builtin_args, ) - # Register built-in subagent types before loading tools because some tools render - # descriptions from the labor market on initialization. - for subagent_name, subagent_spec in agent_spec.subagents.items(): - logger.debug( - "Registering builtin subagent type: {subagent_name}", subagent_name=subagent_name - ) - builtin_spec = load_agent_spec(subagent_spec.path) - tool_policy = ( - ToolPolicy(mode="allowlist", tools=tuple(builtin_spec.allowed_tools)) - if builtin_spec.allowed_tools is not None - else ToolPolicy(mode="inherit") - ) - runtime.labor_market.add_builtin_type( - AgentTypeDefinition( - name=subagent_name, - description=subagent_spec.description, - agent_file=subagent_spec.path, - when_to_use=builtin_spec.when_to_use, - default_model=builtin_spec.model, - tool_policy=tool_policy, - supports_background=not builtin_spec.hidden, - ) - ) - - external_agents = await discover_markdown_agents(await resolve_agent_roots(runtime.work_dir)) - for type_def in materialize_markdown_agent_specs( - external_agents, - output_dir=runtime.session.dir / "external_agents", - available_models=set(runtime.config.models), - ): - if runtime.labor_market.get_builtin_type(type_def.name) is not None: - logger.warning( - "Skipping external markdown agent {name}: would override a built-in subagent type", - name=type_def.name, - ) - continue - logger.debug("Registering external markdown agent type: {name}", name=type_def.name) - runtime.labor_market.add_builtin_type(type_def) - toolset = PythinkerToolset(runtime) # Wire the live MCP startup state so the subagent-spawn gate can reject an agent whose # required MCP servers are absent (root only — subagents never spawn other agents). @@ -726,19 +803,14 @@ async def build_builtin_system_prompt_args( load_agents_md(work_dir), Environment.detect(), ) - scoped_roots = await resolve_skills_roots( - work_dir, - merge_brands=config.merge_all_available_skills, - extra_skill_dirs=config.extra_skill_dirs or None, - ) - skills_formatted = format_skills_for_prompt(await discover_skills_from_roots(scoped_roots)) + skill_catalog, _ = await discover_runtime_skill_catalog(work_dir, config) return BuiltinSystemPromptArgs( PYTHINKER_NOW=datetime.now().astimezone().isoformat(), PYTHINKER_WORK_DIR=work_dir, PYTHINKER_WORK_DIR_LS=ls_output, PYTHINKER_AGENTS_MD=agents_md or "", PYTHINKER_AGENTS_MD_FENCE=_agents_md_fence(agents_md or ""), - PYTHINKER_SKILLS=skills_formatted or "No skills found.", + PYTHINKER_SKILLS=format_skill_catalog_policy(skill_catalog), PYTHINKER_ADDITIONAL_DIRS_INFO="", PYTHINKER_OS=environment.os_kind, PYTHINKER_SHELL=f"{environment.shell_name} (`{environment.shell_path}`)", @@ -746,6 +818,33 @@ async def build_builtin_system_prompt_args( ) +async def discover_runtime_skill_catalog( + work_dir: HostPath, + config: Config, + *, + skills_dirs: list[HostPath] | None = None, +) -> tuple[SkillCatalog, list[ScopedSkillsRoot]]: + """Construct the catalogue used by runtime creation and prompt inspection.""" + scoped_roots = await resolve_skills_roots( + work_dir, + skills_dirs=skills_dirs, + merge_brands=config.merge_all_available_skills, + extra_skill_dirs=config.extra_skill_dirs or None, + ) + return await SkillCatalog.discover(scoped_roots), scoped_roots + + +def format_skill_catalog_policy(catalog: SkillCatalog) -> str: + """Render stable catalogue metadata without task-dependent candidates.""" + count = len(catalog.exhaustive_mapping()) + return ( + "Task-relevant skill candidates arrive with each request. " + f"The catalogue contains {count} skill(s); candidate rendering is capped at " + f"{SKILL_PROMPT_MAX_CHARACTERS} characters. Exact names remain available through " + "ReadSkill even when omitted from a candidate view." + ) + + # Separates the system prompt from the appended AGENTS.md reminder in dump output, making # clear the reminder is delivered as its own message and is not part of the system prompt. _AGENTS_MD_DUMP_HEADER = ( diff --git a/src/pythinker_code/soul/btw.py b/src/pythinker_code/soul/btw.py index 6631db9b..4bce07c5 100644 --- a/src/pythinker_code/soul/btw.py +++ b/src/pythinker_code/soul/btw.py @@ -12,7 +12,7 @@ import uuid from collections.abc import Callable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol import pythinker_core from pythinker_core.message import Message, ToolCall @@ -27,10 +27,29 @@ if TYPE_CHECKING: from pythinker_core.chat_provider import StreamedMessagePart + from pythinker_code.soul.agent import Agent, Runtime + from pythinker_code.soul.context import Context from pythinker_code.soul.pythinkersoul import PythinkerSoul + from pythinker_code.soul.request_assembly import AssembledRequest _BTW_MAX_TURNS = 2 + +class _SideQuestionSoul(Protocol): + @property + def runtime(self) -> Runtime: ... + + @property + def agent(self) -> Agent: ... + + @property + def context(self) -> Context: ... + + async def assemble_side_request( + self, question: str, reminder_text: str + ) -> AssembledRequest: ... + + SIDE_QUESTION_SYSTEM_REMINDER = """\ This is a side question from the user. Answer directly in a single response. @@ -91,7 +110,7 @@ def handle(self, tool_call: ToolCall) -> ToolResult: def _build_btw_context( - soul: PythinkerSoul, + soul: _SideQuestionSoul, question: str, *, system_reminder_text: str = SIDE_QUESTION_SYSTEM_REMINDER, @@ -103,13 +122,13 @@ def _build_btw_context( ``system_reminder_text`` selects the framing (side question vs. max-steps handoff); both run tools-denied over the current history. """ - system_prompt = soul._agent.system_prompt # pyright: ignore[reportPrivateUsage] + system_prompt = soul.agent.system_prompt effective_history = normalize_history(soul.context.history) wrapped = f"{system_reminder(system_reminder_text).text}\n\n{question}" side_message = Message(role="user", content=wrapped) - toolset = _DenyAllToolset(soul._agent.toolset.tools) # pyright: ignore[reportPrivateUsage] + toolset = _DenyAllToolset(soul.agent.toolset.tools) return system_prompt, [*effective_history, side_message], toolset @@ -120,7 +139,7 @@ def _build_btw_context( async def execute_side_question( - soul: PythinkerSoul, + soul: _SideQuestionSoul, question: str, on_text_chunk: Callable[[str], None] | None = None, *, @@ -140,14 +159,20 @@ async def execute_side_question( Returns: (response_text, None) on success, (None, error_message) on failure. """ - if soul._runtime.llm is None: # pyright: ignore[reportPrivateUsage] + if soul.runtime.llm is None: return None, "LLM is not set." try: - chat_provider = soul._runtime.llm.chat_provider # pyright: ignore[reportPrivateUsage] - system_prompt, history, toolset = _build_btw_context( - soul, question, system_reminder_text=system_reminder_text - ) + chat_provider = soul.runtime.llm.chat_provider + if system_reminder_text == SIDE_QUESTION_SYSTEM_REMINDER: + assembled = await soul.assemble_side_request(question, system_reminder_text) + system_prompt = assembled.system_prompt + history = list(assembled.provider_history) + toolset = _DenyAllToolset(soul.agent.toolset.tools) + else: + system_prompt, history, toolset = _build_btw_context( + soul, question, system_reminder_text=system_reminder_text + ) text_chunks: list[str] = [] diff --git a/src/pythinker_code/soul/compaction_restore.py b/src/pythinker_code/soul/compaction_restore.py index 5d2c3098..83876e53 100644 --- a/src/pythinker_code/soul/compaction_restore.py +++ b/src/pythinker_code/soul/compaction_restore.py @@ -3,7 +3,7 @@ import json import os import re -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from typing import Any, cast @@ -67,7 +67,7 @@ async def build_compaction_restore_context( work_dir: HostPath, additional_dirs: Sequence[HostPath] = (), active_skill_names: Sequence[str] = (), - skills_by_name: dict[str, Skill] | None = None, + skills_by_name: Mapping[str, Skill] | None = None, max_files: int = MAX_RESTORED_FILES, ) -> CompactionRestoreContext: """Build post-compaction reminders for facts that summaries often drop. @@ -278,7 +278,7 @@ def _most_recent(paths: list[str], limit: int) -> list[str]: async def _restore_active_skills( active_skill_names: Sequence[str], - skills_by_name: dict[str, Skill], + skills_by_name: Mapping[str, Skill], ) -> tuple[str, tuple[str, ...]]: if not active_skill_names or not skills_by_name: return "", () diff --git a/src/pythinker_code/soul/context.py b/src/pythinker_code/soul/context.py index 16978531..0429d5d7 100644 --- a/src/pythinker_code/soul/context.py +++ b/src/pythinker_code/soul/context.py @@ -2,15 +2,16 @@ import asyncio import contextlib +import errno import json import os import tempfile -from collections.abc import Sequence +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass, replace from pathlib import Path from typing import Any, cast import aiofiles -import aiofiles.os from pydantic import ValidationError from pythinker_core.message import Message, TextPart @@ -20,11 +21,310 @@ from pythinker_code.utils.logging import logger from pythinker_code.utils.path import next_available_rotation +_MAX_REVERT_CONFLICT_ATTEMPTS = 3 + _LOST_RESULT_NOTE = ( "Tool call result was lost before it could be recorded (the session ended " "unexpectedly). Re-run the tool if its output is still needed." ) +_ContextRecord = Message | dict[str, object] + + +@dataclass(frozen=True, slots=True) +class ContextReplacement: + system_prompt: str | None + messages: tuple[Message, ...] + token_count: int + create_checkpoint: bool + checkpoint_user_marker: bool = False + checkpoint_positions: tuple[int, ...] = () + pending_message_count: int = 0 + persist_token_count: bool = True + repair_history: bool = False + + +@dataclass(frozen=True, slots=True) +class ContextCommit: + checkpoint_id: int | None + rotated_file: Path | None + message_count: int + generation: int + + +class ContextPersistenceError(OSError): + def __init__( + self, + operation: str, + category: str, + path: Path, + *, + commit: ContextCommit | None = None, + ) -> None: + self.operation = operation + self.category = category + self.path = path + self.commit = commit + safe_path = path.name or "context storage" + message = f"{operation} failed ({category}) for {safe_path}" + if category == "visible_commit_durability": + message += "; the new generation is visible but power-loss durability is uncertain" + super().__init__(message) + + +class ContextCommittedCancellation(asyncio.CancelledError): + def __init__(self, commit: ContextCommit) -> None: + self.commit = commit + super().__init__("context replacement committed before cancellation") + + +class ContextGenerationConflictError(RuntimeError): + """A semantic replacement was prepared from a stale Context generation.""" + + def __init__(self, expected_generation: int, actual_generation: int) -> None: + self.expected_generation = expected_generation + self.actual_generation = actual_generation + super().__init__( + "context generation changed while preparing history replacement " + f"(expected {expected_generation}, found {actual_generation})" + ) + + +@dataclass(frozen=True, slots=True) +class _ContextState: + history: tuple[Message, ...] + token_count: int + pending_messages: tuple[Message, ...] + pending_token_estimate: int + next_checkpoint_id: int + system_prompt: str | None + tail_repaired: bool + + +def _empty_context_state() -> _ContextState: + return _ContextState((), 0, (), 0, 0, None, False) + + +def _serialize_context_records(records: Sequence[_ContextRecord]) -> str: + serialized: list[str] = [] + for record in records: + if isinstance(record, Message): + serialized.append(record.model_dump_json(exclude_none=True)) + else: + serialized.append(json.dumps(record)) + return "".join(f"{record}\n" for record in serialized) + + +def _persistence_error(category: str, path: Path) -> ContextPersistenceError: + return ContextPersistenceError("replace_history", category, path) + + +def _cleanup_replacement_path(path: Path | None, primary_error: BaseException) -> None: + if path is None: + return + try: + path.unlink(missing_ok=True) + except OSError as cleanup_error: + logger.warning( + "Failed to clean context replacement artifact {path}: {error}", + path=path.name, + error=cleanup_error, + ) + primary_error.add_note(f"Cleanup also failed for {path.name}") + + +def _prepare_replacement_file(file_backend: Path, serialized_records: Sequence[str]) -> Path: + try: + fd, tmp_name = tempfile.mkstemp( + dir=file_backend.parent, + prefix=file_backend.name, + suffix=".tmp", + ) + except OSError as error: + raise _persistence_error("temporary_creation", file_backend) from error + + tmp_path = Path(tmp_name) + descriptor_owned = True + try: + try: + replacement_file = os.fdopen(fd, "w", encoding="utf-8") + descriptor_owned = False + except OSError as error: + raise _persistence_error("write", file_backend) from error + with replacement_file: + for record in serialized_records: + try: + replacement_file.write(record) + except OSError as error: + raise _persistence_error("write", file_backend) from error + try: + replacement_file.flush() + except OSError as error: + raise _persistence_error("write", file_backend) from error + try: + os.fsync(replacement_file.fileno()) + except OSError as error: + raise _persistence_error("synchronization", file_backend) from error + except BaseException as error: + primary_error = error + if isinstance(error, OSError) and not isinstance(error, ContextPersistenceError): + primary_error = _persistence_error("write", file_backend) + if descriptor_owned: + with contextlib.suppress(OSError): + os.close(fd) + _cleanup_replacement_path(tmp_path, primary_error) + if primary_error is not error: + raise primary_error from error + raise + return tmp_path + + +def _read_live_bytes(file_backend: Path) -> bytes | None: + if not file_backend.exists(): + return None + try: + return file_backend.read_bytes() + except OSError as error: + raise _persistence_error("rotation_archive", file_backend) from error + + +def _write_rotation_archive(rotated_file: Path, live_bytes: bytes) -> None: + try: + with rotated_file.open("wb") as archive_file: + archive_file.write(live_bytes) + archive_file.flush() + os.fsync(archive_file.fileno()) + except OSError as error: + raise _persistence_error("rotation_archive", rotated_file) from error + + +def _sync_parent_directory(parent: Path) -> bool: + if os.name != "posix": + return False + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + try: + directory_fd = os.open(parent, flags) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + except OSError as error: + unsupported = {errno.EINVAL, getattr(errno, "ENOTSUP", errno.EINVAL)} + if error.errno in unsupported: + return False + raise + return True + + +async def _settle_awaitable[T]( + operation: Awaitable[T], +) -> tuple[T, asyncio.CancelledError | None]: + task = asyncio.ensure_future(operation) + cancellation: asyncio.CancelledError | None = None + try: + await asyncio.shield(task) + except asyncio.CancelledError as error: + cancellation = error + while not task.done(): + with contextlib.suppress(asyncio.CancelledError): + await asyncio.wait({task}) + + try: + result = task.result() + except BaseException as operation_error: + if cancellation is not None: + raise cancellation from operation_error + raise + return result, cancellation + + +async def _settle_thread[T]( + operation: Callable[[], T], +) -> tuple[T, asyncio.CancelledError | None]: + return await _settle_awaitable(asyncio.to_thread(operation)) + + +async def _before_replacement_commit() -> None: + await asyncio.sleep(0) + + +def _rollback_context_append(file_backend: Path, existed: bool, original_size: int) -> None: + if not existed: + file_backend.unlink(missing_ok=True) + return + with file_backend.open("r+b") as rollback_file: + rollback_file.truncate(original_size) + rollback_file.flush() + + +def _append_context_sync(file_backend: Path, payload: str) -> None: + existed = file_backend.exists() + original_size = file_backend.stat().st_size if existed else 0 + try: + with file_backend.open("a", encoding="utf-8") as context_file: + context_file.write(payload) + context_file.flush() + except BaseException as append_error: + try: + _rollback_context_append(file_backend, existed, original_size) + except BaseException as rollback_error: + raise BaseExceptionGroup( + "Context append and rollback both failed", + (append_error, rollback_error), + ) from append_error + raise + + +def _write_system_prompt_sync(file_backend: Path, prompt_line: str) -> None: + fd, tmp_name = tempfile.mkstemp( + dir=file_backend.parent, + prefix=file_backend.name, + suffix=".tmp", + ) + tmp_path = Path(tmp_name) + fd_owned_by_file = False + try: + with os.fdopen(fd, "w", encoding="utf-8") as prompt_file: + fd_owned_by_file = True + prompt_file.write(prompt_line) + if file_backend.exists() and file_backend.stat().st_size > 0: + with file_backend.open(encoding="utf-8") as source_file: + while chunk := source_file.read(64 * 1024): + prompt_file.write(chunk) + prompt_file.flush() + tmp_path.replace(file_backend) + except BaseException as write_error: + if not fd_owned_by_file: + os.close(fd) + try: + tmp_path.unlink(missing_ok=True) + except OSError as cleanup_error: + raise BaseExceptionGroup( + "System prompt write and cleanup both failed", + (write_error, cleanup_error), + ) from write_error + raise + + +async def _settle_sync_commit(operation: Callable[[], None]) -> asyncio.CancelledError | None: + commit_task = asyncio.create_task(asyncio.to_thread(operation)) + cancellation: asyncio.CancelledError | None = None + try: + await asyncio.shield(commit_task) + except asyncio.CancelledError as error: + cancellation = error + while not commit_task.done(): + with contextlib.suppress(asyncio.CancelledError): + await asyncio.wait({commit_task}) + + try: + commit_task.result() + except BaseException as commit_error: + if cancellation is not None: + raise cancellation from commit_error + raise + return cancellation + def repair_history_invariants(history: Sequence[Message]) -> list[Message]: """Restore tool call/result pairing broken by a crash mid-persistence. @@ -78,66 +378,316 @@ def _synthesize_lost_results() -> None: return repaired +def _reduce_context_records( + state: _ContextState, + records: Sequence[Message | dict[str, Any]], + *, + file_backend: Path, + line_numbers: Sequence[int] | None = None, +) -> tuple[_ContextState, tuple[bool, ...]]: + history = list(state.history) + pending_messages = list(state.pending_messages) + pending_token_estimate = state.pending_token_estimate + pending_segment: list[Message] = [] + token_count = state.token_count + next_checkpoint_id = state.next_checkpoint_id + system_prompt = state.system_prompt + accepted: list[bool] = [] + + for index, record in enumerate(records): + line_no = line_numbers[index] if line_numbers is not None else 0 + if isinstance(record, Message): + history.append(record) + pending_messages.append(record) + pending_segment.append(record) + accepted.append(True) + continue + + role = record.get("role") + if not isinstance(role, str): + logger.warning( + "Skipping context line {line_no} in {file}: missing or invalid role", + line_no=line_no, + file=file_backend, + ) + accepted.append(False) + continue + if role == "_system_prompt": + content = record.get("content") + if not isinstance(content, str): + logger.warning( + "Skipping invalid system prompt line {line_no} in {file}", + line_no=line_no, + file=file_backend, + ) + accepted.append(False) + continue + system_prompt = content + accepted.append(True) + continue + if role == "_usage": + usage_token_count = record.get("token_count") + if ( + not isinstance(usage_token_count, int) + or isinstance(usage_token_count, bool) + or usage_token_count < 0 + ): + logger.warning( + "Skipping invalid usage line {line_no} in {file}", + line_no=line_no, + file=file_backend, + ) + accepted.append(False) + continue + token_count = usage_token_count + pending_messages.clear() + pending_token_estimate = 0 + pending_segment.clear() + accepted.append(True) + continue + if role == "_checkpoint": + checkpoint_id = record.get("id") + if ( + not isinstance(checkpoint_id, int) + or isinstance(checkpoint_id, bool) + or checkpoint_id < 0 + ): + logger.warning( + "Skipping invalid checkpoint line {line_no} in {file}", + line_no=line_no, + file=file_backend, + ) + accepted.append(False) + continue + next_checkpoint_id = checkpoint_id + 1 + accepted.append(True) + continue + try: + message = Message.model_validate(record) + except ValidationError as exc: + logger.warning( + "Skipping invalid context message line {line_no} in {file}: {error}", + line_no=line_no, + file=file_backend, + error=exc, + ) + accepted.append(False) + continue + history.append(message) + pending_messages.append(message) + pending_segment.append(message) + accepted.append(True) + + pending_token_estimate += estimate_text_tokens(pending_segment) + return ( + _ContextState( + history=tuple(history), + token_count=token_count, + pending_messages=tuple(pending_messages), + pending_token_estimate=pending_token_estimate, + next_checkpoint_id=next_checkpoint_id, + system_prompt=system_prompt, + tail_repaired=state.tail_repaired, + ), + tuple(accepted), + ) + + +def _repair_context_state(state: _ContextState) -> _ContextState: + history = tuple(repair_history_invariants(state.history)) + pending_messages = tuple(repair_history_invariants(state.pending_messages)) + return replace( + state, + history=history, + pending_messages=pending_messages, + pending_token_estimate=estimate_text_tokens(pending_messages), + ) + + +def _runtime_replacement_field(value: object) -> object: + return value + + +def _replacement_records(replacement: ContextReplacement) -> tuple[_ContextRecord, ...]: + system_prompt = _runtime_replacement_field(replacement.system_prompt) + messages = _runtime_replacement_field(replacement.messages) + token_count = _runtime_replacement_field(replacement.token_count) + create_checkpoint = _runtime_replacement_field(replacement.create_checkpoint) + checkpoint_user_marker = _runtime_replacement_field(replacement.checkpoint_user_marker) + checkpoint_positions = _runtime_replacement_field(replacement.checkpoint_positions) + pending_message_count = _runtime_replacement_field(replacement.pending_message_count) + persist_token_count = _runtime_replacement_field(replacement.persist_token_count) + repair_history = _runtime_replacement_field(replacement.repair_history) + + if system_prompt is not None and not isinstance(system_prompt, str): + raise TypeError("system_prompt must be a string or None") + if not isinstance(messages, tuple): + raise TypeError("messages must be a tuple containing only Message values") + message_values = cast(tuple[object, ...], messages) + if not all(isinstance(message, Message) for message in message_values): + raise TypeError("messages must be a tuple containing only Message values") + validated_messages = cast(tuple[Message, ...], message_values) + if type(token_count) is not int: + raise TypeError("token_count must be an integer") + if type(create_checkpoint) is not bool: + raise TypeError("create_checkpoint must be a boolean") + if type(checkpoint_user_marker) is not bool: + raise TypeError("checkpoint_user_marker must be a boolean") + if not isinstance(checkpoint_positions, tuple): + raise TypeError("checkpoint_positions must be a tuple of integers") + checkpoint_position_values = cast(tuple[object, ...], checkpoint_positions) + if not all(type(position) is int for position in checkpoint_position_values): + raise TypeError("checkpoint_positions must be a tuple of integers") + validated_checkpoint_positions = cast(tuple[int, ...], checkpoint_position_values) + if type(pending_message_count) is not int: + raise TypeError("pending_message_count must be an integer") + if type(persist_token_count) is not bool: + raise TypeError("persist_token_count must be a boolean") + if type(repair_history) is not bool: + raise TypeError("repair_history must be a boolean") + if token_count < 0: + raise ValueError("token_count must be a non-negative integer") + if checkpoint_user_marker and not create_checkpoint: + raise ValueError("checkpoint_user_marker requires create_checkpoint") + if create_checkpoint and validated_checkpoint_positions: + raise ValueError("create_checkpoint cannot be combined with checkpoint_positions") + if create_checkpoint and pending_message_count: + raise ValueError("create_checkpoint cannot retain pending messages") + if pending_message_count < 0 or pending_message_count > len(validated_messages): + raise ValueError("pending_message_count must identify a suffix of messages") + if any( + position < 0 or position > len(validated_messages) + for position in validated_checkpoint_positions + ): + raise ValueError("checkpoint_positions must refer to message boundaries") + if tuple(sorted(validated_checkpoint_positions)) != validated_checkpoint_positions: + raise ValueError("checkpoint_positions must be ordered") + + records: list[_ContextRecord] = [] + if system_prompt is not None: + records.append({"role": "_system_prompt", "content": system_prompt}) + if create_checkpoint: + records.append({"role": "_checkpoint", "id": 0}) + if checkpoint_user_marker: + records.append(Message(role="user", content=[system("CHECKPOINT 0")])) + records.extend(validated_messages) + if persist_token_count: + records.append({"role": "_usage", "token_count": token_count}) + return tuple(records) + + usage_position = len(validated_messages) - pending_message_count + checkpoint_index = 0 + for position in range(len(validated_messages) + 1): + if persist_token_count and position == usage_position: + records.append({"role": "_usage", "token_count": token_count}) + while ( + checkpoint_index < len(validated_checkpoint_positions) + and validated_checkpoint_positions[checkpoint_index] == position + ): + records.append({"role": "_checkpoint", "id": checkpoint_index}) + checkpoint_index += 1 + if position < len(validated_messages): + records.append(validated_messages[position]) + return tuple(records) + + +def _serialize_replacement_records( + file_backend: Path, + records: Sequence[_ContextRecord], +) -> tuple[str, ...]: + try: + return tuple(_serialize_context_records((record,)) for record in records) + except (TypeError, ValueError) as error: + raise _persistence_error("serialization", file_backend) from error + + class Context: def __init__(self, file_backend: Path): self._file_backend = file_backend self._history: list[Message] = [] self._token_count: int = 0 + self._pending_messages: tuple[Message, ...] = () self._pending_token_estimate: int = 0 self._next_checkpoint_id: int = 0 """The ID of the next checkpoint, starting from 0, incremented after each checkpoint.""" self._system_prompt: str | None = None self._tail_repaired: bool = False + self._mutation_lock = asyncio.Lock() + self._mutation_generation = 0 - def _tail_repair_prefix(self) -> str: + def _tail_repair_prefix(self, state: _ContextState) -> str: """One-time torn-line terminator for the append paths. A crash mid-append can leave an unterminated final line; without the repair the next record glues onto it and readers skip both lines. """ - if self._tail_repaired: + if state.tail_repaired: return "" - self._tail_repaired = True return "" if ends_with_newline(self._file_backend) else "\n" + def _state(self) -> _ContextState: + return _ContextState( + history=tuple(self._history), + token_count=self._token_count, + pending_messages=self._pending_messages, + pending_token_estimate=self._pending_token_estimate, + next_checkpoint_id=self._next_checkpoint_id, + system_prompt=self._system_prompt, + tail_repaired=self._tail_repaired, + ) + + def _swap_state(self, state: _ContextState) -> None: + self._history[:] = state.history + self._token_count = state.token_count + self._pending_messages = state.pending_messages + self._pending_token_estimate = state.pending_token_estimate + self._next_checkpoint_id = state.next_checkpoint_id + self._system_prompt = state.system_prompt + self._tail_repaired = state.tail_repaired + self._mutation_generation += 1 + async def restore(self) -> bool: - logger.debug("Restoring context from file: {file_backend}", file_backend=self._file_backend) - if self._history: - logger.error("The context storage is already modified") - raise RuntimeError("The context storage is already modified") - if not self._file_backend.exists(): - logger.debug("No context file found, skipping restoration") - return False - if self._file_backend.stat().st_size == 0: - logger.debug("Empty context file, skipping restoration") - return False + async with self._mutation_lock: + logger.debug( + "Restoring context from file: {file_backend}", file_backend=self._file_backend + ) + if self._history: + logger.error("The context storage is already modified") + raise RuntimeError("The context storage is already modified") + if not self._file_backend.exists(): + logger.debug("No context file found, skipping restoration") + return False + if self._file_backend.stat().st_size == 0: + logger.debug("Empty context file, skipping restoration") + return False - messages_after_last_usage: list[Message] = [] - async with aiofiles.open(self._file_backend, encoding="utf-8", errors="replace") as f: - line_no = 0 - async for line in f: - line_no += 1 - if not line.strip(): - continue - line_json = self._parse_context_line( - line, - file_backend=self._file_backend, - line_no=line_no, - ) - if line_json is None: - continue - self._apply_context_record( - line_json, - history=self._history, - messages_after_last_usage=messages_after_last_usage, - file_backend=self._file_backend, - line_no=line_no, - ) + state = _empty_context_state() + records: list[dict[str, Any]] = [] + line_numbers: list[int] = [] + async with aiofiles.open(self._file_backend, encoding="utf-8", errors="replace") as f: + line_no = 0 + async for line in f: + line_no += 1 + if not line.strip(): + continue + line_json = self._parse_context_line( + line, + file_backend=self._file_backend, + line_no=line_no, + ) + if line_json is None: + continue + records.append(line_json) + line_numbers.append(line_no) - self._history[:] = repair_history_invariants(self._history) - messages_after_last_usage[:] = repair_history_invariants(messages_after_last_usage) - self._pending_token_estimate = estimate_text_tokens(messages_after_last_usage) - return True + state, _ = _reduce_context_records( + state, + records, + file_backend=self._file_backend, + line_numbers=line_numbers, + ) + self._swap_state(_repair_context_state(state)) + return True @property def history(self) -> Sequence[Message]: @@ -163,6 +713,10 @@ def system_prompt(self) -> str | None: def file_backend(self) -> Path: return self._file_backend + @property + def mutation_generation(self) -> int: + return self._mutation_generation + async def write_system_prompt(self, prompt: str) -> None: """Write the system prompt as the first record of the context file. @@ -171,58 +725,194 @@ async def write_system_prompt(self, prompt: str) -> None: temporary file to avoid corruption on crash and avoid loading the entire file into memory. """ - prompt_line = json.dumps({"role": "_system_prompt", "content": prompt}) + "\n" - - def _write_system_prompt_sync() -> None: - if not self._file_backend.exists() or self._file_backend.stat().st_size == 0: - self._file_backend.write_text(prompt_line, encoding="utf-8") - return - - # Unique temp name (NOT a fixed .tmp suffix): two processes - # resuming the same session would otherwise interleave writes into - # the same temp file and os.replace the garbage into place. - fd, tmp_name = tempfile.mkstemp( - dir=self._file_backend.parent, prefix=self._file_backend.name, suffix=".tmp" + prompt_record: dict[str, object] = {"role": "_system_prompt", "content": prompt} + prompt_line = _serialize_context_records((prompt_record,)) + + async with self._mutation_lock: + state, _ = _reduce_context_records( + self._state(), + (prompt_record,), + file_backend=self._file_backend, + ) + cancellation = await _settle_sync_commit( + lambda: _write_system_prompt_sync(self._file_backend, prompt_line) ) - tmp_path = Path(tmp_name) + self._swap_state(state) + if cancellation is not None: + raise cancellation + + async def replace_history( + self, + replacement: ContextReplacement, + *, + expected_generation: int | None = None, + ) -> ContextCommit: + if expected_generation is not None and type(expected_generation) is not int: + raise TypeError("expected_generation must be an integer or None") + records = _replacement_records(replacement) + serialized_records = _serialize_replacement_records(self._file_backend, records) + + async with self._mutation_lock: + if expected_generation is not None and expected_generation != self._mutation_generation: + raise ContextGenerationConflictError( + expected_generation, + self._mutation_generation, + ) + next_state, accepted = _reduce_context_records( + _empty_context_state(), + records, + file_backend=self._file_backend, + ) + if not all(accepted): + raise ValueError("replacement contains an invalid context record") + if replacement.repair_history: + next_state = _repair_context_state(next_state) + next_state = replace(next_state, tail_repaired=False) + + temp_path, cancellation = await _settle_thread( + lambda: _prepare_replacement_file(self._file_backend, serialized_records) + ) + if cancellation is not None: + _cleanup_replacement_path(temp_path, cancellation) + raise cancellation + try: - with ( - os.fdopen(fd, "w", encoding="utf-8") as tmp_f, - self._file_backend.open(encoding="utf-8") as src_f, - ): - tmp_f.write(prompt_line) - while True: - chunk = src_f.read(64 * 1024) - if not chunk: - break - tmp_f.write(chunk) - tmp_path.replace(self._file_backend) - except BaseException: - with contextlib.suppress(OSError): - tmp_path.unlink() + live_bytes, cancellation = await _settle_thread( + lambda: _read_live_bytes(self._file_backend) + ) + except BaseException as error: + _cleanup_replacement_path(temp_path, error) raise + if cancellation is not None: + _cleanup_replacement_path(temp_path, cancellation) + raise cancellation + rotated_file: Path | None = None + if live_bytes is not None: + try: + rotated_file, cancellation = await _settle_awaitable( + next_available_rotation(self._file_backend) + ) + except asyncio.CancelledError as error: + _cleanup_replacement_path(temp_path, error) + raise + except Exception as reservation_error: + error = _persistence_error("rotation_archive", self._file_backend) + _cleanup_replacement_path(temp_path, error) + raise error from reservation_error + except BaseException as error: + _cleanup_replacement_path(temp_path, error) + raise + if rotated_file is None: + error = _persistence_error("rotation_archive", self._file_backend) + _cleanup_replacement_path(temp_path, error) + raise error + if cancellation is not None: + _cleanup_replacement_path(rotated_file, cancellation) + _cleanup_replacement_path(temp_path, cancellation) + raise cancellation + try: + _, cancellation = await _settle_thread( + lambda: _write_rotation_archive(rotated_file, live_bytes) + ) + except BaseException as error: + _cleanup_replacement_path(rotated_file, error) + _cleanup_replacement_path(temp_path, error) + raise + if cancellation is not None: + _cleanup_replacement_path(rotated_file, cancellation) + _cleanup_replacement_path(temp_path, cancellation) + raise cancellation - await asyncio.to_thread(_write_system_prompt_sync) + try: + await _before_replacement_commit() + except BaseException as error: + _cleanup_replacement_path(rotated_file, error) + _cleanup_replacement_path(temp_path, error) + raise - self._system_prompt = prompt + async def commit_visible_generation() -> None: + try: + await asyncio.to_thread(os.replace, temp_path, self._file_backend) + except OSError as error: + raise _persistence_error("atomic_replacement", self._file_backend) from error + self._swap_state(next_state) - async def checkpoint(self, add_user_message: bool): - checkpoint_id = self._next_checkpoint_id - self._next_checkpoint_id += 1 - logger.debug("Checkpointing, ID: {id}", id=checkpoint_id) + try: + _, cancellation = await _settle_awaitable(commit_visible_generation()) + except BaseException as error: + _cleanup_replacement_path(temp_path, error) + raise - async with aiofiles.open(self._file_backend, "a", encoding="utf-8") as f: - await f.write( - self._tail_repair_prefix() - + json.dumps({"role": "_checkpoint", "id": checkpoint_id}) - + "\n" + commit = ContextCommit( + checkpoint_id=0 if replacement.create_checkpoint else None, + rotated_file=rotated_file, + message_count=len(next_state.history), + generation=self._mutation_generation, ) - if add_user_message: - await self.append_message( - Message(role="user", content=[system(f"CHECKPOINT {checkpoint_id}")]) + + def synchronize_visible_generation() -> bool: + try: + return _sync_parent_directory(self._file_backend.parent) + except OSError as error: + raise _persistence_error( + "visible_commit_durability", self._file_backend + ) from error + + try: + _, sync_cancellation = await _settle_thread(synchronize_visible_generation) + except ContextPersistenceError as error: + error.commit = commit + if cancellation is not None: + raise ContextCommittedCancellation(commit) from error + raise + if cancellation is None: + cancellation = sync_cancellation + if cancellation is not None: + raise ContextCommittedCancellation(commit) from cancellation + + return commit + + async def _append_serialized( + self, + payload: str, + state: _ContextState, + next_state: _ContextState, + ) -> None: + append_payload = self._tail_repair_prefix(state) + payload + cancellation = await _settle_sync_commit( + lambda: _append_context_sync(self._file_backend, append_payload) + ) + self._swap_state(next_state) + if cancellation is not None: + raise cancellation + + async def checkpoint(self, add_user_message: bool) -> None: + async with self._mutation_lock: + state = self._state() + checkpoint_id = state.next_checkpoint_id + logger.debug("Checkpointing, ID: {id}", id=checkpoint_id) + checkpoint_record: dict[str, object] = { + "role": "_checkpoint", + "id": checkpoint_id, + } + records: tuple[_ContextRecord, ...] + if add_user_message: + records = ( + checkpoint_record, + Message(role="user", content=[system(f"CHECKPOINT {checkpoint_id}")]), + ) + else: + records = (checkpoint_record,) + payload = _serialize_context_records(records) + next_state, _ = _reduce_context_records( + state, + records, + file_backend=self._file_backend, ) + next_state = replace(next_state, tail_repaired=True) + await self._append_serialized(payload, state, next_state) - async def revert_to(self, checkpoint_id: int): + async def revert_to(self, checkpoint_id: int) -> ContextCommit: """ Revert the context to the specified checkpoint. After this, the specified checkpoint and all subsequent content will be @@ -234,64 +924,85 @@ async def revert_to(self, checkpoint_id: int): Raises: ValueError: When the checkpoint does not exist. RuntimeError: When no available rotation path is found. + ContextGenerationConflictError: When concurrent mutations exhaust the retry budget. """ logger.debug("Reverting checkpoint, ID: {id}", id=checkpoint_id) - if checkpoint_id >= self._next_checkpoint_id: - logger.error("Checkpoint {checkpoint_id} does not exist", checkpoint_id=checkpoint_id) - raise ValueError(f"Checkpoint {checkpoint_id} does not exist") - - # rotate the context file - rotated_file_path = await next_available_rotation(self._file_backend) - if rotated_file_path is None: - logger.error("No available rotation path found") - raise RuntimeError("No available rotation path found") - await aiofiles.os.replace(self._file_backend, rotated_file_path) - logger.debug( - "Rotated context file: {rotated_file_path}", rotated_file_path=rotated_file_path - ) + for attempt in range(_MAX_REVERT_CONFLICT_ATTEMPTS): + if checkpoint_id >= self._next_checkpoint_id: + logger.error( + "Checkpoint {checkpoint_id} does not exist", checkpoint_id=checkpoint_id + ) + raise ValueError(f"Checkpoint {checkpoint_id} does not exist") - # restore the context until the specified checkpoint - self._history.clear() - self._token_count = 0 - self._next_checkpoint_id = 0 - self._system_prompt = None - messages_after_last_usage: list[Message] = [] - async with ( - aiofiles.open(rotated_file_path, encoding="utf-8", errors="replace") as old_file, - aiofiles.open(self._file_backend, "w", encoding="utf-8") as new_file, - ): - line_no = 0 - async for line in old_file: - line_no += 1 + source_generation = self._mutation_generation + source_bytes = await asyncio.to_thread(self._file_backend.read_bytes) + records: list[dict[str, Any]] = [] + line_numbers: list[int] = [] + found_checkpoint = False + source_text = source_bytes.decode(encoding="utf-8", errors="replace") + for line_no, line in enumerate(source_text.splitlines(), 1): if not line.strip(): continue line_json = self._parse_context_line( line, - file_backend=rotated_file_path, + file_backend=self._file_backend, line_no=line_no, ) if line_json is None: continue if line_json.get("role") == "_checkpoint" and line_json.get("id") == checkpoint_id: + found_checkpoint = True break + records.append(line_json) + line_numbers.append(line_no) + if not found_checkpoint: + raise ValueError(f"Checkpoint {checkpoint_id} does not exist") - keep_line = self._apply_context_record( - line_json, - history=self._history, - messages_after_last_usage=messages_after_last_usage, - file_backend=rotated_file_path, - line_no=line_no, + target_state, accepted = _reduce_context_records( + _empty_context_state(), + records, + file_backend=self._file_backend, + line_numbers=line_numbers, + ) + checkpoint_positions: list[int] = [] + message_count = 0 + persist_token_count = False + for record, keep in zip(records, accepted, strict=True): + if not keep: + continue + role = record.get("role") + if role == "_checkpoint": + checkpoint_positions.append(message_count) + elif role == "_usage": + persist_token_count = True + elif role not in {"_system_prompt", "_usage"}: + message_count += 1 + + try: + commit = await self.replace_history( + ContextReplacement( + system_prompt=target_state.system_prompt, + messages=target_state.history, + token_count=target_state.token_count, + create_checkpoint=False, + checkpoint_positions=tuple(checkpoint_positions), + pending_message_count=len(target_state.pending_messages), + persist_token_count=persist_token_count, + repair_history=True, + ), + expected_generation=source_generation, ) - if keep_line: - await new_file.write(line) + except ContextGenerationConflictError: + if attempt + 1 == _MAX_REVERT_CONFLICT_ATTEMPTS: + raise + continue + return commit - self._history[:] = repair_history_invariants(self._history) - messages_after_last_usage[:] = repair_history_invariants(messages_after_last_usage) - self._pending_token_estimate = estimate_text_tokens(messages_after_last_usage) + raise AssertionError("revert conflict retry loop exhausted without an outcome") - async def clear(self): + async def clear(self, system_prompt: str | None = None) -> ContextCommit: """ Clear the context history. This is almost equivalent to revert_to(0), but without relying on the assumption @@ -303,46 +1014,51 @@ async def clear(self): """ logger.debug("Clearing context") - - # rotate the context file - rotated_file_path = await next_available_rotation(self._file_backend) - if rotated_file_path is None: - logger.error("No available rotation path found") - raise RuntimeError("No available rotation path found") - await aiofiles.os.replace(self._file_backend, rotated_file_path) - self._file_backend.touch() - logger.debug( - "Rotated context file: {rotated_file_path}", rotated_file_path=rotated_file_path + return await self.replace_history( + ContextReplacement( + system_prompt=system_prompt, + messages=(), + token_count=0, + create_checkpoint=False, + persist_token_count=False, + ) ) - self._history.clear() - self._token_count = 0 - self._pending_token_estimate = 0 - self._next_checkpoint_id = 0 - self._system_prompt = None + async def append_message(self, message: Message | Sequence[Message]) -> None: + messages = (message,) if isinstance(message, Message) else message + await self.append_messages(messages) - async def append_message(self, message: Message | Sequence[Message]): - logger.debug("Appending message(s) to context: {message}", message=message) - messages = [message] if isinstance(message, Message) else message - self._history.extend(messages) - self._pending_token_estimate += estimate_text_tokens(messages) - - async with aiofiles.open(self._file_backend, "a", encoding="utf-8") as f: - await f.write(self._tail_repair_prefix()) - for message in messages: - await f.write(message.model_dump_json(exclude_none=True) + "\n") + async def append_messages(self, messages: Sequence[Message]) -> None: + logger.debug("Appending messages to context: {messages}", messages=messages) + message_batch = tuple(messages) + payload = _serialize_context_records(message_batch) + async with self._mutation_lock: + state = self._state() + next_state, _ = _reduce_context_records( + state, + message_batch, + file_backend=self._file_backend, + ) + next_state = replace(next_state, tail_repaired=True) + await self._append_serialized(payload, state, next_state) - async def update_token_count(self, token_count: int): + async def update_token_count(self, token_count: int) -> None: logger.debug("Updating token count in context: {token_count}", token_count=token_count) - self._token_count = token_count - self._pending_token_estimate = 0 - - async with aiofiles.open(self._file_backend, "a", encoding="utf-8") as f: - await f.write( - self._tail_repair_prefix() - + json.dumps({"role": "_usage", "token_count": token_count}) - + "\n" + if type(token_count) is not int: + raise TypeError("token_count must be an integer") + if token_count < 0: + raise ValueError("token_count must be a non-negative integer") + usage_record: dict[str, object] = {"role": "_usage", "token_count": token_count} + payload = _serialize_context_records((usage_record,)) + async with self._mutation_lock: + state = self._state() + next_state, _ = _reduce_context_records( + state, + (usage_record,), + file_backend=self._file_backend, ) + next_state = replace(next_state, tail_repaired=True) + await self._append_serialized(payload, state, next_state) def _parse_context_line( self, @@ -369,68 +1085,3 @@ def _parse_context_line( ) return None return cast(dict[str, Any], line_json) - - def _apply_context_record( - self, - line_json: dict[str, Any], - *, - history: list[Message], - messages_after_last_usage: list[Message], - file_backend: Path, - line_no: int, - ) -> bool: - role = line_json.get("role") - if not isinstance(role, str): - logger.warning( - "Skipping context line {line_no} in {file}: missing or invalid role", - line_no=line_no, - file=file_backend, - ) - return False - if role == "_system_prompt": - content = line_json.get("content") - if not isinstance(content, str): - logger.warning( - "Skipping invalid system prompt line {line_no} in {file}", - line_no=line_no, - file=file_backend, - ) - return False - self._system_prompt = content - return True - if role == "_usage": - token_count = line_json.get("token_count") - if not isinstance(token_count, int): - logger.warning( - "Skipping invalid usage line {line_no} in {file}", - line_no=line_no, - file=file_backend, - ) - return False - self._token_count = token_count - messages_after_last_usage.clear() - return True - if role == "_checkpoint": - checkpoint_id = line_json.get("id") - if not isinstance(checkpoint_id, int): - logger.warning( - "Skipping invalid checkpoint line {line_no} in {file}", - line_no=line_no, - file=file_backend, - ) - return False - self._next_checkpoint_id = checkpoint_id + 1 - return True - try: - message = Message.model_validate(line_json) - except ValidationError as exc: - logger.warning( - "Skipping invalid context message line {line_no} in {file}: {error}", - line_no=line_no, - file=file_backend, - error=exc, - ) - return False - history.append(message) - messages_after_last_usage.append(message) - return True diff --git a/src/pythinker_code/soul/dynamic_injection.py b/src/pythinker_code/soul/dynamic_injection.py index 8b65aebd..a02c2ab8 100644 --- a/src/pythinker_code/soul/dynamic_injection.py +++ b/src/pythinker_code/soul/dynamic_injection.py @@ -7,7 +7,21 @@ from pythinker_core.message import Message -from pythinker_code.notifications import is_notification_message +from pythinker_code.soul.request_assembly import ( + FragmentBudgetClass, + FragmentPersistence, + FragmentRequirement, + FragmentStatus, + FragmentTruncation, + RequestFragment, + RequestSourceResult, + SourceApplicability, + SourceResultStatus, + TrustedSourcePolicy, + admit_source_results, +) +from pythinker_code.soul.request_primitives import estimate_injection_tokens +from pythinker_code.soul.request_primitives import normalize_history as normalize_history if TYPE_CHECKING: from pythinker_code.soul.agent import Runtime @@ -22,6 +36,22 @@ class DynamicInjection: content: str # text content (will be wrapped in tags) +@dataclass(frozen=True, slots=True) +class PreparedInjection: + """Provider-owned stable identity paired with one dynamic injection.""" + + identity: str + injection: DynamicInjection + + @property + def type(self) -> str: + return self.injection.type + + @property + def content(self) -> str: + return self.injection.content + + @dataclass(frozen=True, slots=True) class InjectionCandidate: """Budgetable dynamic prompt content candidate.""" @@ -47,11 +77,6 @@ def injection_budget_tokens(self) -> int: return max(0, min(self.injection_ceiling_tokens, available)) -def estimate_injection_tokens(text: str) -> int: - """Estimate dynamic-injection tokens using the project-wide len/4 heuristic.""" - return max(1, len(text) // 4) - - def injection_budget_from_runtime(runtime: Runtime) -> ContextBudget: """Build a dynamic-injection budget from runtime model/config values.""" llm = getattr(runtime, "llm", None) @@ -78,51 +103,61 @@ def collect_within_budget( Oversize candidates are truncated at a line boundary when possible; otherwise they are dropped if no useful prefix fits. The input order is the tie-breaker for equal priorities. """ - if budget_tokens <= 0: - return [] - ordered = sorted(enumerate(candidates), key=lambda item: (-item[1].priority, item[0])) - out: list[InjectionCandidate] = [] - used = 0 - truncation_used = False - for _index, candidate in ordered: - estimate = candidate.token_estimate or estimate_injection_tokens(candidate.content) - if estimate <= 0: - continue - if used + estimate <= budget_tokens: - out.append(replace(candidate, token_estimate=estimate)) - used += estimate - continue - # Whole-fit failed. Truncate at most once per call, for the first - # (highest-priority) candidate that didn't whole-fit. Lower-priority - # candidates further down the loop may still whole-fit in remaining - # budget; don't break — keep scanning. - if truncation_used: - continue - truncation_used = True - remaining = budget_tokens - used - if remaining <= 0: - continue - truncated = _truncate_to_tokens(candidate.content, remaining) - if not truncated: - continue - truncated_estimate = estimate_injection_tokens(truncated) - if truncated_estimate <= 0 or used + truncated_estimate > budget_tokens: - continue - out.append(replace(candidate, content=truncated, token_estimate=truncated_estimate)) - used += truncated_estimate - return out - - -def _truncate_to_tokens(text: str, budget_tokens: int) -> str: - max_chars = max(0, budget_tokens * 4) - if max_chars <= 1: - return "" - truncated = text[: max_chars - 1].rstrip() - if "\n" in truncated: - truncated = truncated.rsplit("\n", 1)[0].rstrip() - if not truncated: - return "" - return f"{truncated}\n…" + policies = tuple( + _legacy_source_policy(index, candidate) for index, candidate in enumerate(candidates) + ) + source_results = tuple( + _legacy_source_result(policy, candidate) + for policy, candidate in zip(policies, candidates, strict=True) + ) + admissions = admit_source_results(policies, source_results, max(0, budget_tokens)) + candidates_by_key = { + f"legacy_{index:012d}": candidate for index, candidate in enumerate(candidates) + } + return [ + replace( + candidates_by_key[admission.fragment.key], + content=admission.fragment.content, + token_estimate=admission.outcome.admitted_tokens, + ) + for admission in admissions + if admission.fragment is not None + and admission.outcome.status in {FragmentStatus.INCLUDED, FragmentStatus.TRUNCATED} + ] + + +def _legacy_source_policy(index: int, candidate: InjectionCandidate) -> TrustedSourcePolicy: + return TrustedSourcePolicy( + source="legacy_dynamic_injection", + key=f"legacy_{index:012d}", + requirement=FragmentRequirement.BEST_EFFORT, + persistence=FragmentPersistence.HISTORY, + priority=candidate.priority, + budget_class=FragmentBudgetClass.BUDGETED, + truncation=FragmentTruncation.ALLOWED, + applicability=SourceApplicability.ALWAYS, + failure_reason_codes=(), + ) + + +def _legacy_source_result( + policy: TrustedSourcePolicy, candidate: InjectionCandidate +) -> RequestSourceResult: + return RequestSourceResult( + source=policy.source, + key=policy.key, + status=SourceResultStatus.PROVIDED, + fragment=RequestFragment( + key=policy.key, + content=candidate.content, + source=policy.source, + requirement=policy.requirement, + persistence=policy.persistence, + priority=policy.priority, + truncatable=policy.truncation is FragmentTruncation.ALLOWED, + ), + reason_code=None, + ) def dynamic_to_candidate(injection: DynamicInjection, *, priority: int = 100) -> InjectionCandidate: @@ -143,6 +178,8 @@ class DynamicInjectionProvider(ABC): (context_usage, runtime, config, etc.). """ + _prepared_injections: tuple[PreparedInjection, ...] = () + @abstractmethod async def get_injections( self, @@ -150,6 +187,38 @@ async def get_injections( soul: PythinkerSoul, ) -> list[DynamicInjection]: ... + async def prepare_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[PreparedInjection]: + """Return retry-stable injections without acknowledging one-shot state.""" + pending = self._prepared_injections + if pending: + return list(pending) + injections = await self.get_injections(history, soul) + if injections: + self._prepared_injections = tuple( + PreparedInjection(self.injection_identity(injection, index), injection) + for index, injection in enumerate(injections) + ) + return list(self._prepared_injections) + + def injection_identity(self, injection: DynamicInjection, index: int) -> str: + """Return the stable identity for one prepared result.""" + return injection.type if index == 0 else f"{injection.type}:{index:04d}" + + def acknowledge_injections(self, keys: Sequence[str]) -> None: + """Acknowledge prepared injections after their history append commits.""" + pending = self._prepared_injections + acknowledged = tuple(item for item in pending if item.identity in keys) + self._prepared_injections = tuple(item for item in pending if item.identity not in keys) + if acknowledged: + self._on_injections_acknowledged(tuple(item.injection for item in acknowledged)) + + def _on_injections_acknowledged(self, injections: Sequence[DynamicInjection]) -> None: + _ = injections + async def on_context_compacted(self) -> None: """Called after the context is compacted (history is rebuilt). @@ -180,32 +249,3 @@ def rearm(self, key: str) -> bool: """ _ = key return False - - -def normalize_history(history: Sequence[Message]) -> list[Message]: - """Merge adjacent user messages to produce a clean API input sequence. - - Dynamic injections are stored as standalone user messages in history; - normalization merges them into the adjacent user message. - - Only ``user`` role messages are merged. Assistant and tool messages - are never merged because their ``tool_calls`` / ``tool_call_id`` - fields form linked pairs that must stay intact. - """ - if not history: - return [] - - result: list[Message] = [] - for msg in history: - if ( - result - and result[-1].role == msg.role - and msg.role == "user" - and not is_notification_message(result[-1]) - and not is_notification_message(msg) - ): - merged_content = list(result[-1].content) + list(msg.content) - result[-1] = Message(role="user", content=merged_content) - else: - result.append(msg) - return result diff --git a/src/pythinker_code/soul/dynamic_injections/agent_list.py b/src/pythinker_code/soul/dynamic_injections/agent_list.py index 7605bbc2..0250501f 100644 --- a/src/pythinker_code/soul/dynamic_injections/agent_list.py +++ b/src/pythinker_code/soul/dynamic_injections/agent_list.py @@ -6,6 +6,7 @@ from pythinker_core.message import Message from pythinker_code.soul import wire_send +from pythinker_code.soul.agent import agent_type_definitions from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider from pythinker_code.subagents.models import AgentTypeDefinition from pythinker_code.utils.logging import logger @@ -48,7 +49,7 @@ async def get_injections( return [] del history agents = sorted( - soul.runtime.labor_market.builtin_types.values(), + agent_type_definitions(soul.runtime).values(), key=lambda item: item.name, ) lines = tuple(format_agent_line(agent) for agent in agents) diff --git a/src/pythinker_code/soul/dynamic_injections/model_defense.py b/src/pythinker_code/soul/dynamic_injections/model_defense.py index 1b774dfd..aabaa5c9 100644 --- a/src/pythinker_code/soul/dynamic_injections/model_defense.py +++ b/src/pythinker_code/soul/dynamic_injections/model_defense.py @@ -21,7 +21,11 @@ from pythinker_core.message import Message -from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider +from pythinker_code.soul.dynamic_injection import ( + DynamicInjection, + DynamicInjectionProvider, + PreparedInjection, +) if TYPE_CHECKING: from pythinker_code.soul.pythinkersoul import PythinkerSoul @@ -73,27 +77,53 @@ class ModelDefenseInjectionProvider(DynamicInjectionProvider): def __init__(self, fragments: Sequence[ModelDefenseFragment] = MODEL_DEFENSE_FRAGMENTS) -> None: self._fragments = tuple(fragments) - # Single-shot guard. Safe without a lock: the soul drives injection providers - # sequentially and there is no ``await`` between the check and the set in - # ``get_injections``, so the read-modify-write cannot interleave. Add a lock - # only if a provider is ever driven from multiple OS threads. + # Legacy direct-call one-shot guard. Request assembly serializes preparation + # per provider in RequestLifecycle, including concurrent main and /btw requests. self._injected = False async def get_injections( self, history: Sequence[Message], soul: PythinkerSoul, + ) -> list[DynamicInjection]: + injections = self._candidate_injections(history, soul) + if injections: + self._injected = True + return injections + + async def prepare_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[PreparedInjection]: + if self._prepared_injections: + return list(self._prepared_injections) + injections = self._matching_injections(history, soul) + if injections: + self._prepared_injections = tuple( + PreparedInjection(injection.type, injection) for injection in injections + ) + return list(self._prepared_injections) + + def _candidate_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, ) -> list[DynamicInjection]: _ = history - if self._injected: - return [] - model_name = soul.model_name - if not model_name: + if self._injected or not soul.model_name: return [] - matched = [fragment for fragment in self._fragments if fragment.matches(model_name)] - if not matched: + return self._matching_injections(history, soul) + + def _matching_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + _ = history + if not soul.model_name: return [] - self._injected = True + matched = [fragment for fragment in self._fragments if fragment.matches(soul.model_name)] return [ DynamicInjection( type=f"{_MODEL_DEFENSE_TYPE}:{fragment.name}", content=fragment.content @@ -101,7 +131,12 @@ async def get_injections( for fragment in matched ] + def _on_injections_acknowledged(self, injections: Sequence[DynamicInjection]) -> None: + if injections: + self._injected = True + async def on_context_compacted(self) -> None: # Compaction rewrites history; the prior defense reminder may have been # summarized away, so re-arm for the next step. self._injected = False + self._prepared_injections = () diff --git a/src/pythinker_code/soul/dynamic_injections/permissions_state.py b/src/pythinker_code/soul/dynamic_injections/permissions_state.py index b3e7e2c4..7bbb7dd1 100644 --- a/src/pythinker_code/soul/dynamic_injections/permissions_state.py +++ b/src/pythinker_code/soul/dynamic_injections/permissions_state.py @@ -1,11 +1,16 @@ from __future__ import annotations +import hashlib from collections.abc import Sequence from typing import TYPE_CHECKING from pythinker_core.message import Message -from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider +from pythinker_code.soul.dynamic_injection import ( + DynamicInjection, + DynamicInjectionProvider, + PreparedInjection, +) from pythinker_code.soul.permission import PermissionProfile, permission_profile_for_runtime if TYPE_CHECKING: @@ -22,19 +27,40 @@ class PermissionsInjectionProvider(DynamicInjectionProvider): which discovered policy through denied tool calls. Re-injects only when the posture fingerprint changes (covers /yolo, /auto, /trust toggles and new session approvals), after compaction, and after auto-mode toggles. - Root-only: subagent overlays already document their profile constraints. + Request assembly treats this source as required for every agent role. The legacy + direct-provider API remains root-only for compatibility. """ def __init__(self) -> None: self._last_fingerprint: tuple[object, ...] | None = None + self._prepared_fingerprint: tuple[object, ...] | None = None async def get_injections( self, history: Sequence[Message], soul: PythinkerSoul, ) -> list[DynamicInjection]: - if soul.is_subagent: + injection, fingerprint = self._candidate(soul) + if injection is None: return [] + self._last_fingerprint = fingerprint + return [injection] + + async def prepare_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[PreparedInjection]: + _ = history + if self._prepared_injections: + return list(self._prepared_injections) + injection, fingerprint = self._current_posture(soul) + self._prepared_fingerprint = fingerprint + digest = hashlib.sha256(repr(fingerprint).encode(encoding="utf-8")).hexdigest()[:16] + self._prepared_injections = (PreparedInjection(f"permissions:{digest}", injection),) + return list(self._prepared_injections) + + def _current_posture(self, soul: PythinkerSoul) -> tuple[DynamicInjection, tuple[object, ...]]: profile = permission_profile_for_runtime(soul.runtime) approval = soul.runtime.approval approved = tuple(sorted(approval.session_approved_actions())) @@ -46,10 +72,7 @@ async def get_injections( approval.is_safe_mode(), approved, ) - if fingerprint == self._last_fingerprint: - return [] - self._last_fingerprint = fingerprint - return [ + return ( DynamicInjection( type=_INJECTION_TYPE, content=_render( @@ -59,15 +82,43 @@ async def get_injections( approval.is_safe_mode(), approved, ), - ) - ] + ), + fingerprint, + ) + + def _candidate( + self, soul: PythinkerSoul + ) -> tuple[DynamicInjection | None, tuple[object, ...] | None]: + if soul.is_subagent: + return None, None + injection, fingerprint = self._current_posture(soul) + if fingerprint == self._last_fingerprint: + return None, fingerprint + return injection, fingerprint + + def _on_injections_acknowledged(self, injections: Sequence[DynamicInjection]) -> None: + if injections: + self._last_fingerprint = self._prepared_fingerprint + self._prepared_fingerprint = None async def on_context_compacted(self) -> None: self._last_fingerprint = None + self._prepared_fingerprint = None + self._prepared_injections = () async def on_auto_changed(self, enabled: bool) -> None: _ = enabled self._last_fingerprint = None + self._prepared_fingerprint = None + self._prepared_injections = () + + def rearm(self, key: str) -> bool: + if key != _INJECTION_TYPE: + return False + self._last_fingerprint = None + self._prepared_fingerprint = None + self._prepared_injections = () + return True def _render( diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index 4d05c41f..7b5c6907 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -41,10 +41,16 @@ NotificationView, build_notification_message, extract_notification_ids, + is_notification_message, ) from pythinker_code.prompt_templates import PromptTemplate, expand_prompt_template from pythinker_code.prompts import BUDGET_CONTINUATION_NUDGE from pythinker_code.skill import Skill, read_skill_text_with_local_specialization +from pythinker_code.skill.catalog import ( + SkillProjectionOutcome, + SkillProjectionStatus, + render_skill_prompt_view, +) from pythinker_code.soul import ( LLMNotSet, LLMNotSupported, @@ -54,8 +60,8 @@ wire_send, ) from pythinker_code.soul.agent import ( + SKILL_PROMPT_MAX_CHARACTERS, Agent, - BuiltinSystemPromptArgs, Runtime, render_agents_md_reminder, ) @@ -78,14 +84,16 @@ build_hook_context_message, compact_summary_text, ) -from pythinker_code.soul.context import Context +from pythinker_code.soul.context import ( + Context, + ContextCommit, + ContextCommittedCancellation, + ContextPersistenceError, + ContextReplacement, +) from pythinker_code.soul.dynamic_injection import ( - DynamicInjection, DynamicInjectionProvider, - collect_within_budget, - dynamic_to_candidate, injection_budget_from_runtime, - normalize_history, ) from pythinker_code.soul.dynamic_injections.active_skills import ActiveSkillInjectionProvider from pythinker_code.soul.dynamic_injections.agent_list import AgentListInjectionProvider @@ -102,6 +110,7 @@ from pythinker_code.soul.live_tokens import add_total_output_tokens from pythinker_code.soul.message import ( check_message, + is_system_reminder_message, system, system_reminder, tool_result_to_message, @@ -111,6 +120,30 @@ reset_step_permission_profile, set_step_permission_profile, ) +from pythinker_code.soul.request_assembly import ( + AGENTS_MD_SOURCE_POLICY, + AssembledRequest, + FragmentBudgetClass, + FragmentPersistence, + FragmentRequirement, + FragmentTruncation, + RequestAssembler, + RequestAssemblyError, + RequestAssemblyInput, + RequestFragment, + RequestManifest, + RequestSourceResult, + SourceApplicability, + SourceResultStatus, + TrustedSourcePolicy, +) +from pythinker_code.soul.request_lifecycle import ( + PreparedSources, + RequestLifecycle, + RequestLifecycleError, + SourceAcknowledgement, + failed_manifest, +) from pythinker_code.soul.slash import registry as soul_slash_registry from pythinker_code.soul.toolset import PythinkerToolset from pythinker_code.subagents.usage import accumulate_usage, estimate_cost_usd @@ -154,6 +187,7 @@ def type_check(soul: PythinkerSoul): SKILL_COMMAND_PREFIX = "skill:" +_EXPLICIT_SKILL_RE = re.compile(r"(?:\$|/skill:)([\w.:-]+)", re.IGNORECASE) def _safe_cwd(fallback: str) -> str: @@ -168,6 +202,24 @@ def _safe_cwd(fallback: str) -> str: return str(fallback) +def _explicit_skill_names(task: str) -> tuple[str, ...]: + """Return explicit skill mentions in left-to-right message order.""" + return tuple(match.group(1) for match in _EXPLICIT_SKILL_RE.finditer(task)) + + +def _latest_real_user_text(history: Sequence[Message]) -> str | None: + """Return the latest user task, excluding injected and notification messages.""" + for message in reversed(history): + if message.role != "user": + continue + if is_notification_message(message) or is_system_reminder_message(message): + continue + text = message.extract_text(" ").strip() + if text: + return text + return None + + def classify_llm_system(chat_provider: object | None) -> str: """Classify a chat provider into a stable gen_ai.system telemetry value.""" try: @@ -356,24 +408,54 @@ def _user_message_with_hook_context( return Message(role="user", content=[*base, reminder]) -def _with_agents_md_preamble( - history: Sequence[Message], builtin_args: BuiltinSystemPromptArgs -) -> list[Message]: - """Return *history* with the merged AGENTS.md prepended as a leading user-role - ````, or a plain copy of *history* when no AGENTS.md applies. +@dataclass(frozen=True, slots=True) +class _PreparedRequest: + assembled: AssembledRequest + acknowledgements: tuple[SourceAcknowledgement, ...] + + +_SKILL_SOURCE = "skill_catalog" +_SKILL_KEY = "task_candidates" +_SIDE_QUESTION_SOURCE = "side_question" +_SIDE_QUESTION_KEY = "question" + + +def _provided_source(policy: TrustedSourcePolicy, content: str) -> RequestSourceResult: + return RequestSourceResult( + source=policy.source, + key=policy.key, + status=SourceResultStatus.PROVIDED, + fragment=RequestFragment( + key=policy.key, + content=content, + source=policy.source, + requirement=policy.requirement, + persistence=policy.persistence, + priority=policy.priority, + truncatable=policy.truncation is FragmentTruncation.ALLOWED, + ), + reason_code=None, + ) - The preamble is assembled fresh from ``builtin_args`` on every step and is NEVER - appended to ``context.history``. That is precisely what keeps the project instructions - immune to the two failure modes a persisted home would hit: context compaction cannot - summarize them away (they are not in the history it rewrites), and the dynamic-injection - token budget cannot truncate them (they are not a budgeted injection). The input is left - unmutated. See :func:`pythinker_code.soul.agent.render_agents_md_reminder`. - """ - reminder = render_agents_md_reminder(builtin_args) - if reminder is None: - return list(history) - preamble = Message(role="user", content=[system_reminder(reminder)]) - return [preamble, *history] + +def _not_applicable_source(policy: TrustedSourcePolicy) -> RequestSourceResult: + return RequestSourceResult( + source=policy.source, + key=policy.key, + status=SourceResultStatus.NOT_APPLICABLE, + fragment=None, + reason_code=None, + ) + + +def _failed_source(policy: TrustedSourcePolicy, reason_code: str) -> RequestSourceResult: + return RequestSourceResult( + source=policy.source, + key=policy.key, + status=SourceResultStatus.FAILED, + fragment=None, + reason_code=reason_code, + ) def _should_nudge_truncation( @@ -532,6 +614,8 @@ def __init__( self._deliberation_generation = 0 self._sleep_inhibitor = SleepInhibitor(enabled=agent.runtime.config.prevent_idle_sleep) self._compaction = SimpleCompaction(base_prompt=self._runtime.config.compact_prompt) + self.latest_skill_projection_outcome: SkillProjectionOutcome | None = None + self.latest_request_manifest: RequestManifest | None = None for tool in agent.toolset.tools: if tool.name == SendDMail_NAME: @@ -588,6 +672,8 @@ def __init__( else [AutoModeInjectionProvider()] ), ] + self._request_lifecycle = RequestLifecycle(self._injection_providers) + self._notified_context_generations: set[int] = set() self._hook_engine: HookEngine = HookEngine() self._stop_hook_active: bool = False if self._runtime.role == "root": @@ -666,69 +752,119 @@ def set_hook_engine(self, engine: HookEngine) -> None: def add_injection_provider(self, provider: DynamicInjectionProvider) -> None: """Register an additional dynamic injection provider.""" self._injection_providers.append(provider) + self._request_lifecycle.sync_providers(self._injection_providers) def rearm_injection(self, key: str) -> None: """Re-arm matching dynamic injection providers after related state changes.""" - for provider in self._injection_providers: - try: - provider.rearm(key) - except Exception: - logger.debug("injection provider rearm failed") + failures = self._request_lifecycle.rearm(self._injection_providers, key) + for failure in failures: + logger.debug("injection provider rearm failed", exc_info=failure) - async def _collect_injections(self) -> list[DynamicInjection]: - """Collect dynamic injections from all registered providers.""" - injections: list[DynamicInjection] = [] - for provider in self._injection_providers: - try: - result = await provider.get_injections(self._context.history, self) - injections.extend(result) - except Exception as exc: - from pythinker_code.telemetry.errors import report_handled_error + async def _required_request_sources(self) -> PreparedSources: + return await self._request_lifecycle.prepare_required( + self._injection_providers, + self._context.history, + self, + self._report_provider_failure, + ) - report_handled_error( - exc, - site="soul.injection.get", - provider=type(provider).__name__, - ) - logger.warning( - "injection provider %s failed", - type(provider).__name__, - exc_info=True, - ) - memory_config = getattr(self._runtime.config, "memory", None) - if not getattr(memory_config, "injection_bus", True): - return injections - candidates = [dynamic_to_candidate(injection) for injection in injections] - budget = injection_budget_from_runtime(self._runtime).injection_budget_tokens - budgeted = collect_within_budget(candidates, budget) - return [ - DynamicInjection(type=candidate.type, content=candidate.content) - for candidate in budgeted - ] + async def _optional_request_sources(self) -> PreparedSources: + return await self._request_lifecycle.prepare_optional( + self._injection_providers, + self._context.history, + self, + self._report_provider_failure, + enabled=self._runtime.config.memory.injection_bus, + ) - async def _notify_injection_providers_compacted(self) -> None: - """Notify all injection providers that the context has been compacted. + @staticmethod + def _report_provider_failure(provider: DynamicInjectionProvider, error: Exception) -> None: + from pythinker_code.telemetry.errors import report_handled_error + + report_handled_error( + error, + site="soul.injection.get", + provider=type(provider).__name__, + ) + logger.warning( + "injection provider %s failed", + type(provider).__name__, + exc_info=True, + ) + + async def notify_history_rebuilt(self) -> None: + """Advance request lifecycle state after history is replaced or cleared. Failures are isolated per-provider so a buggy third-party provider - cannot abort compaction (which would skip CompactionEnd wire events - and PostCompact telemetry). + cannot prevent other providers from rearming against the new history. """ - for provider in self._injection_providers: + self._request_lifecycle.context_rebuilt() + + async def notify_providers() -> None: + for provider in self._injection_providers: + try: + await provider.on_context_compacted() + except asyncio.CancelledError as exc: + from pythinker_code.telemetry.errors import report_handled_error + + report_handled_error( + exc, + site="soul.injection.on_context_compacted", + provider=type(provider).__name__, + ) + logger.warning( + "injection provider %s cancelled its context callback", + type(provider).__name__, + exc_info=True, + ) + except Exception as exc: + from pythinker_code.telemetry.errors import report_handled_error + + report_handled_error( + exc, + site="soul.injection.on_context_compacted", + provider=type(provider).__name__, + ) + logger.warning( + "injection provider %s on_context_compacted failed", + type(provider).__name__, + exc_info=True, + ) + + notification_task = asyncio.create_task(notify_providers()) + try: + await asyncio.shield(notification_task) + except asyncio.CancelledError as cancellation: + await _settle_shielded(notification_task) try: - await provider.on_context_compacted() - except Exception as exc: - from pythinker_code.telemetry.errors import report_handled_error + notification_task.result() + except BaseException as notification_error: + raise cancellation from notification_error + raise - report_handled_error( - exc, - site="soul.injection.on_context_compacted", - provider=type(provider).__name__, - ) - logger.warning( - "injection provider %s on_context_compacted failed", - type(provider).__name__, - exc_info=True, - ) + async def _notify_context_commit(self, commit: ContextCommit) -> None: + if commit.generation in self._notified_context_generations: + return + self._notified_context_generations.add(commit.generation) + await self.notify_history_rebuilt() + + async def _complete_history_replacement(self, operation: Awaitable[ContextCommit]) -> None: + try: + commit = await operation + except ContextCommittedCancellation as cancellation: + await self._notify_context_commit(cancellation.commit) + raise + except ContextPersistenceError as error: + if error.commit is not None: + await self._notify_context_commit(error.commit) + raise + await self._notify_context_commit(commit) + + async def clear_context(self) -> None: + await self._complete_history_replacement(self._context.clear(self._agent.system_prompt)) + + async def _revert_context_to(self, checkpoint_id: int) -> None: + await self._complete_history_replacement(self._context.revert_to(checkpoint_id)) async def notify_auto_changed(self, enabled: bool) -> None: """Notify dynamic injection providers that auto mode changed.""" @@ -1797,7 +1933,7 @@ async def _agent_loop(self) -> TurnOutcome: ) if back_to_the_future is not None: - await self._context.revert_to(back_to_the_future.checkpoint_id) + await self._revert_context_to(back_to_the_future.checkpoint_id) # The reverted history no longer contains the last step's calls, # so they must not seed cross-step dedup for the next step. self._last_tool_calls = [] @@ -1807,6 +1943,188 @@ async def _agent_loop(self) -> TurnOutcome: # Consume any pending steers between steps await self._consume_pending_steers() + def _agents_request_source( + self, + ) -> tuple[TrustedSourcePolicy, RequestSourceResult]: + reminder = render_agents_md_reminder(self._runtime.builtin_args) + if reminder is None: + return AGENTS_MD_SOURCE_POLICY, _not_applicable_source(AGENTS_MD_SOURCE_POLICY) + return AGENTS_MD_SOURCE_POLICY, _provided_source(AGENTS_MD_SOURCE_POLICY, reminder) + + def _skill_request_sources( + self, task: str + ) -> tuple[tuple[TrustedSourcePolicy, ...], tuple[RequestSourceResult, ...]]: + policy = TrustedSourcePolicy( + source=_SKILL_SOURCE, + key=_SKILL_KEY, + requirement=FragmentRequirement.BEST_EFFORT, + persistence=FragmentPersistence.REQUEST_ONLY, + priority=0, + budget_class=FragmentBudgetClass.BUDGETED, + truncation=FragmentTruncation.FORBIDDEN, + applicability=SourceApplicability.MAY_BE_NOT_APPLICABLE, + failure_reason_codes=( + "invalid_projection_budget", + "projection_budget_too_small", + "skill_projection_failed", + ), + ) + if not task: + self.latest_skill_projection_outcome = None + return (policy,), (_not_applicable_source(policy),) + try: + outcome = self._runtime.skill_catalog.prompt_view( + task, + max_characters=SKILL_PROMPT_MAX_CHARACTERS - len(system_reminder("").text), + explicit_names=_explicit_skill_names(task), + active_names=self._runtime.session.state.active_skills, + ) + except Exception: + logger.warning("Skill candidate projection failed", exc_info=True) + self.latest_skill_projection_outcome = None + return (policy,), (_failed_source(policy, "skill_projection_failed"),) + self.latest_skill_projection_outcome = outcome + if outcome.status is not SkillProjectionStatus.READY: + logger.warning( + "Skill candidate projection status={status} reason={reason}", + status=outcome.status.value, + reason=outcome.reason_code or "none", + ) + if outcome.view is None or not outcome.view.matches: + if outcome.status is SkillProjectionStatus.FAILED: + return (policy,), (_failed_source(policy, "skill_projection_failed"),) + return (policy,), (_not_applicable_source(policy),) + return (policy,), (_provided_source(policy, render_skill_prompt_view(outcome.view)),) + + async def _assemble_request( + self, + task: str, + extra_sources: Sequence[tuple[TrustedSourcePolicy, RequestSourceResult]] = (), + ) -> _PreparedRequest: + assembly_started = time.monotonic() + try: + agents_policy, agents_result = self._agents_request_source() + request = RequestAssemblyInput( + system_prompt=self._agent.system_prompt, + persisted_history=tuple(self._context.history), + current_task=task, + budget_tokens=injection_budget_from_runtime(self._runtime).injection_budget_tokens, + history_generation=self._request_lifecycle.history_generation, + ) + required = await self._required_request_sources() + required_policies = (agents_policy, *required.policies) + required_results = (agents_result, *required.results) + if extra_sources: + extra_policies, extra_results = zip(*extra_sources, strict=True) + required_policies = (*required_policies, *extra_policies) + required_results = (*required_results, *extra_results) + await RequestAssembler(required_policies, required_results).assemble(request) + + optional = await self._optional_request_sources() + skill_policies, skill_results = self._skill_request_sources(task) + policies = (*required_policies, *optional.policies, *skill_policies) + source_results = (*required_results, *optional.results, *skill_results) + assembled = await RequestAssembler(policies, source_results).assemble(request) + except RequestAssemblyError as error: + self.latest_request_manifest = error.manifest + self._record_request_assembly_telemetry(error.manifest, assembly_started) + raise + except asyncio.CancelledError: + raise + except RequestLifecycleError as error: + manifest = failed_manifest(error.reason_code, None) + self.latest_request_manifest = manifest + self._record_request_assembly_telemetry(manifest, assembly_started) + raise + except Exception as error: + failure = RequestLifecycleError("request_source_adapter_failed") + manifest = failed_manifest(failure.reason_code, None) + self.latest_request_manifest = manifest + self._record_request_assembly_telemetry(manifest, assembly_started) + raise failure from error + self.latest_request_manifest = assembled.manifest + self._record_request_assembly_telemetry(assembled.manifest, assembly_started) + return _PreparedRequest( + assembled, + (*required.acknowledgements, *optional.acknowledgements), + ) + + @staticmethod + def _record_request_assembly_telemetry( + manifest: RequestManifest, + assembly_started: float, + ) -> None: + from pythinker_code.telemetry import metrics + + try: + metrics.record_request_assembly( + manifest, + duration_seconds=time.monotonic() - assembly_started, + ) + except Exception: + logger.warning("Request assembly telemetry failed", exc_info=True) + + async def _persist_assembled_history(self, prepared: _PreparedRequest) -> None: + if not prepared.assembled.history_appends: + return + + async def _commit_and_finalize() -> None: + try: + await self._context.append_message(prepared.assembled.history_appends) + except Exception as error: + raise RequestLifecycleError("context_persistence_failed") from error + try: + self._request_lifecycle.finalize( + prepared.assembled.manifest, + prepared.acknowledgements, + ) + except RequestLifecycleError: + raise + except Exception as error: + raise RequestLifecycleError("provider_finalization_failed") from error + + commit_task = asyncio.create_task(_commit_and_finalize()) + try: + await asyncio.shield(commit_task) + except asyncio.CancelledError as cancellation: + await _settle_shielded(commit_task) + commit_error: BaseException | None = None + try: + commit_task.result() + except BaseException as error: + commit_error = error + self.latest_request_manifest = failed_manifest( + "context_persistence_cancelled", + prepared.assembled.manifest, + ) + if commit_error is not None: + raise cancellation from commit_error + raise + except RequestLifecycleError as error: + self.latest_request_manifest = failed_manifest( + error.reason_code, + prepared.assembled.manifest, + ) + raise + + async def assemble_side_request(self, question: str, reminder_text: str) -> AssembledRequest: + policy = TrustedSourcePolicy( + source=_SIDE_QUESTION_SOURCE, + key=_SIDE_QUESTION_KEY, + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.REQUEST_ONLY, + priority=-100, + budget_class=FragmentBudgetClass.BUDGETED, + truncation=FragmentTruncation.FORBIDDEN, + applicability=SourceApplicability.ALWAYS, + failure_reason_codes=("side_question_invalid",), + ) + content = f"{reminder_text}\n\n{question}" + prepared = await self._assemble_request( + question, ((policy, _provided_source(policy, content)),) + ) + return prepared.assembled + async def _step(self) -> StepOutcome | None: """Run a single step and return a stop outcome, or None to continue.""" # already checked in `run` @@ -1854,23 +2172,10 @@ async def _append_notification(view: NotificationView) -> None: on_notification=_append_notification, ) - # Dynamic injection - injections = await self._collect_injections() - if injections: - combined_reminders = "\n".join(system_reminder(inj.content).text for inj in injections) - await self._context.append_message( - Message( - role="user", - content=[TextPart(text=combined_reminders)], - ) - ) - - # Prepend the merged AGENTS.md as a leading (assembled fresh from - # runtime args, never persisted to history) so the project instructions are immune to - # compaction and the injection budget, then normalize to merge adjacent user messages. - effective_history = normalize_history( - _with_agents_md_preamble(self._context.history, self._runtime.builtin_args) - ) + task = _latest_real_user_text(self._context.history) or "" + prepared_request = await self._assemble_request(task) + await self._persist_assembled_history(prepared_request) + effective_history = prepared_request.assembled.provider_history # Capture tool results as they stream in. If the batch is interrupted # mid-flight, already-completed calls must keep their real output rather @@ -2353,8 +2658,10 @@ async def prune_context(self) -> bool: when there is nothing worth pruning. Runs silently — no compaction wire events — since it may fire often and is not a user-visible summary. """ + snapshot_generation = self._context.mutation_generation + snapshot = tuple(self._context.history) capped, cap_freed = cap_stale_tool_result_bodies( - self._context.history, + snapshot, protect_last=self._loop_control.prune_protect_last, max_chars=self._loop_control.prune_tool_result_max_chars, ) @@ -2368,11 +2675,6 @@ async def prune_context(self) -> bool: return False before_tokens = self._context.token_count - # Snapshot history first: clear() rotates the backing file, so a mid-rebuild - # failure would otherwise leave the context as just the system prompt. Reuse the - # same clear+rebuild primitive compact_context uses (the supported way to mutate - # the append-only JSONL context), but roll back to the snapshot if it throws. - snapshot = list(self._context.history) # Reduce the AUTHORITATIVE pre-prune count by the estimated tokens freed, rather # than replacing it with a full heuristic re-estimate of the remaining history. A # full re-estimate can over-count the survivors (chars/4 overshoots code/markup), @@ -2381,20 +2683,16 @@ async def prune_context(self) -> bool: # pruning can only lower the count (pruned ⊆ snapshot ⇒ delta ≥ 0). freed_tokens = estimate_text_tokens(snapshot) - estimate_text_tokens(pruned) pruned_tokens = max(0, before_tokens - max(0, freed_tokens)) - await self._context.clear() - try: - await self._context.write_system_prompt(self._agent.system_prompt) - await self._checkpoint() - await self._context.append_message(pruned) - await self._context.update_token_count(pruned_tokens) - except Exception: - await self._context.clear() - await self._context.write_system_prompt(self._agent.system_prompt) - await self._checkpoint() - if snapshot: - await self._context.append_message(snapshot) - await self._context.update_token_count(before_tokens) - raise + await self._context.replace_history( + ContextReplacement( + system_prompt=self._agent.system_prompt, + messages=tuple(pruned), + token_count=pruned_tokens, + create_checkpoint=True, + checkpoint_user_marker=self._checkpoint_with_user_message, + ), + expected_generation=snapshot_generation, + ) # Unlike full compaction, pruning preserves every non-tool message verbatim # (only tool-result *bodies* are elided), so prior dynamic injections survive in # history. Do NOT re-arm injection providers here, or one-shot fragments (e.g. the @@ -2445,6 +2743,7 @@ async def _compact_with_retry() -> CompactionResult: trigger_reason = "manual" if custom_instruction else "auto" before_tokens = self._context.token_count + history_generation = self._context.mutation_generation history_before_compaction = tuple(self._context.history) from pythinker_code.hooks import events @@ -2488,85 +2787,71 @@ async def _compact_with_retry() -> CompactionResult: self._session_cost_usd += estimate_cost_usd( compaction_result.usage, self.model_name ) - await self._context.clear() - try: - await self._context.write_system_prompt(self._agent.system_prompt) - await self._checkpoint() - await self._context.append_message(compaction_result.messages) - estimated_token_count = compaction_result.estimated_token_count - summary_text = compact_summary_text(compaction_result.messages) - - if restore_context.messages: - await self._context.append_message(restore_context.messages) - estimated_token_count += estimate_text_tokens(restore_context.messages) - - if self._runtime.role == "root": - active_task_snapshot = build_active_task_snapshot( - self._runtime.background_tasks + replacement_messages = list(compaction_result.messages) + estimated_token_count = compaction_result.estimated_token_count + summary_text = compact_summary_text(compaction_result.messages) + + if restore_context.messages: + replacement_messages.extend(restore_context.messages) + estimated_token_count += estimate_text_tokens(restore_context.messages) + + if self._runtime.role == "root": + active_task_snapshot = build_active_task_snapshot(self._runtime.background_tasks) + if active_task_snapshot is not None: + active_task_message = Message( + role="user", + content=[ + system( + "The following background tasks are still active" + " after compaction. Use TaskList if you need to" + " re-enumerate them later." + ), + TextPart(text=active_task_snapshot), + ], ) - if active_task_snapshot is not None: - active_task_message = Message( - role="user", - content=[ - system( - "The following background tasks are still active" - " after compaction. Use TaskList if you need to" - " re-enumerate them later." - ), - TextPart(text=active_task_snapshot), - ], - ) - await self._context.append_message(active_task_message) - estimated_token_count += estimate_text_tokens([active_task_message]) + replacement_messages.append(active_task_message) + estimated_token_count += estimate_text_tokens([active_task_message]) - post_compact_results = await self._hook_engine.trigger( - "PostCompact", - matcher_value=trigger_reason, - input_data=events.post_compact( - session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.work_dir)), - trigger=trigger_reason, - estimated_token_count=estimated_token_count, - compact_summary=summary_text, - ), - ) - session_start_results = await self._hook_engine.trigger( - "SessionStart", - matcher_value="compact", - input_data=events.session_start( - session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.work_dir)), - source="compact", + post_compact_results = await self._hook_engine.trigger( + "PostCompact", + matcher_value=trigger_reason, + input_data=events.post_compact( + session_id=self._runtime.session.id, + cwd=_safe_cwd(str(self._runtime.work_dir)), + trigger=trigger_reason, + estimated_token_count=estimated_token_count, + compact_summary=summary_text, + ), + ) + session_start_results = await self._hook_engine.trigger( + "SessionStart", + matcher_value="compact", + input_data=events.session_start( + session_id=self._runtime.session.id, + cwd=_safe_cwd(str(self._runtime.work_dir)), + source="compact", + ), + ) + hook_context_message = build_hook_context_message( + result.additional_context + for result in [*post_compact_results, *session_start_results] + ) + if hook_context_message is not None: + replacement_messages.append(hook_context_message) + estimated_token_count += estimate_text_tokens([hook_context_message]) + + await self._complete_history_replacement( + self._context.replace_history( + ContextReplacement( + system_prompt=self._agent.system_prompt, + messages=tuple(replacement_messages), + token_count=estimated_token_count, + create_checkpoint=True, + checkpoint_user_marker=self._checkpoint_with_user_message, ), + expected_generation=history_generation, ) - hook_context_message = build_hook_context_message( - result.additional_context - for result in [*post_compact_results, *session_start_results] - ) - if hook_context_message is not None: - await self._context.append_message(hook_context_message) - estimated_token_count += estimate_text_tokens([hook_context_message]) - - # Estimate token count so context_usage is not reported as 0% - await self._context.update_token_count(estimated_token_count) - - # Notify dynamic injection providers that history has been rebuilt so - # they can reset any one-shot throttling state. Failures are isolated - # per-provider so compaction completion (wire event + telemetry) is - # not affected by a buggy provider. - await self._notify_injection_providers_compacted() - except Exception: - # Rebuild faulted after clear() rotated the backing file. Restore - # the pre-compaction history so an I/O fault cannot truncate the - # live context to just the system prompt. Same primitive as - # prune_context. - await self._context.clear() - await self._context.write_system_prompt(self._agent.system_prompt) - await self._checkpoint() - if history_before_compaction: - await self._context.append_message(list(history_before_compaction)) - await self._context.update_token_count(before_tokens) - raise + ) except Exception: from pythinker_code.telemetry import track diff --git a/src/pythinker_code/soul/request_assembly.py b/src/pythinker_code/soul/request_assembly.py new file mode 100644 index 00000000..44044bd1 --- /dev/null +++ b/src/pythinker_code/soul/request_assembly.py @@ -0,0 +1,770 @@ +from __future__ import annotations + +import hashlib +import re +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import StrEnum + +from pythinker_core.message import Message, TextPart + +from pythinker_code.soul.message import system_reminder +from pythinker_code.soul.request_primitives import ( + estimate_injection_tokens, + normalize_history, +) + + +class FragmentRequirement(StrEnum): + REQUIRED = "required" + BEST_EFFORT = "best_effort" + + +class FragmentPersistence(StrEnum): + REQUEST_ONLY = "request_only" + HISTORY = "history" + + +class FragmentStatus(StrEnum): + INCLUDED = "included" + NOT_APPLICABLE = "not_applicable" + TRUNCATED = "truncated" + OMITTED_BUDGET = "omitted_budget" + DEGRADED = "degraded" + FAILED = "failed" + + +class RequestStatus(StrEnum): + SUCCEEDED = "succeeded" + DEGRADED = "degraded" + FAILED = "failed" + + +class SourceResultStatus(StrEnum): + PROVIDED = "provided" + ALREADY_SATISFIED = "already_satisfied" + NOT_APPLICABLE = "not_applicable" + FAILED = "failed" + + +class FragmentBudgetClass(StrEnum): + BUDGETED = "budgeted" + NON_BUDGETED = "non_budgeted" + + +class FragmentTruncation(StrEnum): + FORBIDDEN = "forbidden" + ALLOWED = "allowed" + + +class SourceApplicability(StrEnum): + ALWAYS = "always" + MAY_BE_NOT_APPLICABLE = "may_be_not_applicable" + + +@dataclass(frozen=True, slots=True) +class RequestFragment: + key: str + content: str + source: str + requirement: FragmentRequirement + persistence: FragmentPersistence + priority: int + truncatable: bool + + +@dataclass(frozen=True, slots=True) +class FragmentOutcome: + key: str + source: str + requirement: FragmentRequirement + persistence: FragmentPersistence + status: FragmentStatus + estimated_tokens: int + admitted_tokens: int + reason_code: str | None + + +@dataclass(frozen=True, slots=True) +class RequestManifest: + status: RequestStatus + reason_code: str | None + outcomes: tuple[FragmentOutcome, ...] + budget_tokens: int + budgeted_admitted_tokens: int + non_budgeted_estimated_tokens: int + + +@dataclass(frozen=True, slots=True) +class RequestAssemblyInput: + system_prompt: str + persisted_history: tuple[Message, ...] + current_task: str + budget_tokens: int + history_generation: int = 0 + + +@dataclass(frozen=True, slots=True) +class AssembledRequest: + system_prompt: str + provider_history: tuple[Message, ...] + history_appends: tuple[Message, ...] + manifest: RequestManifest + + +@dataclass(frozen=True, slots=True) +class TrustedSourcePolicy: + source: str + key: str + requirement: FragmentRequirement + persistence: FragmentPersistence + priority: int + budget_class: FragmentBudgetClass + truncation: FragmentTruncation + applicability: SourceApplicability + failure_reason_codes: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class RequestSourceResult: + source: str + key: str + status: SourceResultStatus + fragment: RequestFragment | None + reason_code: str | None + history_generation: int | None = None + + +class RequestAssemblyError(RuntimeError): + manifest: RequestManifest + reason_code: str + + def __init__(self, reason_code: str, manifest: RequestManifest) -> None: + if manifest.status is not RequestStatus.FAILED or manifest.reason_code != reason_code: + raise ValueError("request assembly errors require a matching failed manifest") + super().__init__(reason_code) + self.reason_code = reason_code + self.manifest = manifest + + +class RequestBudgetError(RequestAssemblyError): + pass + + +class RequestSourceError(RequestAssemblyError): + pass + + +class RequestHistoryError(RequestAssemblyError): + pass + + +class RequestInvariantError(RequestAssemblyError): + pass + + +AGENTS_MD_SOURCE_POLICY = TrustedSourcePolicy( + source="agents_md", + key="agents_preamble", + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.REQUEST_ONLY, + priority=1_000, + budget_class=FragmentBudgetClass.NON_BUDGETED, + truncation=FragmentTruncation.FORBIDDEN, + applicability=SourceApplicability.MAY_BE_NOT_APPLICABLE, + failure_reason_codes=("agents_md_unavailable", "agents_md_invalid"), +) + + +@dataclass(frozen=True, slots=True) +class _Admission: + policy: TrustedSourcePolicy + fragment: RequestFragment | None + outcome: FragmentOutcome + + +@dataclass(frozen=True, slots=True) +class _RequestProjection: + leading_messages: tuple[Message, ...] + trailing_messages: tuple[Message, ...] + history_appends: tuple[Message, ...] + outcomes: tuple[FragmentOutcome, ...] + + +class _AssemblyFailure(Exception): + def __init__(self, reason_code: str, outcomes: tuple[FragmentOutcome, ...]) -> None: + super().__init__(reason_code) + self.reason_code = reason_code + self.outcomes = outcomes + + +@dataclass(frozen=True, slots=True) +class _TrustedSourceRegistry: + policies: tuple[TrustedSourcePolicy, ...] + by_identity: dict[tuple[str, str], TrustedSourcePolicy] + source_rank: dict[str, int] + key_rank: dict[tuple[str, str], int] + + @classmethod + def build(cls, policies: Sequence[TrustedSourcePolicy]) -> _TrustedSourceRegistry: + _validate_policies(policies) + by_identity = {(policy.source, policy.key): policy for policy in policies} + source_rank: dict[str, int] = {} + for policy in policies: + source_rank.setdefault(policy.source, len(source_rank)) + ordered_identities = sorted(by_identity) + key_rank = { + identity: index + for source in source_rank + for index, identity in enumerate( + candidate for candidate in ordered_identities if candidate[0] == source + ) + } + return cls(tuple(policies), by_identity, source_rank, key_rank) + + def ordered(self) -> tuple[TrustedSourcePolicy, ...]: + return tuple( + sorted( + self.policies, + key=lambda policy: ( + policy.requirement is not FragmentRequirement.REQUIRED, + -policy.priority, + self.source_rank[policy.source], + self.key_rank[(policy.source, policy.key)], + policy.key, + ), + ) + ) + + +HistoryNormalizer = Callable[[Sequence[Message]], list[Message]] +_SAFE_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]*") +_MAX_IDENTIFIER_LENGTH = 64 +_DEGRADED_STATUSES = frozenset( + {FragmentStatus.TRUNCATED, FragmentStatus.OMITTED_BUDGET, FragmentStatus.DEGRADED} +) + + +def opaque_manifest_identifier(identifier: str) -> str: + """Return a stable content-free identifier for diagnostic surfaces.""" + identifier_bytes = identifier.encode(encoding="utf-8") + digest = hashlib.sha256(identifier_bytes).hexdigest()[:16] + return f"id:{digest}" + + +def _validate_policies(policies: Sequence[TrustedSourcePolicy]) -> None: + identities = [(policy.source, policy.key) for policy in policies] + if len(identities) != len(set(identities)): + raise _AssemblyFailure("internal_invariant_violation", ()) + for policy in policies: + _validate_policy_identifiers(policy) + if policy.source == AGENTS_MD_SOURCE_POLICY.source and policy != AGENTS_MD_SOURCE_POLICY: + raise _AssemblyFailure("invalid_agents_policy", ()) + if ( + policy.budget_class is FragmentBudgetClass.NON_BUDGETED + and policy != AGENTS_MD_SOURCE_POLICY + ): + raise _AssemblyFailure("invalid_non_budgeted_policy", ()) + + +def _validate_policy_identifiers(policy: TrustedSourcePolicy) -> None: + identifiers = (policy.source, policy.key, *policy.failure_reason_codes) + if any(not _is_safe_identifier(identifier) for identifier in identifiers): + raise _AssemblyFailure("unsafe_source_policy", ()) + + +def _is_safe_identifier(identifier: str) -> bool: + return ( + 0 < len(identifier) <= _MAX_IDENTIFIER_LENGTH + and _SAFE_IDENTIFIER.fullmatch(identifier) is not None + ) + + +def admit_source_results( + policies: Sequence[TrustedSourcePolicy], + source_results: Sequence[RequestSourceResult], + budget_tokens: int, + history_generation: int | None = None, +) -> tuple[_Admission, ...]: + if budget_tokens < 0: + raise _AssemblyFailure("invalid_budget", ()) + registry = _TrustedSourceRegistry.build(policies) + results_by_identity = _validated_results(registry, source_results, history_generation) + ordered_pairs = tuple( + (policy, results_by_identity[(policy.source, policy.key)]) for policy in registry.ordered() + ) + initial = _initial_admissions(ordered_pairs) + _raise_required_source_failure(initial) + _reserve_required_budget(initial, budget_tokens) + return _admit_with_budget(initial, budget_tokens) + + +def _validated_results( + registry: _TrustedSourceRegistry, + source_results: Sequence[RequestSourceResult], + history_generation: int | None, +) -> dict[tuple[str, str], RequestSourceResult]: + identities = [(source_result.source, source_result.key) for source_result in source_results] + if len(identities) != len(set(identities)) or len(source_results) != len(registry.policies): + raise _AssemblyFailure("internal_invariant_violation", ()) + if any(identity not in registry.by_identity for identity in identities): + raise _AssemblyFailure("unknown_source_result", ()) + results = { + identity: source_result + for identity, source_result in zip(identities, source_results, strict=True) + } + if set(results) != set(registry.by_identity): + raise _AssemblyFailure("internal_invariant_violation", ()) + for identity, source_result in results.items(): + _validate_source_result( + registry.by_identity[identity], + source_result, + history_generation, + ) + return results + + +def _validate_source_result( + policy: TrustedSourcePolicy, + source_result: RequestSourceResult, + history_generation: int | None, +) -> None: + if source_result.status in { + SourceResultStatus.PROVIDED, + SourceResultStatus.ALREADY_SATISFIED, + }: + if source_result.fragment is None or source_result.reason_code is not None: + raise _AssemblyFailure("source_result_invalid", ()) + _validate_fragment_matches_policy(policy, source_result.fragment) + if source_result.status is SourceResultStatus.ALREADY_SATISFIED: + if ( + policy.persistence is not FragmentPersistence.HISTORY + or source_result.history_generation is None + or source_result.history_generation != history_generation + ): + raise _AssemblyFailure("source_history_proof_invalid", ()) + elif source_result.history_generation is not None: + raise _AssemblyFailure("source_history_proof_invalid", ()) + return + if source_result.history_generation is not None: + raise _AssemblyFailure("source_history_proof_invalid", ()) + if source_result.fragment is not None: + raise _AssemblyFailure("source_result_invalid", ()) + if source_result.status is SourceResultStatus.NOT_APPLICABLE: + if source_result.reason_code is not None: + raise _AssemblyFailure("source_result_invalid", ()) + if policy.applicability is SourceApplicability.ALWAYS: + raise _AssemblyFailure("source_policy_mismatch", ()) + return + if source_result.reason_code not in policy.failure_reason_codes: + raise _AssemblyFailure("unsafe_source_reason", ()) + + +def _validate_fragment_matches_policy( + policy: TrustedSourcePolicy, fragment: RequestFragment +) -> None: + expected_truncatable = policy.truncation is FragmentTruncation.ALLOWED + if ( + fragment.source != policy.source + or fragment.key != policy.key + or fragment.requirement is not policy.requirement + or fragment.persistence is not policy.persistence + or fragment.priority != policy.priority + or fragment.truncatable is not expected_truncatable + ): + raise _AssemblyFailure("source_policy_mismatch", ()) + + +def _initial_admissions( + ordered_pairs: Sequence[tuple[TrustedSourcePolicy, RequestSourceResult]], +) -> tuple[_Admission, ...]: + return tuple( + _initial_admission(policy, source_result) for policy, source_result in ordered_pairs + ) + + +def _initial_admission( + policy: TrustedSourcePolicy, source_result: RequestSourceResult +) -> _Admission: + if source_result.status is SourceResultStatus.NOT_APPLICABLE: + return _Admission(policy, None, _outcome(policy, FragmentStatus.NOT_APPLICABLE, 0, 0)) + if source_result.status is SourceResultStatus.FAILED: + status = ( + FragmentStatus.FAILED + if policy.requirement is FragmentRequirement.REQUIRED + else FragmentStatus.DEGRADED + ) + return _Admission( + policy, + None, + _outcome(policy, status, 0, 0, source_result.reason_code), + ) + if source_result.status is SourceResultStatus.ALREADY_SATISFIED: + fragment = source_result.fragment + if fragment is None or not fragment.content: + raise _AssemblyFailure("required_source_invalid", ()) + return _Admission( + policy, + None, + _outcome(policy, FragmentStatus.INCLUDED, 0, 0, "already_satisfied"), + ) + fragment = source_result.fragment + if fragment is None: + raise _AssemblyFailure("internal_invariant_violation", ()) + if not fragment.content and policy.requirement is FragmentRequirement.REQUIRED: + outcome = _outcome( + policy, + FragmentStatus.FAILED, + 0, + 0, + "required_source_invalid", + ) + return _Admission(policy, None, outcome) + estimate = estimate_injection_tokens(fragment.content) + return _Admission(policy, fragment, _outcome(policy, FragmentStatus.INCLUDED, estimate, 0)) + + +def _raise_required_source_failure(admissions: Sequence[_Admission]) -> None: + observed: list[FragmentOutcome] = [] + for admission in admissions: + observed.append(admission.outcome) + if ( + admission.policy.requirement is FragmentRequirement.REQUIRED + and admission.outcome.status is FragmentStatus.FAILED + ): + reason_code = admission.outcome.reason_code or "required_source_invalid" + raise _AssemblyFailure(reason_code, tuple(observed)) + + +def _reserve_required_budget(admissions: Sequence[_Admission], budget_tokens: int) -> None: + required_tokens = sum( + admission.outcome.estimated_tokens + for admission in admissions + if admission.policy.requirement is FragmentRequirement.REQUIRED + and admission.policy.budget_class is FragmentBudgetClass.BUDGETED + ) + if required_tokens <= budget_tokens: + return + outcomes = tuple(_required_budget_outcome(admission) for admission in admissions) + raise _AssemblyFailure("required_content_exceeds_budget", outcomes) + + +def _required_budget_outcome(admission: _Admission) -> FragmentOutcome: + policy = admission.policy + estimate = admission.outcome.estimated_tokens + if ( + policy.requirement is FragmentRequirement.REQUIRED + and policy.budget_class is FragmentBudgetClass.NON_BUDGETED + ): + return _outcome(policy, FragmentStatus.INCLUDED, estimate, estimate) + if policy.requirement is FragmentRequirement.REQUIRED: + return _outcome( + policy, + FragmentStatus.FAILED, + estimate, + 0, + "required_content_exceeds_budget", + ) + if admission.outcome.status in {FragmentStatus.DEGRADED, FragmentStatus.NOT_APPLICABLE}: + return admission.outcome + return _outcome(policy, FragmentStatus.OMITTED_BUDGET, estimate, 0, "assembly_stopped") + + +def _admit_with_budget(initial: Sequence[_Admission], budget_tokens: int) -> tuple[_Admission, ...]: + admitted: list[_Admission] = [] + used_tokens = 0 + truncation_budget = budget_tokens + for admission in initial: + remaining_tokens = budget_tokens - used_tokens + updated, charged_tokens = _admit_one( + admission, remaining_tokens, min(remaining_tokens, truncation_budget) + ) + admitted.append(updated) + used_tokens += charged_tokens + if _used_truncation_attempt(admission, remaining_tokens): + truncation_budget = 0 + return tuple(admitted) + + +def _admit_one( + admission: _Admission, remaining_tokens: int, truncation_budget: int +) -> tuple[_Admission, int]: + fragment = admission.fragment + if fragment is None: + return admission, 0 + estimate = admission.outcome.estimated_tokens + if admission.policy.budget_class is FragmentBudgetClass.NON_BUDGETED: + return _included_admission(admission, estimate), 0 + if estimate <= remaining_tokens: + return _included_admission(admission, estimate), estimate + if admission.policy.requirement is FragmentRequirement.REQUIRED: + raise _AssemblyFailure("internal_invariant_violation", ()) + return _admit_optional(admission, truncation_budget) + + +def _included_admission(admission: _Admission, estimate: int) -> _Admission: + return _Admission( + admission.policy, + admission.fragment, + _outcome(admission.policy, FragmentStatus.INCLUDED, estimate, estimate), + ) + + +def _admit_optional(admission: _Admission, truncation_budget: int) -> tuple[_Admission, int]: + fragment = admission.fragment + if fragment is None: + raise _AssemblyFailure("internal_invariant_violation", ()) + estimate = admission.outcome.estimated_tokens + if admission.policy.truncation is FragmentTruncation.FORBIDDEN or truncation_budget <= 0: + outcome = _outcome( + admission.policy, FragmentStatus.OMITTED_BUDGET, estimate, 0, "budget_exceeded" + ) + return _Admission(admission.policy, fragment, outcome), 0 + truncated = _truncate_to_tokens(fragment.content, truncation_budget) + if not truncated: + outcome = _outcome( + admission.policy, FragmentStatus.OMITTED_BUDGET, estimate, 0, "budget_exceeded" + ) + return _Admission(admission.policy, fragment, outcome), 0 + admitted_estimate = estimate_injection_tokens(truncated) + truncated_fragment = _replace_fragment_content(fragment, truncated) + outcome = _outcome( + admission.policy, + FragmentStatus.TRUNCATED, + estimate, + admitted_estimate, + "budget_truncated", + ) + return _Admission(admission.policy, truncated_fragment, outcome), admitted_estimate + + +def _used_truncation_attempt(admission: _Admission, remaining_tokens: int) -> bool: + return bool( + admission.fragment is not None + and admission.policy.requirement is FragmentRequirement.BEST_EFFORT + and admission.policy.truncation is FragmentTruncation.ALLOWED + and admission.outcome.estimated_tokens > remaining_tokens + ) + + +def _replace_fragment_content(fragment: RequestFragment, content: str) -> RequestFragment: + return RequestFragment( + key=fragment.key, + content=content, + source=fragment.source, + requirement=fragment.requirement, + persistence=fragment.persistence, + priority=fragment.priority, + truncatable=fragment.truncatable, + ) + + +def _truncate_to_tokens(text: str, budget_tokens: int) -> str: + max_characters = max(0, budget_tokens * 4) + if max_characters <= 1: + return "" + truncated = text[: max_characters - 1].rstrip() + if "\n" in truncated: + truncated = truncated.rsplit("\n", 1)[0].rstrip() + return f"{truncated}\n…" if truncated else "" + + +def _outcome( + policy: TrustedSourcePolicy, + status: FragmentStatus, + estimated_tokens: int, + admitted_tokens: int, + reason_code: str | None = None, +) -> FragmentOutcome: + return FragmentOutcome( + key=policy.key, + source=policy.source, + requirement=policy.requirement, + persistence=policy.persistence, + status=status, + estimated_tokens=estimated_tokens, + admitted_tokens=admitted_tokens, + reason_code=reason_code, + ) + + +class RequestAssembler: + def __init__( + self, + policies: Sequence[TrustedSourcePolicy], + source_results: Sequence[RequestSourceResult], + *, + history_normalizer: HistoryNormalizer = normalize_history, + ) -> None: + self._policies = tuple(policies) + self._source_results = tuple(source_results) + self._history_normalizer = history_normalizer + + async def assemble(self, request: RequestAssemblyInput) -> AssembledRequest: + outcomes: tuple[FragmentOutcome, ...] = () + boundary_reason = "internal_invariant_violation" + try: + admissions = admit_source_results( + self._policies, + self._source_results, + request.budget_tokens, + request.history_generation, + ) + projection = _project_admissions(admissions, self._policies) + outcomes = projection.outcomes + boundary_reason = "history_normalization_failed" + provider_history = tuple( + self._history_normalizer( + ( + *projection.leading_messages, + *request.persisted_history, + *projection.trailing_messages, + ) + ) + ) + boundary_reason = "internal_invariant_violation" + return AssembledRequest( + system_prompt=request.system_prompt, + provider_history=provider_history, + history_appends=projection.history_appends, + manifest=_successful_manifest(request.budget_tokens, admissions), + ) + except RequestAssemblyError: + raise + except _AssemblyFailure as failure: + raise _categorized_error(request.budget_tokens, failure, self._policies) from failure + except Exception as error: + failure = _AssemblyFailure(boundary_reason, outcomes) + raise _categorized_error(request.budget_tokens, failure, self._policies) from error + + +def _project_admissions( + admissions: Sequence[_Admission], policies: Sequence[TrustedSourcePolicy] +) -> _RequestProjection: + registry = _TrustedSourceRegistry.build(policies) + projection_order = sorted( + admissions, + key=lambda admission: ( + -admission.policy.priority, + registry.source_rank[admission.policy.source], + registry.key_rank[(admission.policy.source, admission.policy.key)], + admission.policy.key, + ), + ) + fragments = tuple( + admission.fragment + for admission in projection_order + if admission.fragment is not None + and admission.outcome.status in {FragmentStatus.INCLUDED, FragmentStatus.TRUNCATED} + ) + leading = tuple( + _fragment_message(fragment) + for fragment in fragments + if fragment.source == AGENTS_MD_SOURCE_POLICY.source + ) + history_fragments = tuple( + fragment for fragment in fragments if fragment.persistence is FragmentPersistence.HISTORY + ) + history_message = _combined_history_message(history_fragments) + request_only = tuple( + _fragment_message(fragment) + for fragment in fragments + if fragment.persistence is FragmentPersistence.REQUEST_ONLY + and fragment.source != AGENTS_MD_SOURCE_POLICY.source + ) + trailing = ((history_message,) if history_message is not None else ()) + request_only + return _RequestProjection( + leading_messages=leading, + trailing_messages=trailing, + history_appends=(history_message,) if history_message is not None else (), + outcomes=tuple(admission.outcome for admission in admissions), + ) + + +def _combined_history_message(fragments: Sequence[RequestFragment]) -> Message | None: + if not fragments: + return None + combined = "\n".join(system_reminder(fragment.content).text for fragment in fragments) + return Message(role="user", content=[TextPart(text=combined)]) + + +def _fragment_message(fragment: RequestFragment) -> Message: + return Message(role="user", content=[system_reminder(fragment.content)]) + + +def _successful_manifest(budget_tokens: int, admissions: Sequence[_Admission]) -> RequestManifest: + outcomes = tuple(admission.outcome for admission in admissions) + status = ( + RequestStatus.DEGRADED + if any(outcome.status in _DEGRADED_STATUSES for outcome in outcomes) + else RequestStatus.SUCCEEDED + ) + budgeted_tokens, non_budgeted_tokens = _aggregate_tokens(admissions) + return RequestManifest( + status=status, + reason_code="optional_fragments_degraded" if status is RequestStatus.DEGRADED else None, + outcomes=outcomes, + budget_tokens=budget_tokens, + budgeted_admitted_tokens=budgeted_tokens, + non_budgeted_estimated_tokens=non_budgeted_tokens, + ) + + +def _aggregate_tokens(admissions: Sequence[_Admission]) -> tuple[int, int]: + budgeted_tokens = sum( + admission.outcome.admitted_tokens + for admission in admissions + if admission.policy.budget_class is FragmentBudgetClass.BUDGETED + ) + non_budgeted_tokens = sum( + admission.outcome.estimated_tokens + for admission in admissions + if admission.policy.budget_class is FragmentBudgetClass.NON_BUDGETED + and admission.outcome.status is FragmentStatus.INCLUDED + ) + return budgeted_tokens, non_budgeted_tokens + + +def _categorized_error( + budget_tokens: int, + failure: _AssemblyFailure, + policies: Sequence[TrustedSourcePolicy], +) -> RequestAssemblyError: + manifest = _failed_manifest(budget_tokens, failure.reason_code, failure.outcomes, policies) + if failure.reason_code in {"invalid_budget", "required_content_exceeds_budget"}: + return RequestBudgetError(failure.reason_code, manifest) + if failure.reason_code == "internal_invariant_violation": + return RequestInvariantError(failure.reason_code, manifest) + if failure.reason_code == "history_normalization_failed": + return RequestHistoryError(failure.reason_code, manifest) + return RequestSourceError(failure.reason_code, manifest) + + +def _failed_manifest( + budget_tokens: int, + reason_code: str, + outcomes: tuple[FragmentOutcome, ...], + policies: Sequence[TrustedSourcePolicy], +) -> RequestManifest: + policy_by_identity = {(policy.source, policy.key): policy for policy in policies} + budgeted_tokens = sum( + outcome.admitted_tokens + for outcome in outcomes + if policy_by_identity[(outcome.source, outcome.key)].budget_class + is FragmentBudgetClass.BUDGETED + ) + non_budgeted_tokens = sum( + outcome.estimated_tokens + for outcome in outcomes + if policy_by_identity[(outcome.source, outcome.key)].budget_class + is FragmentBudgetClass.NON_BUDGETED + and outcome.status is FragmentStatus.INCLUDED + ) + return RequestManifest( + status=RequestStatus.FAILED, + reason_code=reason_code, + outcomes=outcomes, + budget_tokens=budget_tokens, + budgeted_admitted_tokens=budgeted_tokens, + non_budgeted_estimated_tokens=non_budgeted_tokens, + ) diff --git a/src/pythinker_code/soul/request_lifecycle.py b/src/pythinker_code/soul/request_lifecycle.py new file mode 100644 index 00000000..1d04fb32 --- /dev/null +++ b/src/pythinker_code/soul/request_lifecycle.py @@ -0,0 +1,481 @@ +"""Private lifecycle owner for dynamic request-source preparation and finalization.""" + +from __future__ import annotations + +import asyncio +import hashlib +import re +from collections import defaultdict +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from pythinker_core.message import Message + +from pythinker_code.soul.dynamic_injection import ( + DynamicInjectionProvider, + PreparedInjection, +) +from pythinker_code.soul.dynamic_injections.model_defense import ModelDefenseInjectionProvider +from pythinker_code.soul.dynamic_injections.permissions_state import PermissionsInjectionProvider +from pythinker_code.soul.request_assembly import ( + FragmentBudgetClass, + FragmentPersistence, + FragmentRequirement, + FragmentStatus, + FragmentTruncation, + RequestFragment, + RequestManifest, + RequestSourceResult, + RequestStatus, + SourceApplicability, + SourceResultStatus, + TrustedSourcePolicy, +) + +if TYPE_CHECKING: + from pythinker_code.soul.pythinkersoul import PythinkerSoul + + +_PERMISSIONS_SOURCE = "permissions_state" +_MODEL_DEFENSE_SOURCE = "model_defense" +_SAFE_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}") + + +class RequestLifecycleError(RuntimeError): + """Categorized request lifecycle failure.""" + + def __init__(self, reason_code: str) -> None: + super().__init__(reason_code) + self.reason_code = reason_code + + +@dataclass(frozen=True, slots=True) +class SourceAcknowledgement: + registration_id: str + source: str + key: str + prepared_identity: str + + +@dataclass(frozen=True, slots=True) +class PreparedSources: + policies: tuple[TrustedSourcePolicy, ...] + results: tuple[RequestSourceResult, ...] + acknowledgements: tuple[SourceAcknowledgement, ...] + + +@dataclass(slots=True) +class _Registration: + registration_id: str + source: str + provider: DynamicInjectionProvider + lock: asyncio.Lock + + +FailureReporter = Callable[[DynamicInjectionProvider, Exception], None] + + +class RequestLifecycle: + """Own provider identities, preparation serialization, dedupe generations, and finalize.""" + + def __init__(self, providers: Sequence[DynamicInjectionProvider]) -> None: + self._registrations: dict[int, _Registration] = {} + self._source_counts: defaultdict[str, int] = defaultdict(int) + self._next_registration = 0 + self._generation = 0 + self._committed: set[tuple[int, str, str]] = set() + self.sync_providers(providers) + self._default_permissions = _exactly_one_role( + providers, + PermissionsInjectionProvider, + ) + self._default_model_defense = _exactly_one_role( + providers, + ModelDefenseInjectionProvider, + ) + + @property + def history_generation(self) -> int: + return self._generation + + @property + def committed_identity_count(self) -> int: + return len(self._committed) + + def sync_providers(self, providers: Sequence[DynamicInjectionProvider]) -> None: + for provider in providers: + provider_key = id(provider) + if provider_key in self._registrations: + continue + source_base = _source_base(provider) + self._source_counts[source_base] += 1 + occurrence = self._source_counts[source_base] + source = ( + source_base + if occurrence == 1 + else _stable_identifier(f"{source_base}:{occurrence}") + ) + self._next_registration += 1 + self._registrations[provider_key] = _Registration( + registration_id=f"provider-{self._next_registration:04d}", + source=source, + provider=provider, + lock=asyncio.Lock(), + ) + + async def prepare_required( + self, + providers: Sequence[DynamicInjectionProvider], + history: Sequence[Message], + soul: PythinkerSoul, + report_failure: FailureReporter, + ) -> PreparedSources: + self.sync_providers(providers) + registrations = self._ordered_required(providers) + return await self._prepare(registrations, history, soul, report_failure) + + async def prepare_optional( + self, + providers: Sequence[DynamicInjectionProvider], + history: Sequence[Message], + soul: PythinkerSoul, + report_failure: FailureReporter, + *, + enabled: bool, + ) -> PreparedSources: + self.sync_providers(providers) + registrations = self._ordered_optional(providers) + if not enabled: + policies = tuple( + _policy(registration, registration.source) for registration in registrations + ) + return PreparedSources( + policies, + tuple(_not_applicable(policy) for policy in policies), + (), + ) + return await self._prepare(registrations, history, soul, report_failure) + + async def _prepare( + self, + registrations: Sequence[_Registration], + history: Sequence[Message], + soul: PythinkerSoul, + report_failure: FailureReporter, + ) -> PreparedSources: + policies: list[TrustedSourcePolicy] = [] + results: list[RequestSourceResult] = [] + acknowledgements: list[SourceAcknowledgement] = [] + for registration in registrations: + batch = await self._prepare_one(registration, history, soul, report_failure) + policies.extend(batch.policies) + results.extend(batch.results) + acknowledgements.extend(batch.acknowledgements) + if any( + result.status is SourceResultStatus.FAILED + and policy.requirement is FragmentRequirement.REQUIRED + for policy, result in zip(batch.policies, batch.results, strict=True) + ): + break + return PreparedSources(tuple(policies), tuple(results), tuple(acknowledgements)) + + async def _prepare_one( + self, + registration: _Registration, + history: Sequence[Message], + soul: PythinkerSoul, + report_failure: FailureReporter, + ) -> PreparedSources: + try: + async with registration.lock: + prepared = await registration.provider.prepare_injections(history, soul) + except asyncio.CancelledError: + raise + except Exception as exc: + report_failure(registration.provider, exc) + policy = _policy(registration, registration.source) + return PreparedSources( + (policy,), + (_failed(policy, _unavailable_reason(registration)),), + (), + ) + if not prepared: + policy = _policy(registration, registration.source) + if isinstance(registration.provider, PermissionsInjectionProvider): + return PreparedSources( + (policy,), + (_failed(policy, "permissions_state_invalid"),), + (), + ) + return PreparedSources((policy,), (_not_applicable(policy),), ()) + try: + return self._prepared_results(registration, prepared) + except RequestLifecycleError as exc: + report_failure(registration.provider, exc) + policy = _policy(registration, registration.source) + reason = ( + "permissions_state_invalid" + if isinstance(registration.provider, PermissionsInjectionProvider) + else ( + "model_defense_invalid" + if isinstance(registration.provider, ModelDefenseInjectionProvider) + else "provider_failed" + ) + ) + return PreparedSources((policy,), (_failed(policy, reason),), ()) + + def _prepared_results( + self, + registration: _Registration, + prepared: Sequence[PreparedInjection], + ) -> PreparedSources: + identities = [item.identity for item in prepared] + if len(identities) != len(set(identities)): + raise RequestLifecycleError("provider_identity_invalid") + policies: list[TrustedSourcePolicy] = [] + results: list[RequestSourceResult] = [] + acknowledgements: list[SourceAcknowledgement] = [] + for item in prepared: + key = _stable_identifier(item.identity) + policy = _policy(registration, key) + policies.append(policy) + invalid_reason = _invalid_security_type(registration.provider, item.type) + if invalid_reason is not None: + results.append(_failed(policy, invalid_reason)) + continue + fragment = RequestFragment( + key=key, + content=item.content, + source=policy.source, + requirement=policy.requirement, + persistence=policy.persistence, + priority=policy.priority, + truncatable=policy.truncation is FragmentTruncation.ALLOWED, + ) + committed_identity = ( + self._generation, + registration.registration_id, + item.identity, + ) + status = ( + SourceResultStatus.ALREADY_SATISFIED + if committed_identity in self._committed + else SourceResultStatus.PROVIDED + ) + results.append( + RequestSourceResult( + policy.source, + key, + status, + fragment, + None, + self._generation if status is SourceResultStatus.ALREADY_SATISFIED else None, + ) + ) + if status is SourceResultStatus.PROVIDED: + acknowledgements.append( + SourceAcknowledgement( + registration.registration_id, + policy.source, + key, + item.identity, + ) + ) + return PreparedSources(tuple(policies), tuple(results), tuple(acknowledgements)) + + def finalize( + self, + manifest: RequestManifest, + acknowledgements: Sequence[SourceAcknowledgement], + ) -> None: + admitted = { + (outcome.source, outcome.key) + for outcome in manifest.outcomes + if outcome.persistence is FragmentPersistence.HISTORY + and outcome.status in {FragmentStatus.INCLUDED, FragmentStatus.TRUNCATED} + } + by_registration: defaultdict[str, list[SourceAcknowledgement]] = defaultdict(list) + for acknowledgement in acknowledgements: + if (acknowledgement.source, acknowledgement.key) in admitted: + by_registration[acknowledgement.registration_id].append(acknowledgement) + failures: list[Exception] = [] + registrations = { + registration.registration_id: registration + for registration in self._registrations.values() + } + for registration_id, batch in by_registration.items(): + registration = registrations[registration_id] + for acknowledgement in batch: + self._committed.add( + (self._generation, registration_id, acknowledgement.prepared_identity) + ) + try: + registration.provider.acknowledge_injections( + tuple(item.prepared_identity for item in batch) + ) + except Exception as exc: + failures.append(exc) + if failures: + from pythinker_code.telemetry.errors import report_handled_error + + for failure in failures: + report_handled_error(failure, site="soul.request_lifecycle.finalize") + raise RequestLifecycleError("provider_finalization_failed") from failures[0] + + def context_rebuilt(self) -> None: + self._generation += 1 + self._committed.clear() + + def rearm( + self, providers: Sequence[DynamicInjectionProvider], key: str + ) -> tuple[Exception, ...]: + self.sync_providers(providers) + failures: list[Exception] = [] + for provider in providers: + registration = self._registrations[id(provider)] + try: + recognized = provider.rearm(key) + except Exception as exc: + failures.append(exc) + continue + if recognized: + self._committed = { + identity + for identity in self._committed + if identity[1] != registration.registration_id + } + return tuple(failures) + + def _ordered_required( + self, providers: Sequence[DynamicInjectionProvider] + ) -> tuple[_Registration, ...]: + permissions = [ + provider for provider in providers if isinstance(provider, PermissionsInjectionProvider) + ] + defenses = [ + provider + for provider in providers + if isinstance(provider, ModelDefenseInjectionProvider) + ] + if len(permissions) > 1 or len(defenses) > 1: + raise RequestLifecycleError("ambiguous_required_provider") + if not permissions: + if self._default_permissions is None: + raise RequestLifecycleError("required_provider_missing") + permissions = [self._default_permissions] + if not defenses: + if self._default_model_defense is None: + raise RequestLifecycleError("required_provider_missing") + defenses = [self._default_model_defense] + return tuple(self._registrations[id(provider)] for provider in (*permissions, *defenses)) + + def _ordered_optional( + self, providers: Sequence[DynamicInjectionProvider] + ) -> tuple[_Registration, ...]: + return tuple( + self._registrations[id(provider)] + for provider in providers + if not isinstance( + provider, + (PermissionsInjectionProvider, ModelDefenseInjectionProvider), + ) + ) + + +def _source_base(provider: DynamicInjectionProvider) -> str: + if isinstance(provider, PermissionsInjectionProvider): + return _PERMISSIONS_SOURCE + if isinstance(provider, ModelDefenseInjectionProvider): + return _MODEL_DEFENSE_SOURCE + return _stable_identifier(type(provider).__name__.lstrip("_")) + + +def _exactly_one_role( + providers: Sequence[DynamicInjectionProvider], + role: type[DynamicInjectionProvider], +) -> DynamicInjectionProvider | None: + matches = [provider for provider in providers if isinstance(provider, role)] + return matches[0] if len(matches) == 1 else None + + +def _stable_identifier(identifier: str) -> str: + if _SAFE_IDENTIFIER.fullmatch(identifier): + return identifier + normalized = re.sub(r"[^A-Za-z0-9_.:-]+", "_", identifier).strip("_.:-") + prefix = normalized[:48] or "provider" + digest = hashlib.sha256(identifier.encode(encoding="utf-8")).hexdigest()[:12] + return f"{prefix}:{digest}" + + +def _policy(registration: _Registration, key: str) -> TrustedSourcePolicy: + provider = registration.provider + required = isinstance(provider, (PermissionsInjectionProvider, ModelDefenseInjectionProvider)) + if isinstance(provider, PermissionsInjectionProvider): + failure_codes = ("permissions_state_unavailable", "permissions_state_invalid") + applicability = SourceApplicability.ALWAYS + elif isinstance(provider, ModelDefenseInjectionProvider): + failure_codes = ("model_defense_unavailable", "model_defense_invalid") + applicability = SourceApplicability.MAY_BE_NOT_APPLICABLE + else: + failure_codes = ("provider_failed", "provider_identity_invalid") + applicability = SourceApplicability.MAY_BE_NOT_APPLICABLE + return TrustedSourcePolicy( + source=registration.source, + key=key, + requirement=FragmentRequirement.REQUIRED if required else FragmentRequirement.BEST_EFFORT, + persistence=FragmentPersistence.HISTORY, + priority=100, + budget_class=FragmentBudgetClass.BUDGETED, + truncation=FragmentTruncation.FORBIDDEN if required else FragmentTruncation.ALLOWED, + applicability=applicability, + failure_reason_codes=failure_codes, + ) + + +def _not_applicable(policy: TrustedSourcePolicy) -> RequestSourceResult: + return RequestSourceResult( + policy.source, + policy.key, + SourceResultStatus.NOT_APPLICABLE, + None, + None, + ) + + +def _failed(policy: TrustedSourcePolicy, reason: str) -> RequestSourceResult: + return RequestSourceResult(policy.source, policy.key, SourceResultStatus.FAILED, None, reason) + + +def _unavailable_reason(registration: _Registration) -> str: + if isinstance(registration.provider, PermissionsInjectionProvider): + return "permissions_state_unavailable" + if isinstance(registration.provider, ModelDefenseInjectionProvider): + return "model_defense_unavailable" + return "provider_failed" + + +def _invalid_security_type(provider: DynamicInjectionProvider, injection_type: str) -> str | None: + if isinstance(provider, PermissionsInjectionProvider): + return None if injection_type == _PERMISSIONS_SOURCE else "permissions_state_invalid" + if isinstance(provider, ModelDefenseInjectionProvider): + return ( + None + if injection_type.startswith(f"{_MODEL_DEFENSE_SOURCE}:") + else "model_defense_invalid" + ) + return None + + +def failed_manifest(reason_code: str, prior: RequestManifest | None) -> RequestManifest: + """Return a fresh sanitized failed manifest without retaining stale success state.""" + return RequestManifest( + status=RequestStatus.FAILED, + reason_code=_stable_identifier(reason_code), + outcomes=prior.outcomes if prior is not None else (), + budget_tokens=prior.budget_tokens if prior is not None else 0, + budgeted_admitted_tokens=prior.budgeted_admitted_tokens if prior is not None else 0, + non_budgeted_estimated_tokens=( + prior.non_budgeted_estimated_tokens if prior is not None else 0 + ), + ) diff --git a/src/pythinker_code/soul/request_primitives.py b/src/pythinker_code/soul/request_primitives.py new file mode 100644 index 00000000..5d95217d --- /dev/null +++ b/src/pythinker_code/soul/request_primitives.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from pythinker_core.message import Message + +from pythinker_code.notifications import is_notification_message + + +def estimate_injection_tokens(text: str) -> int: + """Estimate request-fragment tokens using the project-wide len/4 heuristic.""" + return max(1, len(text) // 4) + + +def normalize_history(history: Sequence[Message]) -> list[Message]: + """Merge adjacent non-notification user messages without altering other roles.""" + normalized: list[Message] = [] + for message in history: + if _can_merge_user_message(normalized, message): + previous = normalized[-1] + normalized[-1] = Message(role="user", content=[*previous.content, *message.content]) + else: + normalized.append(message) + return normalized + + +def _can_merge_user_message(normalized: Sequence[Message], message: Message) -> bool: + return bool( + normalized + and normalized[-1].role == "user" + and message.role == "user" + and not is_notification_message(normalized[-1]) + and not is_notification_message(message) + ) diff --git a/src/pythinker_code/soul/slash.py b/src/pythinker_code/soul/slash.py index 904ea951..9a6c2ee4 100644 --- a/src/pythinker_code/soul/slash.py +++ b/src/pythinker_code/soul/slash.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import tempfile from collections.abc import Awaitable, Callable from pathlib import Path @@ -15,6 +16,7 @@ from pythinker_code.soul.context import Context from pythinker_code.soul.dynamic_injections.auto_mode import AUTO_DISABLED_REMINDER from pythinker_code.soul.message import system, system_reminder +from pythinker_code.soul.request_assembly import opaque_manifest_identifier from pythinker_code.utils.logging import logger from pythinker_code.utils.path import sanitize_cli_path, shorten_home from pythinker_code.utils.slashcmd import SlashCommandRegistry @@ -32,6 +34,60 @@ """ registry = SlashCommandRegistry[SoulSlashCmdFunc]() +_MANIFEST_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]*") +_MANIFEST_SECRET_IDENTIFIER = re.compile( + r"(?:sk-(?:ant|proj|[A-Za-z0-9])[A-Za-z0-9_-]{16,}|" + r"xox(?:a|b|p|r|s)-[A-Za-z0-9-]{10,}|" + r"gh[pousr]_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16})" +) +_MAX_MANIFEST_IDENTIFIER_LENGTH = 64 + + +def _safe_manifest_reason(identifier: str) -> str: + if ( + 0 < len(identifier) <= _MAX_MANIFEST_IDENTIFIER_LENGTH + and _MANIFEST_IDENTIFIER.fullmatch(identifier) is not None + and _MANIFEST_SECRET_IDENTIFIER.search(identifier) is None + ): + return identifier + return "" + + +def _render_prompt_manifest(soul: PythinkerSoul) -> str: + manifest = soul.latest_request_manifest + if manifest is None: + return "No request has been assembled in this session." + + overall_reason = ( + f" reason={_safe_manifest_reason(manifest.reason_code)}" + if manifest.reason_code is not None + else "" + ) + lines = [ + f"Prompt manifest: {manifest.status.value.upper()}{overall_reason}", + ( + "Budget: " + f"limit={manifest.budget_tokens} " + f"budgeted_admitted={manifest.budgeted_admitted_tokens} " + f"non_budgeted_estimated={manifest.non_budgeted_estimated_tokens}" + ), + "Fragments:", + ] + for outcome in manifest.outcomes: + reason = ( + f" reason={_safe_manifest_reason(outcome.reason_code)}" + if outcome.reason_code is not None + else "" + ) + lines.append( + f"- {opaque_manifest_identifier(outcome.key)} " + f"[{opaque_manifest_identifier(outcome.source)}]: " + f"{outcome.requirement.value} {outcome.persistence.value} {outcome.status.value} " + f"estimated={outcome.estimated_tokens} admitted={outcome.admitted_tokens}{reason}" + ) + if not manifest.outcomes: + lines.append("- none") + return "\n".join(lines) @registry.command @@ -108,6 +164,13 @@ async def recap(soul: PythinkerSoul, args: str) -> None: wire_send(TextPart(text=text)) +@registry.command(name="prompt-manifest") +def prompt_manifest(soul: PythinkerSoul, args: str) -> None: + """Show the latest sanitized request assembly manifest""" + del args + wire_send(TextPart(text=_render_prompt_manifest(soul))) + + @registry.command async def compact(soul: PythinkerSoul, args: str): """Compact the context (optionally with a custom focus, e.g. /compact keep db discussions)""" @@ -131,8 +194,7 @@ async def compact(soul: PythinkerSoul, args: str): async def clear(soul: PythinkerSoul, args: str): """Clear the context""" logger.info("Running `/clear`") - await soul.context.clear() - await soul.context.write_system_prompt(soul.agent.system_prompt) + await soul.clear_context() wire_send(TextPart(text="The context has been cleared.")) snap = soul.status wire_send( diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index a9c4a2fb..bfb8bd77 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -635,11 +635,20 @@ def _rebuild_published_mcp_tools(self, runtime: Runtime) -> None: synchronously (no ``await`` between the drop and the republish), so the two registries are never observed half-rebuilt. """ - stale = [name for name, tool in self._tool_dict.items() if isinstance(tool, MCPTool)] - for name in stale: - del self._tool_dict[name] - runtime.mcp_tools.clear() - self._publish_connected_mcp_tools(runtime) + prior_tools = dict(self._tool_dict) + prior_runtime_tools = dict(runtime.mcp_tools) + try: + stale = [name for name, tool in self._tool_dict.items() if isinstance(tool, MCPTool)] + for name in stale: + del self._tool_dict[name] + runtime.mcp_tools.clear() + self._publish_connected_mcp_tools(runtime) + except Exception: + self._tool_dict.clear() + self._tool_dict.update(prior_tools) + runtime.mcp_tools.clear() + runtime.mcp_tools.update(prior_runtime_tools) + raise def hide(self, tool_name: str) -> bool: """Hide a tool from the LLM tool list. Returns True if the tool exists.""" diff --git a/src/pythinker_code/subagents/catalogue.py b/src/pythinker_code/subagents/catalogue.py new file mode 100644 index 00000000..3bc2d12e --- /dev/null +++ b/src/pythinker_code/subagents/catalogue.py @@ -0,0 +1,398 @@ +"""Immutable resolution of required YAML and optional markdown agent definitions.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field, replace +from enum import StrEnum +from pathlib import Path +from types import MappingProxyType +from typing import cast + +from pydantic import ConfigDict + +from pythinker_code.agentspec import ( + AgentSpecSourceValidation, + ResolvedAgentSpec, + SubagentSpec, + load_agent_spec_validated, + render_agent_field_segment, +) +from pythinker_code.exception import AgentSpecError +from pythinker_code.subagents.discovery import ( + MarkdownAgentSource, + ScopedAgentRoot, + discover_markdown_agent_sources, + materialize_markdown_agent_specs, + parse_markdown_agent, +) +from pythinker_code.utils.frontmatter import MalformedFrontmatterError, parse_frontmatter + +_MARKDOWN_FIELDS = frozenset( + { + "description", + "disallowed_tools", + "exclude_tools", + "max_turns", + "model", + "name", + "required_mcp_servers", + "steps", + "tools", + "when_to_use", + } +) + + +class UnknownFieldPolicy(StrEnum): + WARN = "warn" + FORBID = "forbid" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AgentProvenance: + source_kind: str + source_id: str + scope: str + precedence: int + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AgentDiagnostic: + source_kind: str + safe_path: str + field_path: str | None + severity: str + reason_code: str + message: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResolvedAgentEntry: + name: str + normalized_name: str + description: str + launch_spec: ResolvedAgentSpec + required_mcp_servers: tuple[str, ...] + supports_background: bool + provenance: AgentProvenance + legacy_agent_file: Path | None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResolvedAgentCatalogue: + entries: Mapping[str, ResolvedAgentEntry] + diagnostics: tuple[AgentDiagnostic, ...] = () + _index: Mapping[str, ResolvedAgentEntry] = field(init=False, repr=False) + + def __post_init__(self) -> None: + copied: dict[str, ResolvedAgentEntry] = {} + for entry in self.entries.values(): + normalized = normalize_agent_name(entry.name) + if normalized in copied: + raise ValueError("Agent catalogue contains a normalized name collision") + copied[normalized] = replace( + entry, + normalized_name=normalized, + launch_spec=_freeze_launch_spec(entry.launch_spec), + required_mcp_servers=tuple(entry.required_mcp_servers), + ) + copied = dict(sorted(copied.items())) + frozen = MappingProxyType(copied) + object.__setattr__(self, "entries", frozen) + object.__setattr__(self, "_index", frozen) + object.__setattr__(self, "diagnostics", tuple(self.diagnostics)) + + def get(self, name: str) -> ResolvedAgentEntry | None: + return self._index.get(normalize_agent_name(name)) + + def require(self, name: str) -> ResolvedAgentEntry: + entry = self.get(name) + if entry is None: + raise KeyError(name) + return entry + + def values(self) -> tuple[ResolvedAgentEntry, ...]: + return tuple(self._index.values()) + + +class _FrozenSubagentSpec(SubagentSpec): + model_config = ConfigDict(frozen=True) + + +def normalize_agent_name(name: str) -> str: + return name.casefold() + + +async def resolve_agent_catalogue( + *, + agent_file: Path, + markdown_roots: Iterable[ScopedAgentRoot], + materialized_dir: Path, + available_models: set[str] | None = None, + unknown_field_policy: UnknownFieldPolicy = UnknownFieldPolicy.WARN, +) -> ResolvedAgentCatalogue: + """Resolve required YAML subagents and optional markdown agents once.""" + root_spec, root_validations = load_agent_spec_validated( + agent_file, + forbid_unknown_fields=unknown_field_policy is UnknownFieldPolicy.FORBID, + ) + entries: dict[str, ResolvedAgentEntry] = {} + diagnostics: list[AgentDiagnostic] = [] + reported_yaml_fields: set[tuple[Path, tuple[str, ...]]] = set() + _append_yaml_diagnostics(diagnostics, root_validations, reported_yaml_fields) + + for declared_name, declared_spec in root_spec.subagents.items(): + normalized = normalize_agent_name(declared_name) + if normalized in entries: + raise AgentSpecError("Required YAML agent name collision after normalization") + launch_spec, validations = load_agent_spec_validated( + declared_spec.path, + forbid_unknown_fields=unknown_field_policy is UnknownFieldPolicy.FORBID, + ) + _append_yaml_diagnostics(diagnostics, validations, reported_yaml_fields) + entries[normalized] = ResolvedAgentEntry( + name=declared_name, + normalized_name=normalized, + description=declared_spec.description, + launch_spec=_freeze_launch_spec(launch_spec), + required_mcp_servers=(), + supports_background=not launch_spec.hidden, + provenance=AgentProvenance( + source_kind="yaml", + source_id=f"yaml:required:{_safe_source_token(declared_spec.path)}", + scope="required", + precedence=0, + ), + legacy_agent_file=declared_spec.path, + ) + + discovery_errors: list[tuple[str, str]] = [] + sources = await discover_markdown_agent_sources( + markdown_roots, + on_error=lambda safe_path, reason: discovery_errors.append((safe_path, reason)), + ) + diagnostics.extend( + AgentDiagnostic( + source_kind="markdown", + safe_path=safe_path, + field_path=None, + severity="warning", + reason_code=reason, + message="Optional markdown source could not be read", + ) + for safe_path, reason in discovery_errors + ) + for source in sources: + _resolve_markdown_source( + source=source, + entries=entries, + diagnostics=diagnostics, + materialized_dir=materialized_dir, + available_models=available_models, + unknown_field_policy=unknown_field_policy, + ) + + return ResolvedAgentCatalogue(entries=entries, diagnostics=tuple(diagnostics)) + + +def _append_yaml_diagnostics( + diagnostics: list[AgentDiagnostic], + validations: tuple[AgentSpecSourceValidation, ...], + reported: set[tuple[Path, tuple[str, ...]]], +) -> None: + for validation in validations: + identity = (validation.source_path, validation.field_paths) + if identity in reported: + continue + reported.add(identity) + diagnostics.append( + AgentDiagnostic( + source_kind="yaml", + safe_path=f"required/{_safe_source_token(validation.source_path)}", + field_path=", ".join(validation.field_paths), + severity="warning", + reason_code="unknown_field", + message="Required agent source contains unknown fields", + ) + ) + + +def _resolve_markdown_source( + *, + source: MarkdownAgentSource, + entries: dict[str, ResolvedAgentEntry], + diagnostics: list[AgentDiagnostic], + materialized_dir: Path, + available_models: set[str] | None, + unknown_field_policy: UnknownFieldPolicy, +) -> None: + try: + frontmatter = parse_frontmatter(source.content) or {} + unknown_fields: list[str] = [] + has_invalid_key = False + for key in cast("dict[object, object]", frontmatter): + if isinstance(key, str) and key in _MARKDOWN_FIELDS: + continue + rendered = render_agent_field_segment(key) + unknown_fields.append(rendered.text) + has_invalid_key = has_invalid_key or rendered.structurally_invalid + unknown_fields.sort() + if unknown_fields: + diagnostics.append( + _unknown_markdown_diagnostic( + source, + tuple(unknown_fields), + unknown_field_policy, + invalid_key=has_invalid_key, + ) + ) + if has_invalid_key or unknown_field_policy is UnknownFieldPolicy.FORBID: + return + spec = parse_markdown_agent( + source.content, + prompt_file=source.prompt_file, + scope=source.scope, + ) + except MalformedFrontmatterError: + diagnostics.append( + _source_diagnostic( + source, + severity="warning", + reason_code="invalid_known_field", + message="Optional markdown agent is invalid and was skipped", + ) + ) + return + + normalized = normalize_agent_name(spec.name) + precedence = source.entry_precedence + existing = entries.get(normalized) + if existing is not None: + reason_code = ( + "same_precedence_collision" + if existing.provenance.precedence == precedence + else "shadowed_source" + ) + diagnostics.append( + _source_diagnostic( + source, + severity="warning", + reason_code=reason_code, + message="Optional markdown agent was skipped by catalogue precedence", + ) + ) + return + + try: + definitions = materialize_markdown_agent_specs( + (spec,), + output_dir=materialized_dir, + available_models=available_models, + ) + if not definitions: + diagnostics.append( + _source_diagnostic( + source, + severity="warning", + reason_code="materialization_failure", + message="Optional markdown agent could not be materialized", + ) + ) + return + definition = definitions[0] + launch_spec, _ = load_agent_spec_validated(definition.agent_file) + except (AgentSpecError, OSError): + diagnostics.append( + _source_diagnostic( + source, + severity="warning", + reason_code="materialization_failure", + message="Optional markdown agent could not be materialized", + ) + ) + return + + entries[normalized] = ResolvedAgentEntry( + name=spec.name, + normalized_name=normalized, + description=spec.description, + launch_spec=_freeze_launch_spec(launch_spec), + required_mcp_servers=spec.required_mcp_servers, + supports_background=not launch_spec.hidden, + provenance=AgentProvenance( + source_kind="markdown", + source_id=(f"markdown:{source.scope}:{source.root_ordinal}:{source.prompt_file.name}"), + scope=source.scope, + precedence=precedence, + ), + legacy_agent_file=definition.agent_file, + ) + + +def _unknown_markdown_diagnostic( + source: MarkdownAgentSource, + field_paths: tuple[str, ...], + policy: UnknownFieldPolicy, + *, + invalid_key: bool = False, +) -> AgentDiagnostic: + return AgentDiagnostic( + source_kind="markdown", + safe_path=source.safe_path, + field_path=", ".join(field_paths), + severity=("error" if invalid_key or policy is UnknownFieldPolicy.FORBID else "warning"), + reason_code="invalid_field_key" if invalid_key else "unknown_field", + message=( + "Optional markdown agent contains an invalid field key" + if invalid_key + else "Optional markdown agent contains unknown fields" + ), + ) + + +def _source_diagnostic( + source: MarkdownAgentSource, + *, + severity: str, + reason_code: str, + message: str, +) -> AgentDiagnostic: + return AgentDiagnostic( + source_kind="markdown", + safe_path=source.safe_path, + field_path=None, + severity=severity, + reason_code=reason_code, + message=message, + ) + + +def _freeze_launch_spec(spec: ResolvedAgentSpec) -> ResolvedAgentSpec: + prompt_args = MappingProxyType(dict(spec.system_prompt_args)) + subagents = MappingProxyType( + { + name: _FrozenSubagentSpec( + path=value.path, + description=value.description, + ) + for name, value in spec.subagents.items() + } + ) + return replace( + spec, + system_prompt_args=cast("dict[str, str]", prompt_args), + tools=cast("list[str]", tuple(spec.tools)), + allowed_tools=( + None if spec.allowed_tools is None else cast("list[str]", tuple(spec.allowed_tools)) + ), + exclude_tools=cast("list[str]", tuple(spec.exclude_tools)), + subagents=cast("dict[str, SubagentSpec]", subagents), + ) + + +def _safe_source_token(path: Path) -> str: + resolved_path = str(path.resolve()) + digest = hashlib.sha256(resolved_path.encode(encoding="utf-8")).hexdigest()[:12] + return f"{digest}:{path.name}" diff --git a/src/pythinker_code/subagents/discovery.py b/src/pythinker_code/subagents/discovery.py index bf16c855..78b9aabe 100644 --- a/src/pythinker_code/subagents/discovery.py +++ b/src/pythinker_code/subagents/discovery.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass from pathlib import Path from typing import Any, Literal, cast @@ -37,6 +37,7 @@ class ScopedAgentRoot: root: HostPath scope: AgentScope + precedence: int | None = None @dataclass(frozen=True, slots=True) @@ -51,6 +52,29 @@ class MarkdownAgentSpec: when_to_use: str = "" required_mcp_servers: tuple[str, ...] = () steps: int | None = None + prompt_content: str | None = None + + +@dataclass(frozen=True, slots=True) +class MarkdownAgentSource: + """One canonical markdown file with its trusted discovery-order metadata.""" + + content: str + prompt_file: HostPath + scope: AgentScope + root_ordinal: int + safe_path: str + + @property + def entry_precedence(self) -> int: + """Resolved-catalogue precedence for this source (lower wins). + + ``0`` is reserved for required YAML entries, so markdown sources start + at ``1``. Owning this ``root_ordinal``-to-precedence mapping here keeps + precedence numbering in the discovery layer instead of re-deriving it in + the catalogue. + """ + return self.root_ordinal + 1 def _project_agent_dir_candidates(project_root: HostPath) -> tuple[HostPath, ...]: @@ -68,8 +92,10 @@ async def resolve_agent_roots(work_dir: HostPath) -> list[ScopedAgentRoot]: roots: list[ScopedAgentRoot] = [] seen: set[str] = set() - async def add_existing(candidates: Iterable[HostPath], scope: AgentScope) -> None: - for candidate in candidates: + async def add_existing( + candidates: Iterable[HostPath], scope: AgentScope, precedence_base: int + ) -> None: + for offset, candidate in enumerate(candidates): try: if not await candidate.is_dir(): continue @@ -81,23 +107,57 @@ async def add_existing(candidates: Iterable[HostPath], scope: AgentScope) -> Non if key in seen: continue seen.add(key) - roots.append(ScopedAgentRoot(root=canon, scope=scope)) + roots.append( + ScopedAgentRoot( + root=canon, + scope=scope, + precedence=precedence_base + offset, + ) + ) - await add_existing(_project_agent_dir_candidates(project_root), "project") + await add_existing(_project_agent_dir_candidates(project_root), "project", 1) # Enabled plugins (pythinker/Claude/Codex installs) contribute agent roots # below project scope, so a project-local agent of the same name wins. from pythinker_code.plugin.integration import plugin_agent_dirs plugin_roots = [HostPath.unsafe_from_local_path(d) for d in plugin_agent_dirs()] - await add_existing(plugin_roots, "plugin") + await add_existing(plugin_roots, "plugin", 1_000) return roots async def discover_markdown_agents(roots: Iterable[ScopedAgentRoot]) -> list[MarkdownAgentSpec]: """Discover Claude/Agents-style ``*.md`` subagent definitions.""" by_name: dict[str, MarkdownAgentSpec] = {} - for scoped in roots: + for source in await discover_markdown_agent_sources(roots): + try: + spec = parse_markdown_agent( + source.content, + prompt_file=source.prompt_file, + scope=source.scope, + ) + except ValueError as exc: + logger.info( + "Skipping invalid markdown agent {path}: {error}", + path=source.prompt_file, + error=exc, + ) + continue + by_name.setdefault(spec.name.casefold(), spec) + return sorted(by_name.values(), key=lambda s: s.name) + + +async def discover_markdown_agent_sources( + roots: Iterable[ScopedAgentRoot], + *, + on_error: Callable[[str, str], None] | None = None, +) -> tuple[MarkdownAgentSource, ...]: + """Read canonical markdown sources in deterministic root and filename order.""" + sources: list[MarkdownAgentSource] = [] + seen: set[str] = set() + ordered_roots = sorted(roots, key=_agent_root_order) + for root_ordinal, scoped in enumerate(ordered_roots): + entries: list[HostPath] = [] try: async for entry in scoped.root.iterdir(): if not entry.name.lower().endswith(".md"): @@ -105,23 +165,76 @@ async def discover_markdown_agents(roots: Iterable[ScopedAgentRoot]) -> list[Mar try: if await entry.is_dir(): continue - content = await entry.read_text(encoding="utf-8") - spec = parse_markdown_agent(content, prompt_file=entry, scope=scoped.scope) - except Exception as exc: + except OSError as exc: + if on_error is not None: + on_error( + f"{scoped.scope}[{root_ordinal}]/{entry.name}", + "unreadable_optional_source", + ) logger.info( - "Skipping invalid markdown agent {path}: {error}", + "Skipping unreadable markdown agent {path}: {error}", path=entry, error=exc, ) continue - by_name.setdefault(spec.name.casefold(), spec) + entries.append(entry) except OSError as exc: + if on_error is not None: + on_error( + f"{scoped.scope}[{root_ordinal}]", + "unreadable_optional_root", + ) logger.warning( "Failed to iterate agent directory {path}: {error}", path=scoped.root, error=exc, ) - return sorted(by_name.values(), key=lambda s: s.name) + continue + for entry in sorted(entries, key=lambda item: (item.name.casefold(), item.name)): + try: + canonical = str(entry.canonical()) + if canonical in seen: + continue + content = await entry.read_text(encoding="utf-8") + except OSError as exc: + if on_error is not None: + on_error( + f"{scoped.scope}[{root_ordinal}]/{entry.name}", + "unreadable_optional_source", + ) + logger.info( + "Skipping unreadable markdown agent {path}: {error}", + path=entry, + error=exc, + ) + continue + seen.add(canonical) + sources.append( + MarkdownAgentSource( + content=content, + prompt_file=entry, + scope=scoped.scope, + root_ordinal=root_ordinal, + safe_path=f"{scoped.scope}[{root_ordinal}]/{entry.name}", + ) + ) + return tuple(sources) + + +def _agent_root_order(scoped: ScopedAgentRoot) -> tuple[int, int, str]: + if scoped.precedence is not None: + return (scoped.precedence, 0, "") + root_text = str(scoped.root.canonical()) + if scoped.scope == "plugin": + return (1_000, 1, root_text) + parent_name = Path(root_text).parent.name + project_rank = { + ".pythinker": 1, + ".claude": 2, + ".agents": 3, + ".codex": 4, + }.get(parent_name, 100) + return (project_rank, 1, root_text) def parse_markdown_agent( @@ -167,6 +280,7 @@ def parse_markdown_agent( when_to_use=when_to_use, required_mcp_servers=required_mcp_servers, steps=steps, + prompt_content=content, ) @@ -195,15 +309,18 @@ def materialize_markdown_agent_specs( seen_filenames.add(filename.casefold()) wrapper_path = output_dir / f"{filename}.yaml" prompt_path = output_dir / f"{filename}.system.md" - try: - prompt_text = Path(str(agent.prompt_file)).read_text(encoding="utf-8") - except OSError as exc: - logger.warning( - "Failed to read markdown agent prompt {path}: {error}", - path=agent.prompt_file, - error=exc, - ) - prompt_text = "" + if agent.prompt_content is not None: + prompt_text = agent.prompt_content + else: + try: + prompt_text = Path(str(agent.prompt_file)).read_text(encoding="utf-8") + except OSError as exc: + logger.warning( + "Failed to read markdown agent prompt {path}: {error}", + path=agent.prompt_file, + error=exc, + ) + continue prompt_path.write_text(strip_frontmatter(prompt_text).strip(), encoding="utf-8") payload: dict[str, Any] = { "version": 1, diff --git a/src/pythinker_code/subagents/runner.py b/src/pythinker_code/subagents/runner.py index 383b9b3e..0d7729ac 100644 --- a/src/pythinker_code/subagents/runner.py +++ b/src/pythinker_code/subagents/runner.py @@ -305,7 +305,9 @@ async def run(self, req: ForegroundRunRequest) -> ToolReturnValue: actual_type = prepared.actual_type resumed = prepared.resumed - type_def = self._runtime.labor_market.require_builtin_type(actual_type) + from pythinker_code.soul.agent import require_agent_type_definition + + type_def = require_agent_type_definition(self._runtime, actual_type) launch_spec = prepared.record.launch_spec if req.model is not None: launch_spec = replace( @@ -487,7 +489,9 @@ async def _prepare_instance(self, req: ForegroundRunRequest) -> PreparedInstance ) actual_type = req.requested_type or "coder" - type_def = self._runtime.labor_market.require_builtin_type(actual_type) + from pythinker_code.soul.agent import require_agent_type_definition + + type_def = require_agent_type_definition(self._runtime, actual_type) agent_id = f"a{uuid.uuid4().hex[:8]}" record = self._store.create_instance( agent_id=agent_id, diff --git a/src/pythinker_code/telemetry/metrics.py b/src/pythinker_code/telemetry/metrics.py index 5e4da53c..59d6c533 100644 --- a/src/pythinker_code/telemetry/metrics.py +++ b/src/pythinker_code/telemetry/metrics.py @@ -17,6 +17,13 @@ from opentelemetry import metrics as _metrics from opentelemetry.metrics import Counter, Histogram, Meter +from pythinker_code.soul.request_assembly import ( + FragmentRequirement, + FragmentStatus, + RequestManifest, + opaque_manifest_identifier, +) + # --------------------------------------------------------------------------- # Module-level instrument handles # --------------------------------------------------------------------------- @@ -41,6 +48,11 @@ description="Number of inner steps (LLM calls + tool loops) per turn.", unit="1", ) +request_assembly_duration_seconds: Histogram = _meter.create_histogram( + "pythinker.request_assembly.duration_seconds", + description="Request assembly duration with sanitized admission aggregates.", + unit="s", +) # --- LLM-level --- llm_calls_total: Counter = _meter.create_counter( @@ -103,6 +115,7 @@ def bind(meter: Meter) -> None: """ global _meter global turn_total, turn_duration_seconds, turn_step_count + global request_assembly_duration_seconds global llm_calls_total, llm_duration_seconds, llm_input_tokens, llm_output_tokens global llm_cache_read_tokens, llm_cache_creation_tokens global tool_calls_total, tool_duration_seconds, errors_total @@ -123,6 +136,11 @@ def bind(meter: Meter) -> None: description="Number of inner steps (LLM calls + tool loops) per turn.", unit="1", ) + request_assembly_duration_seconds = meter.create_histogram( + "pythinker.request_assembly.duration_seconds", + description="Request assembly duration with sanitized admission aggregates.", + unit="s", + ) llm_calls_total = meter.create_counter( "pythinker.llm.calls_total", description="Number of LLM API calls.", @@ -183,6 +201,35 @@ def record_turn(*, duration_seconds: float, step_count: int, stop_reason: str) - turn_step_count.record(step_count, attrs) +def record_request_assembly( + manifest: RequestManifest, + *, + duration_seconds: float, +) -> None: + """Record content-free request admission aggregates.""" + outcomes = manifest.outcomes + attrs: dict[str, Any] = { + "source_ids": tuple( + dict.fromkeys(opaque_manifest_identifier(outcome.source) for outcome in outcomes) + ), + "required_count": sum( + outcome.requirement is FragmentRequirement.REQUIRED for outcome in outcomes + ), + "optional_count": sum( + outcome.requirement is FragmentRequirement.BEST_EFFORT for outcome in outcomes + ), + "included_count": sum(outcome.status is FragmentStatus.INCLUDED for outcome in outcomes), + "omitted_count": sum( + outcome.status is FragmentStatus.OMITTED_BUDGET for outcome in outcomes + ), + "truncated_count": sum(outcome.status is FragmentStatus.TRUNCATED for outcome in outcomes), + "degraded_count": sum(outcome.status is FragmentStatus.DEGRADED for outcome in outcomes), + "failed_count": sum(outcome.status is FragmentStatus.FAILED for outcome in outcomes), + "budget_limit": manifest.budget_tokens, + } + request_assembly_duration_seconds.record(duration_seconds, attrs) + + # Model-name → family table. ``gen_ai.system`` only reflects the transport # provider class, so every OpenAI-compatible endpoint (Alibaba DashScope, # Moonshot, Zhipu, …) collapses into "openai" — the family attribute keeps the diff --git a/src/pythinker_code/tools/agent/__init__.py b/src/pythinker_code/tools/agent/__init__.py index eccb12ab..dc978c3c 100644 --- a/src/pythinker_code/tools/agent/__init__.py +++ b/src/pythinker_code/tools/agent/__init__.py @@ -12,7 +12,12 @@ from pythinker_core.tooling import CallableTool2, ToolError, ToolReturnValue from pythinker_code.execution_profiles import resolve_execution_policy -from pythinker_code.soul.agent import Runtime +from pythinker_code.soul.agent import ( + Runtime, + agent_type_definitions, + get_agent_type_definition, + require_agent_type_definition, +) from pythinker_code.soul.toolset import get_current_tool_call_or_none from pythinker_code.subagents.codenames import generate_codename, is_generic_agent_name from pythinker_code.subagents.models import AgentLaunchSpec, AgentTypeDefinition @@ -98,7 +103,7 @@ def _emit_subagent_tool_fallback( SubagentToolFallback( reason=reason, requested_type=requested_type, - available_types=tuple(sorted(runtime.labor_market.builtin_types)), + available_types=tuple(sorted(agent_type_definitions(runtime))), ) ) except Exception as exc: # noqa: BLE001 - observability must not break Agent tool errors @@ -302,7 +307,7 @@ def __init__(self, runtime: Runtime): @staticmethod def _builtin_type_lines(runtime: Runtime) -> str: lines: list[str] = [] - for name, type_def in runtime.labor_market.builtin_types.items(): + for name, type_def in agent_type_definitions(runtime).items(): tool_names = AgentTool._tool_summary(type_def) model = type_def.default_model or "inherit" suffix = ( @@ -364,7 +369,7 @@ def check_required_mcp_servers(self, requested_type: str) -> ToolError | None: type-validation path reports that), or MCP is still loading. Unconfigured/failed required servers once loading settles are surfaced so the model can self-correct. """ - type_def = self._runtime.labor_market.get_builtin_type(requested_type) + type_def = get_agent_type_definition(self._runtime, requested_type) if type_def is None or not type_def.required_mcp_servers: return None snapshot = self._runtime.mcp_status() if self._runtime.mcp_status is not None else None @@ -514,9 +519,9 @@ async def __call__(self, params: Params) -> ToolReturnValue: return ToolError( message=( f"{exc.args[0] if exc.args else exc}." - f"{_did_you_mean(requested_type, self._runtime.labor_market.builtin_types)}" + f"{_did_you_mean(requested_type, agent_type_definitions(self._runtime))}" f" Available types: " - f"{', '.join(sorted(self._runtime.labor_market.builtin_types))}." + f"{', '.join(sorted(agent_type_definitions(self._runtime)))}." ), brief="Invalid subagent type", ) @@ -563,7 +568,7 @@ async def _run_in_background(self, params: Params) -> ToolReturnValue: # the instance was created. params.model is already validated in # __call__, so only check the stored effective_model fallback here. if params.model is None: - type_def = self._runtime.labor_market.require_builtin_type(actual_type) + type_def = require_agent_type_definition(self._runtime, actual_type) effective = record.launch_spec.effective_model or type_def.default_model if effective is not None and effective not in self._runtime.config.models: return ToolError( @@ -579,7 +584,7 @@ async def _run_in_background(self, params: Params) -> ToolReturnValue: created_instance = False if not params.resume: - type_def = self._runtime.labor_market.require_builtin_type(actual_type) + type_def = require_agent_type_definition(self._runtime, actual_type) self._runtime.subagent_store.create_instance( agent_id=agent_id, description=params.description.strip(), @@ -684,9 +689,9 @@ async def _run_in_background(self, params: Params) -> ToolReturnValue: return ToolError( message=( f"{exc.args[0] if exc.args else exc}." - f"{_did_you_mean(requested_type, self._runtime.labor_market.builtin_types)}" + f"{_did_you_mean(requested_type, agent_type_definitions(self._runtime))}" f" Available types: " - f"{', '.join(sorted(self._runtime.labor_market.builtin_types))}." + f"{', '.join(sorted(agent_type_definitions(self._runtime)))}." ), brief="Invalid subagent type", ) @@ -862,14 +867,14 @@ async def __call__(self, params: RunAgentsParams) -> ToolReturnValue: # discovering it mid-loop leaves earlier children running, and a # corrected retry then double-launches them (the orchestration # fingerprint is already approved by that point). - if self._runtime.labor_market.get_builtin_type(requested_type) is None: + if get_agent_type_definition(self._runtime, requested_type) is None: return ToolError( message=( f"Unknown subagent type {requested_type!r} for agent " f"{child.name!r}." - f"{_did_you_mean(requested_type, self._runtime.labor_market.builtin_types)}" + f"{_did_you_mean(requested_type, agent_type_definitions(self._runtime))}" f" Available types: " - f"{', '.join(sorted(self._runtime.labor_market.builtin_types))}." + f"{', '.join(sorted(agent_type_definitions(self._runtime)))}." ), brief="Invalid subagent type", ) @@ -1396,7 +1401,7 @@ async def __call__(self, params: ImplementAndJudgeParams) -> ToolReturnValue: brief="ImplementAndJudge unavailable", ) for subagent_type in ("implementer", "judge"): - if self._runtime.labor_market.get_builtin_type(subagent_type) is None: + if get_agent_type_definition(self._runtime, subagent_type) is None: return ToolError( message=( f"Subagent type {subagent_type!r} is not registered. " diff --git a/src/pythinker_code/tools/skill/__init__.py b/src/pythinker_code/tools/skill/__init__.py index 5fb25852..35d2fb10 100644 --- a/src/pythinker_code/tools/skill/__init__.py +++ b/src/pythinker_code/tools/skill/__init__.py @@ -35,26 +35,37 @@ async def __call__(self, params: Params) -> ToolReturnValue: return ToolError(message="Skill name is required.", brief="Missing skill name") lookup_keys = skill_lookup_keys(skill_name) - skill = None - for key in lookup_keys: - skill = self._runtime.skills.get(key) - if skill is not None: - break + skill = self._runtime.skill_catalog.resolve(skill_name) + if skill is None: + for key in lookup_keys: + skill = self._runtime.skills.get(key) + if skill is not None: + break mcp_match = find_mcp_server_for_skill_name(skill_name, self._runtime.mcp_tools) if skill is None: + diagnostic = self._runtime.skill_catalog.unavailable_diagnostic(skill_name) + if diagnostic is not None: + return ToolError( + message=( + f"status: unavailable\nSkill unavailable: {skill_name}. " + f"{diagnostic.safe_reason}" + ), + brief="Skill unavailable", + ) if mcp_match is not None: server, tools = mcp_match content = mcp_skill_bridge_content(server, tools) return ToolReturnValue( is_error=False, - output=f"skill: {server} (MCP bridge)\n\n{content}", + output=f"status: mcp_fallback\nskill: {server} (MCP bridge)\n\n{content}", message=f"Resolved {skill_name} to MCP server {server}.", display=[], ) - available = ", ".join(sorted(s.name for s in self._runtime.skills.values())) or "(none)" + suggestions = self._runtime.skill_catalog.search(skill_name, limit=5) + suggestion_text = ", ".join(match.skill.name for match in suggestions) or "(none)" mcp_hint = "" if mcp_match is None and self._runtime.mcp_tools: servers = sorted( @@ -65,10 +76,14 @@ async def __call__(self, params: Params) -> ToolReturnValue: } ) if servers: - mcp_hint = f" Connected MCP servers: {', '.join(servers)}." + shown_servers = servers[:5] + omitted = len(servers) - len(shown_servers) + omission = f"; {omitted} omitted" if omitted else "" + mcp_hint = f" Connected MCP servers: {', '.join(shown_servers)}{omission}." return ToolError( message=( - f"Skill not found: {skill_name}. Available skills: {available}.{mcp_hint}" + f"status: not_found\nSkill not found: {skill_name}. " + f"Suggestions: {suggestion_text}.{mcp_hint}" ), brief="Skill not found", ) @@ -76,7 +91,11 @@ async def __call__(self, params: Params) -> ToolReturnValue: content = await read_skill_text_with_local_specialization(skill, self._runtime.skills) if content is None: return ToolError( - message=f"Failed to read skill: {skill.name}", brief="Skill read failed" + message=( + f"status: unavailable\nSkill unavailable: {skill.name}. " + "Skill source could not be read." + ), + brief="Skill unavailable", ) return ToolReturnValue( diff --git a/src/pythinker_code/ui/shell/slash.py b/src/pythinker_code/ui/shell/slash.py index 262be8ff..28b9ebd4 100644 --- a/src/pythinker_code/ui/shell/slash.py +++ b/src/pythinker_code/ui/shell/slash.py @@ -185,8 +185,9 @@ def agents(app: Shell, args: str): if soul is None: return - labor_market = getattr(soul.runtime, "labor_market", None) - builtin_types = getattr(labor_market, "builtin_types", {}) or {} + from pythinker_code.soul.agent import agent_type_definitions + + builtin_types = agent_type_definitions(soul.runtime) type_defs = sorted(builtin_types.values(), key=lambda item: item.name) from pythinker_code.ui.theme import get_tui_tokens, tui_rich_style diff --git a/src/pythinker_code/utils/frontmatter.py b/src/pythinker_code/utils/frontmatter.py index 0c920b74..7d48cbd2 100644 --- a/src/pythinker_code/utils/frontmatter.py +++ b/src/pythinker_code/utils/frontmatter.py @@ -6,12 +6,23 @@ import yaml +class MalformedFrontmatterError(ValueError): + """Frontmatter content is syntactically invalid or not a mapping. + + Subclasses :class:`ValueError` for backward compatibility with callers that + catch ``ValueError``, while letting callers that need to distinguish an + expected "malformed input" skip from an unexpected programming defect catch + this narrower type instead of every ``ValueError``. + """ + + def parse_frontmatter(text: str) -> dict[str, Any] | None: """ Parse YAML frontmatter from a text blob. Raises: - ValueError: If the frontmatter YAML is invalid. + MalformedFrontmatterError: If the frontmatter YAML is invalid or is not + a mapping. This is a ``ValueError`` subclass. """ lines = text.splitlines() if not lines or lines[0].strip() != "---": @@ -32,10 +43,10 @@ def parse_frontmatter(text: str) -> dict[str, Any] | None: try: raw_data: Any = yaml.safe_load(frontmatter) except yaml.YAMLError as exc: - raise ValueError("Invalid frontmatter YAML.") from exc + raise MalformedFrontmatterError("Invalid frontmatter YAML.") from exc if not isinstance(raw_data, dict): - raise ValueError("Frontmatter YAML must be a mapping.") + raise MalformedFrontmatterError("Frontmatter YAML must be a mapping.") return cast(dict[str, Any], raw_data) diff --git a/tasks/lessons.md b/tasks/lessons.md index 60ce6c39..7df5d11e 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -83,12 +83,33 @@ Format: trigger → rule. NOT `.claude/` config. Transcripts showing `~/.pythinker/sessions/` paths are pythinker runs; behavioral fixes belong in the product. +## Typed policy boundaries + +- **When a deep module classifies trusted and untrusted contributions**, represent source lifecycle + (`provided` / `not_applicable` / `failed`) and trusted metadata permissions in the initial typed + contract; identifier-shape validation is sanitization, not source authorization. +- **When a persisted prompt fragment is deduplicated**, scope its committed identity to both the + provider registration and the current history generation; rearm, compaction, and revert must + invalidate the relevant identity, and acknowledgement must happen synchronously only after the + durable history append completes. + ## TUI prompt chrome - **When hiding the first-load editable input row to prevent ghost prompts**, keep the empty card visible: `_turn_starting` and the live-view first-commit gate may suppress editable content, but the top border and `❯` row should remain visible so the prompt bar does not disappear while the agent loads. +- **When routing live preview text through the existing Markdown renderer**, verify unsupported + constructs against the installed library before treating the renderer as a complete cleanup + boundary. Rich renders HTML comments literally, so a preview that must hide them needs a narrow, + fence-aware filter while malformed comments remain visible. +- **When narrowing a regex that strips whole-line delimited blocks (HTML comments, fences)**, a + non-greedy `.*?` between the open and close delimiters can backtrack across an embedded closer and + silently swallow visible text on a mixed line (` text ` collapsed to `""`). + Bound the body with a tempered token `(?:(?!-->).)*?` so a failed end-anchor simply fails the + match. Then derive test assertions from the *anchored* semantics: a line-anchored stripper leaves + a mixed prose+comment line fully intact (markers included), so asserting the markers vanish is + wrong — that was a self-contradictory test spec the implementer correctly blocked on. ## Spec/profile consistency @@ -107,6 +128,10 @@ Format: trigger → rule. ## Verification gates +- **When a repo-required skill is absent from the advertised Codex skill roots**, check the + project-documented legacy skill roots (especially `~/.claude/skills/`) before reporting it as + unavailable; an incomplete root search is not evidence that the skill is missing. + - **When running a gate command (make check, pytest, ruff) through a pipe or in the background**, the pipeline exit code is the LAST command's (e.g. `tail`), and background notifications report that masked code. Never claim diff --git a/tasks/todo.md b/tasks/todo.md index e9bbbec7..029dacf6 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -2,6 +2,116 @@ ## Active +### TUI thinking Markdown and activity motion (2026-07-11) + +- [x] Execute `docs/superpowers/plans/2026-07-11-tui-thinking-markdown-and-activity-motion.md` + with TDD and the Pythinker guard checkpoints. +- [x] Render complete thinking-preview Markdown without leaking top-level HTML comments. +- [x] Keep activity-tree detail text static and reserve coral shimmer for the verb spinner. +- [x] Add the required `CHANGELOG.md` Unreleased entry. +- [x] Run focused UI tests, `make check-pythinker-code`, and `make test-pythinker-code`. +- [x] Run clean-code, test, docs, Pythinker guard, verification, and final diff review passes. + +Acceptance: complete Markdown emphasis renders without delimiters; complete top-level HTML comments +are hidden; malformed Markdown and comments remain readable; fenced literal comment examples remain +visible; activity-tree details do not shimmer; the bottom verb spinner retains its coral shimmer; +reduced-motion and lifecycle-marker contracts remain green. + +#### Review: TUI thinking Markdown and activity motion + +Executed subagent-driven (implementer → task review → fix loop → final whole-branch review) across +four commits `3505c47c..be252dbf` on `a20bfda0`. + +**Resulting behavior (all verified by tests):** +- Complete Markdown: the live thinking preview renders through `render_agent_body`, so `**bold**` + and other emphasis delimiters no longer leak into the preview. +- Complete top-level HTML comments: a line that is entirely a top-level `` comment is + removed before the Markdown renderable is built. +- Malformed/incomplete input: an unterminated `` inside a fenced code block renders verbatim (fence-aware + split via `iter_fence_aware_lines`). +- Mixed prose+comment line: left fully intact (markers included) — the stripper is line-anchored, so + it only removes whole-line comments and never deletes visible prose. +- Stable tree rows: every activity-tree running detail renders with `shell_style(ShellTone.MUTED)`; + the lifecycle marker running pulse (`blink_visible`) is unchanged. +- Verb-spinner shimmer: `activity_status_line` / `_todo_activity_line` remain the only coral verb + shimmer, including reduced-motion and no-color fallbacks. + +**Two confirmed root causes:** (1) `_ContentBlock._compose_thinking_stream` built a plain +`Text(preview, ...)`, bypassing Markdown; (2) `render_activity_tree` called `shimmer_text` on each +`running` detail, animating one tree row. + +**Evidence (fresh terminal output):** +- Focused TUI set (6 modules): `178 passed, 1 warning` (the warning is pre-existing pytest temp-dir + cleanup from unrelated `knowledge_base` tests, not this change). +- Full package unit `tests`: `6927 passed, 9 skipped, 1 xfailed`. +- Separate `tests_e2e`: `65 passed, 4 skipped`. `make test-pythinker-code` exit 0, no failures. +- Static gate `make check-pythinker-code`: ruff `All checks passed!`, `1250 files already formatted`, + pyright `0 errors, 0 warnings, 0 informations`, ty clean, `All checks passed!`. +- `git diff --check`: clean (no output). + +**Quality-review verdicts:** Task 1 review — spec ✅, one Important plan-mandated regex bug +(non-greedy `.*?` backtracking swallowed visible prose on a mixed line); fixed (tempered token +`(?:(?!-->).)*?`) + regression test, re-review ✅. Task 2 review — ✅ approved, no issues (RED +corroborated: `#c68d7e` = `activity_verb`). Task 3 docs review — ✅ accurate & truthful. Final +whole-branch review (Opus, C01–C15 + failure-truthfulness) — **Ready to merge: Yes**, zero +Critical/Important/Minor. + +**Approved deviations:** the plan-mandated comment regex was tightened with maintainer approval to +stop same-line silent prose deletion. **Remaining blockers:** none. A line of two adjacent complete +comments with zero prose between them (``) is now preserved — an accepted +safe-direction narrowing (under-stripping a marker beats deleting prose), pinned by test. + +### Agent core deepening program (2026-07-10) + +- [x] Execute `docs/superpowers/plans/2026-07-10-agent-core-deepening.md` on + `feat/agent-core-deepening` using TDD and subagent-driven task reviews. +- [x] Phase 1: characterize provider handoff, static prompt, JSONL, agent projections, + and Toolset facade behavior. +- [x] Phase 2: ship bounded `SkillCatalog` discovery with exhaustive compatibility. +- [x] Phase 3: ship observable request assembly and `/prompt-manifest`. +- [x] Phase 4: ship transactional Context replacement and disk-first appends. +- [x] Phase 5: ship WARN-mode resolved agent catalogue with strict-mode tests. +- [x] Phase 6: characterize Toolset, record thresholds, and extract only if measured. +- [x] Run full guards, gates, two-axis review, and document the final result here. + +#### Review: agent core deepening + +- Outcome: all six approved phases shipped on the umbrella branch. Bounded skill + discovery, observable request assembly, transactional Context persistence, and + resolved agent definitions retain their documented compatibility projections. + `/prompt-manifest` exposes only sanitized in-memory assembly metadata. +- Toolset decision: NO-GO on private extraction. The controlled execution-pipeline + attempt worsened median framework overhead from 99.568352% to 99.606995% and was + reverted. The reproducible schema-v2 report contains 18 scenarios and seven + command-generated decisions. Five true 500-tool registry p95 values were + 3.245000, 2.934625, 3.091542, 4.716500, and 3.079834 ms; none crossed 5 ms. +- Deviation: deterministic fault characterization exposed an exception-atomicity + defect in MCP registry publication. The surgical rollback preserves the previous + registry and re-raises the original registration failure. Final review also + replaced mock state with real `CompactionResult` and `TaskView` instances and + bounded optimistic revert conflicts to three attempts before surfacing the typed + generation conflict. +- Compatibility windows: `Runtime.skills` remains until internal exact lookups have + migrated and repository search proves it removable. Unknown agent fields warn in + this release and become errors in the following minor release. `LaborMarket`, + `AgentTypeDefinition`, and generated Markdown wrappers remain through that + strict-default release; their earliest removal is the next minor release, subject + to direct-launch parity and migration checks. +- Verification: `make check-pythinker-code` passed Ruff, formatting, Pyright, and ty; + `make test-pythinker-code` exited 0 after collecting 6,932 package tests and then + passed 65 E2E tests with four skips; provider snapshots passed 39 tests; the + Toolset/fault matrix passed 96 tests; focused guard fixes passed 32 tests; and + `git diff --check` passed. Expected Loguru/Python deprecation and pytest temporary + cleanup warnings remain non-blocking. +- Review: Task 13 rereview approved the reproducible decision builder, true p95 + samples, handle-level cancellation recovery, and MCP observability. Final Standards + and Spec reviews found no Critical/Important implementation defect; their release + metadata blockers were resolved here. Pythinker, clean-code, test, and docs guard + passes found no remaining ship blocker. +- Blockers: none. + ### Plan: publishable benchmark comparison (2026-07-05) - [ ] Execute `docs/superpowers/plans/2026-07-05-publishable-benchmark-comparison.md` diff --git a/tests/conftest.py b/tests/conftest.py index 7338190a..70fde276 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -243,6 +243,9 @@ def runtime( environment: Environment, ) -> Runtime: """Create a Runtime instance.""" + from pythinker_code.skill import SkillCatalog + + skill_catalog = SkillCatalog({}, ()) notifications = NotificationManager( session.context_file.parent / "notifications", config.notifications ) @@ -261,7 +264,10 @@ def runtime( config.background, notifications=notifications, ), - skills={}, + skill_catalog=skill_catalog, + skills=skill_catalog.exhaustive_mapping(), + agent_catalogue=None, + agent_type_projection=None, oauth=OAuthManager(config), additional_dirs=[], skills_dirs=[], diff --git a/tests/core/test_agent_catalogue_compat.py b/tests/core/test_agent_catalogue_compat.py new file mode 100644 index 00000000..d38bf848 --- /dev/null +++ b/tests/core/test_agent_catalogue_compat.py @@ -0,0 +1,413 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace +from typing import cast +from unittest.mock import patch + +import pytest +from pythinker_core.message import Message +from pythinker_core.tooling import ToolError + +from pythinker_code.agentspec import DEFAULT_AGENT_FILE, load_agent_spec +from pythinker_code.background.models import ( + TaskConsumerState, + TaskControl, + TaskRuntime, + TaskSpec, + TaskView, +) +from pythinker_code.soul.agent import ( + Agent as SoulAgent, +) +from pythinker_code.soul.agent import ( + Runtime, + agent_type_definitions, + get_agent_type_definition, + load_agent, +) +from pythinker_code.soul.context import Context +from pythinker_code.soul.dynamic_injections.agent_list import AgentListInjectionProvider +from pythinker_code.soul.pythinkersoul import PythinkerSoul +from pythinker_code.subagents.models import AgentTypeDefinition, ToolPolicy +from pythinker_code.subagents.runner import ForegroundRunRequest, ForegroundSubagentRunner +from pythinker_code.tools.agent import AgentTool +from pythinker_code.ui.shell import Shell +from pythinker_code.wire.types import MCPServerSnapshot, MCPStatusSnapshot, TextPart + + +def _write_yaml_agent_pair( + tmp_path: Path, + *, + default_model: str | None = "model-a", +) -> tuple[Path, Path]: + (tmp_path / "root.md").write_text("root prompt", encoding="utf-8") + (tmp_path / "child.md").write_text("child prompt", encoding="utf-8") + child = tmp_path / "child.yaml" + child.write_text( + "version: 1\n" + "agent:\n" + " name: child-runtime-name\n" + " system_prompt_path: ./child.md\n" + " when_to_use: Use for compatibility checks.\n" + + (f" model: {default_model}\n" if default_model is not None else "") + + " hidden: true\n" + " tools: []\n" + " allowed_tools: [pythinker_code.tools.think:Think]\n", + encoding="utf-8", + ) + root = tmp_path / "root.yaml" + root.write_text( + "version: 1\n" + "agent:\n" + " name: root\n" + " system_prompt_path: ./root.md\n" + " tools: []\n" + " subagents:\n" + " Analyst:\n" + " path: ./child.yaml\n" + " description: Compatibility analyst\n", + encoding="utf-8", + ) + return root, child + + +async def test_load_agent_publishes_exact_warn_catalogue_projection( + runtime: Runtime, + tmp_path: Path, +) -> None: + root, child = _write_yaml_agent_pair(tmp_path) + + await load_agent(root, runtime, mcp_configs=[]) + + catalogue = runtime.agent_catalogue + assert catalogue is not None + entry = catalogue.require("analyst") + assert catalogue.require("ANALYST") is entry + assert runtime.labor_market.require_builtin_type("Analyst") == AgentTypeDefinition( + name="Analyst", + description="Compatibility analyst", + agent_file=child, + when_to_use="Use for compatibility checks.", + default_model="model-a", + tool_policy=ToolPolicy( + mode="allowlist", + tools=("pythinker_code.tools.think:Think",), + ), + supports_background=False, + required_mcp_servers=(), + ) + assert runtime.labor_market.get_builtin_type("analyst") is None + + +async def test_root_and_child_runtimes_share_catalogue_identity( + runtime: Runtime, +) -> None: + await load_agent(DEFAULT_AGENT_FILE, runtime, mcp_configs=[]) + + child = runtime.copy_for_subagent(agent_id="a1", subagent_type="coder") + + assert child.agent_catalogue is runtime.agent_catalogue + assert child.agent_type_projection is runtime.agent_type_projection + assert child.labor_market is runtime.labor_market + + +async def test_empty_markdown_discovery_preserves_wrapper_directory(runtime: Runtime) -> None: + await load_agent(DEFAULT_AGENT_FILE, runtime, mcp_configs=[]) + + assert (runtime.session.dir / "external_agents").is_dir() + + +async def test_markdown_catalogue_projection_keeps_generated_wrapper_parity( + runtime: Runtime, +) -> None: + work_dir = Path(str(runtime.work_dir)) + markdown = work_dir / ".pythinker" / "agents" / "worker.md" + markdown.parent.mkdir(parents=True) + markdown.write_text( + "---\n" + "name: Worker\n" + "description: Compatibility worker\n" + "model: model-a\n" + "when_to_use: Use for wrapper parity.\n" + "tools: [Read]\n" + "required_mcp_servers: [database]\n" + "---\n" + "Follow the compatibility prompt.\n", + encoding="utf-8", + ) + + await load_agent(DEFAULT_AGENT_FILE, runtime, mcp_configs=[]) + + catalogue = runtime.agent_catalogue + assert catalogue is not None + entry = catalogue.require("worker") + type_def = runtime.labor_market.require_builtin_type("Worker") + assert type_def.agent_file == entry.legacy_agent_file + wrapper_spec = load_agent_spec(type_def.agent_file) + assert wrapper_spec.name == entry.launch_spec.name + assert wrapper_spec.system_prompt_path == entry.launch_spec.system_prompt_path + assert wrapper_spec.system_prompt_args == entry.launch_spec.system_prompt_args + assert tuple(wrapper_spec.tools) == tuple(entry.launch_spec.tools) + assert tuple(wrapper_spec.allowed_tools or ()) == tuple(entry.launch_spec.allowed_tools or ()) + assert tuple(wrapper_spec.exclude_tools) == tuple(entry.launch_spec.exclude_tools) + assert wrapper_spec.model == entry.launch_spec.model + assert wrapper_spec.steps == entry.launch_spec.steps + assert type_def.required_mcp_servers == ("database",) + assert type_def.tool_policy == ToolPolicy( + mode="allowlist", + tools=("pythinker_code.tools.file:ReadFile",), + ) + + +async def test_markdown_compatibility_projection_uses_global_display_name_order( + runtime: Runtime, +) -> None: + agents_dir = Path(str(runtime.work_dir)) / ".pythinker" / "agents" + agents_dir.mkdir(parents=True) + (agents_dir / "a.md").write_text( + "---\nname: Zed\ndescription: last by display name\n---\nZed prompt\n", + encoding="utf-8", + ) + (agents_dir / "b.md").write_text( + "---\nname: Alpha\ndescription: first by display name\n---\nAlpha prompt\n", + encoding="utf-8", + ) + + await load_agent(DEFAULT_AGENT_FILE, runtime, mcp_configs=[]) + + projected_names = tuple(agent_type_definitions(runtime)) + assert projected_names[-2:] == ("Alpha", "Zed") + assert tuple(runtime.labor_market.builtin_types)[-2:] == ("Alpha", "Zed") + + +async def test_catalogue_projection_is_immutable_when_labor_market_diverges( + runtime: Runtime, +) -> None: + await load_agent(DEFAULT_AGENT_FILE, runtime, mcp_configs=[]) + before = agent_type_definitions(runtime) + coder = get_agent_type_definition(runtime, "CODER") + replacement = AgentTypeDefinition( + name="coder", + description="mutated compatibility adapter", + agent_file=Path("mutated.yaml"), + ) + + runtime.labor_market.add_builtin_type(replacement) + runtime.labor_market.add_builtin_type( + AgentTypeDefinition( + name="late-only", + description="late compatibility entry", + agent_file=Path("late.yaml"), + ) + ) + + assert agent_type_definitions(runtime) is before + assert get_agent_type_definition(runtime, "coder") is coder + assert "late-only" not in agent_type_definitions(runtime) + assert runtime.labor_market.require_builtin_type("coder") is replacement + assert runtime.labor_market.require_builtin_type("late-only").name == "late-only" + + +async def test_populated_catalogue_drives_casefolded_foreground_launch( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root, child_file = _write_yaml_agent_pair(tmp_path, default_model=None) + await load_agent(root, runtime, mcp_configs=[]) + + async def complete_soul( + soul: PythinkerSoul, + _prompt: str, + _ui_loop_fn: object, + _cancel_event: object, + **_kwargs: object, + ) -> None: + await soul.context.append_message( + Message(role="assistant", content=[TextPart(text="completed catalogue launch")]) + ) + + monkeypatch.setattr("pythinker_code.subagents.runner.run_soul", complete_soul) + result = await ForegroundSubagentRunner(runtime).run( + ForegroundRunRequest( + description="launch analyst", + prompt="perform compatibility analysis", + requested_type="aNaLySt", + model=None, + resume=None, + ) + ) + + assert not result.is_error + assert "completed catalogue launch" in result.output + projected = get_agent_type_definition(runtime, "ANALYST") + assert projected is not None + assert projected.agent_file == child_file + assert projected.supports_background is False + assert projected.tool_policy.tools == ("pythinker_code.tools.think:Think",) + + +async def test_populated_catalogue_drives_agent_list_injection( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root, _ = _write_yaml_agent_pair(tmp_path, default_model=None) + agent = await load_agent(root, runtime, mcp_configs=[]) + soul = PythinkerSoul( + agent, + context=Context(file_backend=tmp_path / "catalogue-agent-list.jsonl"), + ) + monkeypatch.setattr( + "pythinker_code.soul.dynamic_injections.agent_list.wire_send", + lambda _message: None, + ) + injections = await AgentListInjectionProvider().get_injections([], soul) + assert len(injections) == 1 + assert "`Analyst`: Compatibility analyst" in injections[0].content + + +async def _load_markdown_worker(runtime: Runtime) -> SoulAgent: + work_dir = Path(str(runtime.work_dir)) + markdown = work_dir / ".pythinker" / "agents" / "worker.md" + markdown.parent.mkdir(parents=True) + markdown.write_text( + "---\n" + "name: Worker\n" + "description: Catalogue worker\n" + "tools: [Read]\n" + "required_mcp_servers: [database]\n" + "---\n" + "Use the generated wrapper.\n", + encoding="utf-8", + ) + return await load_agent(DEFAULT_AGENT_FILE, runtime, mcp_configs=[]) + + +async def test_populated_markdown_catalogue_drives_required_mcp_gate(runtime: Runtime) -> None: + await _load_markdown_worker(runtime) + tool = AgentTool(runtime) + runtime.mcp_status = lambda: MCPStatusSnapshot( + loading=False, + connected=0, + total=1, + tools=0, + servers=(MCPServerSnapshot(name="database", status="failed"),), + ) + + mcp_error = tool.check_required_mcp_servers("wOrKeR") + assert isinstance(mcp_error, ToolError) + assert "database" in mcp_error.message + + +async def test_populated_markdown_catalogue_drives_casefolded_background_launch( + runtime: Runtime, + monkeypatch: pytest.MonkeyPatch, +) -> None: + await _load_markdown_worker(runtime) + tool = AgentTool(runtime) + runtime.mcp_status = lambda: MCPStatusSnapshot( + loading=False, + connected=1, + total=1, + tools=1, + servers=(MCPServerSnapshot(name="database", status="connected"),), + ) + created: list[dict[str, object]] = [] + + def create_agent_task(**kwargs: object) -> TaskView: + created.append(kwargs) + return TaskView( + spec=TaskSpec( + id="catalogue-task", + kind="agent", + session_id=runtime.session.id, + description="worker", + tool_call_id="test", + ), + runtime=TaskRuntime(status="starting"), + control=TaskControl(), + consumer=TaskConsumerState(), + ) + + monkeypatch.setattr(runtime.background_tasks, "create_agent_task", create_agent_task) + from pythinker_code.soul.toolset import current_tool_call + from pythinker_code.wire.types import ToolCall + + token = current_tool_call.set( + ToolCall(id="test", function=ToolCall.FunctionBody(name="Agent", arguments="{}")) + ) + try: + result = await tool( + tool.params( + description="launch worker", + prompt="use the generated wrapper", + subagent_type="wOrKeR", + run_in_background=True, + ) + ) + finally: + current_tool_call.reset(token) + + assert not result.is_error + assert created and created[0]["subagent_type"] == "wOrKeR" + + +async def test_populated_markdown_catalogue_keeps_wrapper_and_tool_policy(runtime: Runtime) -> None: + await _load_markdown_worker(runtime) + worker = get_agent_type_definition(runtime, "worker") + assert worker is not None + assert worker.agent_file.name.endswith(".yaml") + assert worker.tool_policy == ToolPolicy( + mode="allowlist", + tools=("pythinker_code.tools.file:ReadFile",), + ) + + +async def test_populated_markdown_catalogue_drives_agents_slash_output( + runtime: Runtime, + capsys: pytest.CaptureFixture[str], +) -> None: + agent = await _load_markdown_worker(runtime) + from pythinker_code.ui.shell.slash import registry as shell_slash_registry + + command = shell_slash_registry.find_command("agents") + assert command is not None + slash_soul = PythinkerSoul( + agent, + context=Context(file_backend=Path(str(runtime.work_dir)) / "catalogue-slash-context.jsonl"), + ) + command.func(cast("Shell", SimpleNamespace(soul=slash_soul)), "") + rendered = capsys.readouterr().out + assert "Worker" in rendered + assert "allow 1" in rendered + + +async def test_warn_diagnostics_surface_once_without_raw_source_details( + runtime: Runtime, + tmp_path: Path, +) -> None: + root, _ = _write_yaml_agent_pair(tmp_path) + text = root.read_text(encoding="utf-8") + root.write_text( + text.replace(" tools: []\n", " tools: []\n future_option: SECRET\n"), encoding="utf-8" + ) + + with patch("pythinker_code.soul.agent.logger") as mocked_logger: + await load_agent(root, runtime, mcp_configs=[]) + await load_agent(root, runtime, mcp_configs=[]) + + catalogue = runtime.agent_catalogue + assert catalogue is not None + assert [diagnostic.severity for diagnostic in catalogue.diagnostics] == ["warning"] + warnings = [ + call + for call in mocked_logger.warning.call_args_list + if call.args and str(call.args[0]).startswith("Agent definition") + ] + assert len(warnings) == 1 + rendered = repr(warnings) + assert "future_option" in rendered + assert "SECRET" not in rendered + assert str(tmp_path) not in rendered diff --git a/tests/core/test_agent_catalogue_markdown.py b/tests/core/test_agent_catalogue_markdown.py new file mode 100644 index 00000000..c7a4d590 --- /dev/null +++ b/tests/core/test_agent_catalogue_markdown.py @@ -0,0 +1,381 @@ +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest +from pythinker_host.path import HostPath + +from pythinker_code.subagents.catalogue import UnknownFieldPolicy, resolve_agent_catalogue +from pythinker_code.subagents.discovery import AgentScope, MarkdownAgentSource, ScopedAgentRoot + + +def _root_agent(tmp_path: Path) -> Path: + root = tmp_path / "root.yaml" + (tmp_path / "system.md").write_text("root", encoding="utf-8") + root.write_text( + "version: 1\nagent:\n name: root\n system_prompt_path: ./system.md\n tools: []\n", + encoding="utf-8", + ) + return root + + +def _markdown_root(path: Path, scope: AgentScope = "project") -> ScopedAgentRoot: + path.mkdir(parents=True, exist_ok=True) + return ScopedAgentRoot( + root=HostPath.unsafe_from_local_path(path), + scope=scope, + ) + + +def _write_markdown(path: Path, *, name: str, description: str, extra: str = "") -> None: + path.write_text( + f"---\nname: {name}\ndescription: {description}\n{extra}---\nPrompt for {name}\n", + encoding="utf-8", + ) + + +@pytest.mark.asyncio +async def test_markdown_warns_once_then_preserves_resolved_launch_semantics(tmp_path: Path) -> None: + agents = _markdown_root(tmp_path / ".pythinker" / "agents") + _write_markdown( + Path(str(agents.root)) / "worker.md", + name="Worker", + description="worker", + extra=( + "tools: [Read, Bash]\n" + "disallowed_tools: [Bash]\n" + "required_mcp_servers: [db]\n" + "max_turns: 4\n" + "unknown_beta: SECRET_VALUE\n" + "unknown_alpha: other\n" + ), + ) + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + unknown_field_policy=UnknownFieldPolicy.WARN, + ) + + entry = catalogue.require("worker") + assert entry.required_mcp_servers == ("db",) + assert entry.launch_spec.steps == 4 + assert tuple(entry.launch_spec.allowed_tools or ()) == ( + "pythinker_code.tools.file:ReadFile", + "pythinker_code.tools.shell:Shell", + ) + assert tuple(entry.launch_spec.exclude_tools) == ("pythinker_code.tools.shell:Shell",) + assert [d.field_path for d in catalogue.diagnostics] == ["unknown_alpha, unknown_beta"] + rendered = repr(catalogue.diagnostics) + assert "SECRET_VALUE" not in rendered + assert str(tmp_path) not in rendered + + +@pytest.mark.asyncio +async def test_markdown_forbid_skips_only_invalid_optional_entry(tmp_path: Path) -> None: + agents = _markdown_root(tmp_path / "agents") + local = Path(str(agents.root)) + _write_markdown(local / "bad.md", name="bad", description="bad", extra="typo: secret\n") + _write_markdown(local / "good.md", name="good", description="good") + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + unknown_field_policy=UnknownFieldPolicy.FORBID, + ) + + assert [entry.name for entry in catalogue.values()] == ["good"] + assert [(d.field_path, d.severity) for d in catalogue.diagnostics] == [("typo", "error")] + + +@pytest.mark.asyncio +async def test_markdown_same_precedence_is_deterministic_first_wins_with_warning( + tmp_path: Path, +) -> None: + agents = _markdown_root(tmp_path / "agents") + local = Path(str(agents.root)) + _write_markdown(local / "z.md", name="helper", description="second") + _write_markdown(local / "a.md", name="HELPER", description="first") + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + ) + + assert catalogue.require("helper").description == "first" + assert [d.reason_code for d in catalogue.diagnostics] == ["same_precedence_collision"] + + +@pytest.mark.asyncio +async def test_higher_precedence_markdown_shadows_lower_source(tmp_path: Path) -> None: + high = _markdown_root(tmp_path / "high") + low = _markdown_root(tmp_path / "low", "plugin") + _write_markdown(Path(str(high.root)) / "helper.md", name="helper", description="project") + _write_markdown(Path(str(low.root)) / "helper.md", name="helper", description="plugin") + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(high, low), + materialized_dir=tmp_path / "generated", + ) + + assert catalogue.require("HELPER").description == "project" + assert [d.reason_code for d in catalogue.diagnostics] == ["shadowed_source"] + + +@pytest.mark.asyncio +async def test_reversed_roots_keep_project_precedence_and_catalogue_order(tmp_path: Path) -> None: + project = _markdown_root(tmp_path / ".claude" / "agents") + plugin = _markdown_root(tmp_path / "plugin" / "agents", "plugin") + _write_markdown( + Path(str(project.root)) / "helper.md", + name="helper", + description="project", + ) + _write_markdown( + Path(str(plugin.root)) / "helper.md", + name="HELPER", + description="plugin", + ) + + forward = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(project, plugin), + materialized_dir=tmp_path / "forward", + ) + reversed_catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(plugin, project), + materialized_dir=tmp_path / "reversed", + ) + + assert forward.require("helper").description == "project" + assert reversed_catalogue.require("helper").description == "project" + assert [entry.normalized_name for entry in forward.values()] == [ + entry.normalized_name for entry in reversed_catalogue.values() + ] + + +@pytest.mark.asyncio +async def test_all_supported_markdown_fields_and_aliases_survive_launch_projection( + tmp_path: Path, +) -> None: + agents = _markdown_root(tmp_path / "agents") + _write_markdown( + Path(str(agents.root)) / "worker.md", + name="worker", + description="worker", + extra=( + "model: model-a\n" + "when_to_use: use deliberately\n" + "tools: [Read]\n" + "exclude_tools: [Write]\n" + "steps: 6\n" + "required_mcp_servers: [db]\n" + ), + ) + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + available_models={"model-a"}, + ) + + entry = catalogue.require("worker") + assert catalogue.diagnostics == () + assert entry.launch_spec.model == "model-a" + assert entry.launch_spec.when_to_use == "use deliberately" + assert entry.launch_spec.steps == 6 + assert tuple(entry.launch_spec.allowed_tools or ()) == ("pythinker_code.tools.file:ReadFile",) + assert tuple(entry.launch_spec.exclude_tools) == ("pythinker_code.tools.file:WriteFile",) + assert entry.required_mcp_servers == ("db",) + + +@pytest.mark.asyncio +async def test_malformed_markdown_isolated_with_safe_diagnostic(tmp_path: Path) -> None: + agents = _markdown_root(tmp_path / "agents") + local = Path(str(agents.root)) + (local / "bad.md").write_text("---\nname: [unterminated\n---\n", encoding="utf-8") + _write_markdown(local / "good.md", name="good", description="good") + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + ) + + assert [entry.name for entry in catalogue.values()] == ["good"] + assert [d.reason_code for d in catalogue.diagnostics] == ["invalid_known_field"] + assert str(tmp_path) not in repr(catalogue.diagnostics) + + +@pytest.mark.asyncio +async def test_catalogue_materializes_captured_markdown_content_without_rereading_source( + tmp_path: Path, +) -> None: + prompt_file = tmp_path / "worker.md" + prompt_file.write_text("MUTATED AFTER DISCOVERY", encoding="utf-8") + source = MarkdownAgentSource( + content="---\nname: worker\ndescription: worker\n---\nCAPTURED BODY", + prompt_file=HostPath.unsafe_from_local_path(prompt_file), + scope="project", + root_ordinal=0, + safe_path="project[0]/worker.md", + ) + + async def captured_sources( + *_args: object, **_kwargs: object + ) -> tuple[MarkdownAgentSource, ...]: + return (source,) + + with patch( + "pythinker_code.subagents.catalogue.discover_markdown_agent_sources", + captured_sources, + ): + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(), + materialized_dir=tmp_path / "generated", + ) + + launch_prompt = catalogue.require("worker").launch_spec.system_prompt_path + assert launch_prompt.read_text(encoding="utf-8") == "CAPTURED BODY" + + +@pytest.mark.asyncio +async def test_materialization_io_failure_skips_optional_entry_with_diagnostic( + tmp_path: Path, +) -> None: + agents = _markdown_root(tmp_path / "agents") + _write_markdown(Path(str(agents.root)) / "worker.md", name="worker", description="worker") + blocked_output = tmp_path / "blocked" + blocked_output.write_text("not a directory", encoding="utf-8") + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=blocked_output, + ) + + assert catalogue.values() == () + assert [d.reason_code for d in catalogue.diagnostics] == ["materialization_failure"] + + +@pytest.mark.asyncio +async def test_unexpected_materialization_value_error_propagates( + tmp_path: Path, +) -> None: + agents = _markdown_root(tmp_path / "agents") + _write_markdown(Path(str(agents.root)) / "worker.md", name="worker", description="worker") + + with ( + patch( + "pythinker_code.subagents.catalogue.materialize_markdown_agent_specs", + side_effect=ValueError("programming defect"), + ), + pytest.raises(ValueError, match="programming defect"), + ): + await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + ) + + +@pytest.mark.asyncio +async def test_unexpected_parse_value_error_is_not_mislabeled_as_config_skip( + tmp_path: Path, +) -> None: + # A programming defect surfacing as a plain ValueError from the parser must + # propagate, not be swallowed as a harmless "invalid_known_field" skip. Only + # malformed frontmatter (a MalformedFrontmatterError) is a legitimate skip. + agents = _markdown_root(tmp_path / "agents") + _write_markdown(Path(str(agents.root)) / "worker.md", name="worker", description="worker") + + with ( + patch( + "pythinker_code.subagents.catalogue.parse_markdown_agent", + side_effect=ValueError("parser defect"), + ), + pytest.raises(ValueError, match="parser defect"), + ): + await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + ) + + +@pytest.mark.asyncio +async def test_unsafe_markdown_keys_are_isolated_without_raw_key_or_value( + tmp_path: Path, +) -> None: + agents = _markdown_root(tmp_path / "agents") + local = Path(str(agents.root)) + (local / "bad.md").write_text( + "---\nname: bad\n42: value\nMY_SECRET_TOKEN: do-not-leak\n---\nBody", + encoding="utf-8", + ) + _write_markdown(local / "good.md", name="good", description="good") + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + ) + + assert [entry.name for entry in catalogue.values()] == ["good"] + rendered = repr(catalogue.diagnostics) + assert "MY_SECRET_TOKEN" not in rendered + assert "do-not-leak" not in rendered + assert "field[" in rendered + + +@pytest.mark.asyncio +async def test_markdown_warn_redacts_sensitive_string_unknown_fields_and_loads_entry( + tmp_path: Path, +) -> None: + agents = _markdown_root(tmp_path / "agents") + (Path(str(agents.root)) / "worker.md").write_text( + "---\n" + "name: worker\n" + "description: worker\n" + "auth_strategy: ignored\n" + "MY_SECRET_TOKEN: do-not-leak\n" + "---\nBody", + encoding="utf-8", + ) + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents,), + materialized_dir=tmp_path / "generated", + unknown_field_policy=UnknownFieldPolicy.WARN, + ) + + assert [entry.name for entry in catalogue.values()] == ["worker"] + rendered = repr(catalogue.diagnostics) + assert "auth_strategy" not in rendered + assert "MY_SECRET_TOKEN" not in rendered + assert "do-not-leak" not in rendered + assert "field[" in rendered + + +@pytest.mark.asyncio +async def test_canonical_markdown_source_is_deduplicated_across_roots(tmp_path: Path) -> None: + agents = _markdown_root(tmp_path / "agents") + _write_markdown(Path(str(agents.root)) / "worker.md", name="worker", description="worker") + + catalogue = await resolve_agent_catalogue( + agent_file=_root_agent(tmp_path), + markdown_roots=(agents, agents), + materialized_dir=tmp_path / "generated", + ) + + assert [entry.name for entry in catalogue.values()] == ["worker"] + assert catalogue.diagnostics == () diff --git a/tests/core/test_agent_catalogue_validation.py b/tests/core/test_agent_catalogue_validation.py new file mode 100644 index 00000000..37f3add4 --- /dev/null +++ b/tests/core/test_agent_catalogue_validation.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +from pathlib import Path +from typing import cast + +import pytest +from pydantic import ValidationError + +from pythinker_code.agentspec import SubagentSpec +from pythinker_code.exception import AgentSpecError +from pythinker_code.subagents.catalogue import ( + UnknownFieldPolicy, + normalize_agent_name, + resolve_agent_catalogue, +) + + +def _write_yaml(path: Path, body: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + +def _write_root_with_subagents(path: Path, subagents: str, *, extra: str = "") -> None: + _write_yaml( + path, + f"""version: 1 +{extra}agent: + name: root + system_prompt_path: ./system.md + tools: [] + subagents: +{subagents} +""", + ) + (path.parent / "system.md").write_text("root", encoding="utf-8") + + +def _write_child(path: Path, *, extra: str = "") -> None: + _write_yaml( + path, + f"""version: 1 +agent: + name: child + system_prompt_path: ./child.md + tools: [] +{extra}""", + ) + (path.parent / "child.md").write_text("child", encoding="utf-8") + + +@pytest.mark.asyncio +async def test_yaml_warn_aggregates_sorted_unknown_paths_without_values_or_absolute_paths( + tmp_path: Path, +) -> None: + child = tmp_path / "child.yaml" + _write_child(child, extra=" zeta: SUPER_SECRET\n alpha: hidden\n") + root = tmp_path / "root.yaml" + _write_root_with_subagents( + root, + " Worker:\n path: ./child.yaml\n description: worker\n typo: nested-secret\n", + extra="unexpected_top: top-secret\n", + ) + + catalogue = await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + unknown_field_policy=UnknownFieldPolicy.WARN, + ) + + assert len(catalogue.values()) == 1 + assert [diagnostic.field_path for diagnostic in catalogue.diagnostics] == [ + "agent.subagents.Worker.typo, unexpected_top", + "agent.alpha, agent.zeta", + ] + rendered = repr(catalogue.diagnostics) + assert "SUPER_SECRET" not in rendered + assert "nested-secret" not in rendered + assert str(tmp_path) not in rendered + + +@pytest.mark.asyncio +async def test_inherited_unknown_field_warns_once_for_defining_source(tmp_path: Path) -> None: + base = tmp_path / "base.yaml" + _write_child(base, extra=" stale_setting: secret\n") + child = tmp_path / "child.yaml" + _write_yaml(child, "version: 1\nagent:\n extend: ./base.yaml\n name: inherited\n") + root = tmp_path / "root.yaml" + _write_root_with_subagents( + root, + " first:\n path: ./child.yaml\n description: first\n" + " second:\n path: ./child.yaml\n description: second\n", + ) + + catalogue = await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + ) + + stale = [d for d in catalogue.diagnostics if d.field_path == "agent.stale_setting"] + assert len(stale) == 1 + + +@pytest.mark.asyncio +async def test_yaml_forbid_fails_required_source_without_leaking_value_or_path( + tmp_path: Path, +) -> None: + root = tmp_path / "root.yaml" + _write_root_with_subagents(root, "", extra="unknown: DO_NOT_LEAK\n") + + with pytest.raises(AgentSpecError) as error: + await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + unknown_field_policy=UnknownFieldPolicy.FORBID, + ) + + assert "unknown" in str(error.value) + assert "DO_NOT_LEAK" not in str(error.value) + assert str(tmp_path) not in str(error.value) + + +@pytest.mark.asyncio +async def test_required_yaml_casefold_collision_is_fatal(tmp_path: Path) -> None: + child = tmp_path / "child.yaml" + _write_child(child) + root = tmp_path / "root.yaml" + _write_root_with_subagents( + root, + " Worker:\n path: ./child.yaml\n description: first\n" + " worker:\n path: ./child.yaml\n description: second\n", + ) + + with pytest.raises(AgentSpecError, match="collision"): + await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + ) + + +@pytest.mark.asyncio +async def test_catalogue_is_casefolded_sorted_and_deeply_immutable(tmp_path: Path) -> None: + child = tmp_path / "child.yaml" + _write_child( + child, + extra=( + " system_prompt_args:\n key: value\n" + " subagents:\n" + " nested:\n" + " path: ./child.yaml\n" + " description: nested\n" + ), + ) + root = tmp_path / "root.yaml" + _write_root_with_subagents( + root, + " Zed:\n path: ./child.yaml\n description: zed\n" + " alpha:\n path: ./child.yaml\n description: alpha\n", + ) + + catalogue = await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + ) + + assert normalize_agent_name("Straße") == "strasse" + assert [entry.name for entry in catalogue.values()] == ["alpha", "Zed"] + assert catalogue.get("ALPHA") is catalogue.require("alpha") + with pytest.raises(KeyError): + catalogue.require("missing") + with pytest.raises(TypeError): + cast("dict[str, object]", catalogue.entries)["new"] = catalogue.require("alpha") + with pytest.raises(AttributeError): + catalogue.require("alpha").launch_spec.tools.append("unsafe") + with pytest.raises(TypeError): + catalogue.require("alpha").launch_spec.system_prompt_args["unsafe"] = "value" + with pytest.raises(TypeError): + subagents = catalogue.require("alpha").launch_spec.subagents + cast("dict[str, SubagentSpec]", subagents)["unsafe"] = subagents["nested"] + with pytest.raises(ValidationError): + catalogue.require("alpha").launch_spec.subagents["nested"].description = "changed" + + +@pytest.mark.asyncio +async def test_yaml_rejects_heterogeneous_and_secret_shaped_keys_without_leaking_them( + tmp_path: Path, +) -> None: + root = tmp_path / "root.yaml" + _write_root_with_subagents( + root, + "", + extra="42: value\nMY_SECRET_TOKEN: do-not-leak\n", + ) + + with pytest.raises(AgentSpecError) as error: + await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + ) + + rendered = str(error.value) + assert "MY_SECRET_TOKEN" not in rendered + assert "do-not-leak" not in rendered + assert str(tmp_path) not in rendered + assert "field[" in rendered + + +@pytest.mark.asyncio +async def test_yaml_warn_redacts_sensitive_string_unknown_fields_and_loads_entry( + tmp_path: Path, +) -> None: + child = tmp_path / "child.yaml" + _write_child( + child, + extra=" auth_strategy: ignored\n MY_SECRET_TOKEN: do-not-leak\n", + ) + root = tmp_path / "root.yaml" + _write_root_with_subagents( + root, + " worker:\n path: ./child.yaml\n description: worker\n", + ) + + catalogue = await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + unknown_field_policy=UnknownFieldPolicy.WARN, + ) + + assert [entry.name for entry in catalogue.values()] == ["worker"] + rendered = repr(catalogue.diagnostics) + assert "auth_strategy" not in rendered + assert "MY_SECRET_TOKEN" not in rendered + assert "do-not-leak" not in rendered + assert "field[" in rendered + + +@pytest.mark.asyncio +async def test_same_basename_yaml_sources_have_unique_safe_identifiers(tmp_path: Path) -> None: + first = tmp_path / "one" / "child.yaml" + second = tmp_path / "two" / "child.yaml" + _write_child(first, extra=" stale: one-secret\n") + _write_child(second, extra=" stale: two-secret\n") + root = tmp_path / "root.yaml" + _write_root_with_subagents( + root, + " first:\n path: ./one/child.yaml\n description: first\n" + " second:\n path: ./two/child.yaml\n description: second\n", + ) + + catalogue = await resolve_agent_catalogue( + agent_file=root, + markdown_roots=(), + materialized_dir=tmp_path / "generated", + ) + + assert len({d.safe_path for d in catalogue.diagnostics}) == 2 + assert len({entry.provenance.source_id for entry in catalogue.values()}) == 2 + rendered = repr((catalogue.diagnostics, tuple(e.provenance for e in catalogue.values()))) + assert str(tmp_path) not in rendered + assert "one-secret" not in rendered + assert "two-secret" not in rendered diff --git a/tests/core/test_agent_list_injection.py b/tests/core/test_agent_list_injection.py index 17085788..3aa32e9d 100644 --- a/tests/core/test_agent_list_injection.py +++ b/tests/core/test_agent_list_injection.py @@ -1,10 +1,18 @@ from __future__ import annotations +import dataclasses from pathlib import Path from types import SimpleNamespace -from pythinker_code.soul.agent import Runtime +import pytest +from pythinker_core.tooling.empty import EmptyToolset + +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.dynamic_injection import DynamicInjection +from pythinker_code.soul.pythinkersoul import PythinkerSoul from pythinker_code.subagents.models import AgentTypeDefinition, ToolPolicy +from pythinker_code.subagents.registry import LaborMarket from pythinker_code.wire.types import AgentListDelta @@ -44,6 +52,84 @@ def test_format_agent_line_no_restrictions() -> None: assert "Tools: *" in line +def test_agent_type_projects_to_literal_prompt_and_wire_contract(tmp_path: Path) -> None: + from pythinker_code.soul.dynamic_injections.agent_list import format_agent_line + + type_definition = AgentTypeDefinition( + name="reviewer", + description="Checks compatibility", + agent_file=tmp_path / "reviewer.yaml", + when_to_use=" Use after changes. ", + default_model="characterized-model", + tool_policy=ToolPolicy( + mode="allowlist", + tools=( + "package.alpha:ReadFile", + "package.beta:ReadFile", + "package.beta:Glob", + ), + ), + supports_background=False, + required_mcp_servers=("context7",), + ) + + assert format_agent_line(type_definition) == ( + "- `reviewer`: Checks compatibility (Tools: ReadFile, Glob). " + "When to use: Use after changes." + ) + + +async def test_provider_projects_literal_agent_type_without_field_drift( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from pythinker_code.soul.dynamic_injections.agent_list import AgentListInjectionProvider + + labor_market = LaborMarket() + labor_market.add_builtin_type( + AgentTypeDefinition( + name="reviewer", + description="Checks compatibility", + agent_file=tmp_path / "reviewer.yaml", + when_to_use="Use after changes.", + default_model="characterized-model", + tool_policy=ToolPolicy( + mode="allowlist", + tools=("package.alpha:ReadFile", "package.beta:Glob"), + ), + supports_background=False, + required_mcp_servers=("context7",), + ) + ) + runtime = dataclasses.replace(runtime, labor_market=labor_market) + agent = Agent( + name="Agent List Contract", + system_prompt="Agent list prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + soul = PythinkerSoul( + agent, + context=Context(file_backend=tmp_path / "agent-list-context.jsonl"), + ) + captured: list[object] = [] + monkeypatch.setattr( + "pythinker_code.soul.dynamic_injections.agent_list.wire_send", + lambda message: captured.append(message), + ) + injections = await AgentListInjectionProvider().get_injections([], soul) + + line = "- `reviewer`: Checks compatibility (Tools: ReadFile, Glob). When to use: Use after changes." + assert injections == [ + DynamicInjection( + type="agent_list", + content="Available agent types (regenerated when subagent specs change):\n" + line, + ) + ] + assert captured == [AgentListDelta(items=(line,), complete=True)] + + async def test_provider_emits_root_agent_list_and_wire_delta(runtime: Runtime, monkeypatch) -> None: from pythinker_code.soul.dynamic_injections.agent_list import AgentListInjectionProvider diff --git a/tests/core/test_auth_error_handling.py b/tests/core/test_auth_error_handling.py index b89cd4d5..92012cf5 100644 --- a/tests/core/test_auth_error_handling.py +++ b/tests/core/test_auth_error_handling.py @@ -290,6 +290,7 @@ def _runtime_with_provider(runtime: Runtime, provider, *, oauth: bool = False) - environment=runtime.environment, notifications=runtime.notifications, background_tasks=runtime.background_tasks, + skill_catalog=runtime.skill_catalog, skills=runtime.skills, oauth=runtime.oauth, additional_dirs=runtime.additional_dirs, diff --git a/tests/core/test_compaction_restore.py b/tests/core/test_compaction_restore.py index 1de83f0b..e26fd550 100644 --- a/tests/core/test_compaction_restore.py +++ b/tests/core/test_compaction_restore.py @@ -1,7 +1,8 @@ from __future__ import annotations +import asyncio from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, patch import pytest from pythinker_core.message import Message, TextPart, ToolCall @@ -11,13 +12,19 @@ from pythinker_code.hooks.runner import HookResult from pythinker_code.skill import Skill from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.compaction import CompactionResult from pythinker_code.soul.compaction_restore import ( _display_path, build_compaction_restore_context, build_hook_context_message, compact_summary_text, ) -from pythinker_code.soul.context import Context +from pythinker_code.soul.context import ( + Context, + ContextCommittedCancellation, + ContextGenerationConflictError, +) +from pythinker_code.soul.dynamic_injection import DynamicInjectionProvider from pythinker_code.soul.pythinkersoul import PythinkerSoul @@ -28,6 +35,32 @@ def _tool_call(name: str, arguments: str) -> ToolCall: ) +class _BlockingRearmProvider(DynamicInjectionProvider): + def __init__(self, entered: asyncio.Event, release: asyncio.Event) -> None: + self.calls = 0 + self._entered = entered + self._release = release + + async def get_injections(self, history, soul): # noqa: ANN001 + return [] + + async def on_context_compacted(self) -> None: + self.calls += 1 + self._entered.set() + await self._release.wait() + + +class _RecordingRearmProvider(DynamicInjectionProvider): + def __init__(self) -> None: + self.calls = 0 + + async def get_injections(self, history, soul): # noqa: ANN001 + return [] + + async def on_context_compacted(self) -> None: + self.calls += 1 + + @pytest.mark.asyncio async def test_restore_context_collects_recent_read_and_referenced_files(tmp_path: Path) -> None: history = [ @@ -134,12 +167,13 @@ async def test_compact_context_restores_files_and_hook_context( ) ) - fake_result = MagicMock() - fake_result.messages = [Message(role="user", content=[TextPart(text="compacted summary")])] - fake_result.estimated_token_count = 10 + fake_result = CompactionResult( + messages=[Message(role="user", content=[TextPart(text="compacted summary")])], + usage=None, + ) soul._run_with_connection_recovery = AsyncMock(return_value=fake_result) # pyright: ignore[reportPrivateUsage] soul._checkpoint = AsyncMock() # pyright: ignore[reportPrivateUsage] - soul._notify_injection_providers_compacted = AsyncMock() # pyright: ignore[reportPrivateUsage] + soul.notify_history_rebuilt = AsyncMock() soul._hook_engine.trigger = AsyncMock( # pyright: ignore[reportPrivateUsage] side_effect=[ [], @@ -147,6 +181,14 @@ async def test_compact_context_restores_files_and_hook_context( [HookResult(additional_context="SessionStart compact context")], ] ) + legacy_clear = AsyncMock(side_effect=AssertionError("legacy clear path used")) + legacy_write_prompt = AsyncMock(side_effect=AssertionError("legacy prompt write used")) + legacy_append = AsyncMock(side_effect=AssertionError("legacy rebuild append used")) + legacy_usage = AsyncMock(side_effect=AssertionError("legacy usage rollback used")) + context.clear = legacy_clear # type: ignore[method-assign] + context.write_system_prompt = legacy_write_prompt # type: ignore[method-assign] + context.append_message = legacy_append # type: ignore[method-assign] + context.update_token_count = legacy_usage # type: ignore[method-assign] sent_texts: list[str] = [] @@ -178,12 +220,19 @@ def _capture_wire(msg): session_start_call = soul._hook_engine.trigger.await_args_list[2] # pyright: ignore[reportPrivateUsage] assert session_start_call.args[0] == "SessionStart" assert session_start_call.kwargs["matcher_value"] == "compact" + soul._checkpoint.assert_not_awaited() # pyright: ignore[reportPrivateUsage] + legacy_clear.assert_not_awaited() + legacy_write_prompt.assert_not_awaited() + legacy_append.assert_not_awaited() + legacy_usage.assert_not_awaited() @pytest.mark.asyncio -async def test_compact_context_restores_history_when_rebuild_fails( +@pytest.mark.parametrize("error", [OSError("disk full"), asyncio.CancelledError()]) +async def test_compact_context_replacement_failure_preserves_generation( runtime: Runtime, tmp_path: Path, + error: BaseException, ) -> None: agent = Agent( name="Test Agent", @@ -201,37 +250,50 @@ async def test_compact_context_restores_history_when_rebuild_fails( await context.append_message(msg_user) await context.append_message(msg_assistant) - before = list(context.history) + await context.write_system_prompt("Test system prompt.") + await context.update_token_count(47) + before_bytes = context.file_backend.read_bytes() + before_memory = ( + tuple(context.history), + context.system_prompt, + context.token_count, + context.token_count_with_pending, + context.n_checkpoints, + ) - fake_result = MagicMock() - fake_result.messages = [Message(role="user", content=[TextPart(text="compacted-summary")])] - fake_result.estimated_token_count = 5 - fake_result.usage = None + fake_result = CompactionResult( + messages=[Message(role="user", content=[TextPart(text="compacted-summary")])], + usage=None, + ) soul._run_with_connection_recovery = AsyncMock(return_value=fake_result) # pyright: ignore[reportPrivateUsage] - soul._checkpoint = AsyncMock() # pyright: ignore[reportPrivateUsage] soul._hook_engine.trigger = AsyncMock(return_value=[]) # pyright: ignore[reportPrivateUsage] - - # Wrap append_message: raise when seeing the compacted summary text so the - # fault lands after clear() has already rotated the backing file. - real_append = context.append_message - - async def flaky_append(message): - msgs = [message] if isinstance(message, Message) else list(message) - for m in msgs: - if "compacted-summary" in m.extract_text(""): - raise RuntimeError("disk full") - return await real_append(message) - - context.append_message = flaky_append # type: ignore[method-assign] + replace_history = AsyncMock(side_effect=error) + context.replace_history = replace_history # type: ignore[method-assign] + soul.notify_history_rebuilt = AsyncMock() + sent: list[str] = [] with ( - patch("pythinker_code.soul.pythinkersoul.wire_send"), + patch( + "pythinker_code.soul.pythinkersoul.wire_send", + lambda message: sent.append(type(message).__name__), + ), patch("pythinker_code.telemetry.track"), - pytest.raises(RuntimeError, match="disk full"), + pytest.raises(type(error)), ): await soul.compact_context() - assert list(context.history) == before + replace_history.assert_awaited_once() + soul.notify_history_rebuilt.assert_not_awaited() + assert sent.count("CompactionBegin") == 1 + assert sent.count("CompactionEnd") == 1 + assert context.file_backend.read_bytes() == before_bytes + assert ( + tuple(context.history), + context.system_prompt, + context.token_count, + context.token_count_with_pending, + context.n_checkpoints, + ) == before_memory @pytest.mark.asyncio @@ -278,6 +340,99 @@ def _capture_wire(msg): ) +@pytest.mark.asyncio +async def test_compact_context_rejects_stale_replacement_after_concurrent_append( + runtime: Runtime, + tmp_path: Path, +) -> None: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "history-concurrent.jsonl") + soul = PythinkerSoul(agent, context=context) + runtime.session.state.active_skills = [] + await context.append_message(Message(role="user", content="compact me")) + fake_result = CompactionResult( + messages=[Message(role="user", content="compacted")], + usage=None, + ) + soul._run_with_connection_recovery = AsyncMock(return_value=fake_result) # pyright: ignore[reportPrivateUsage] + soul._hook_engine.trigger = AsyncMock(return_value=[]) # pyright: ignore[reportPrivateUsage] + replacement_entered = asyncio.Event() + release_replacement = asyncio.Event() + real_replace = context.replace_history + + async def delayed_replace(replacement, **kwargs): # noqa: ANN001, ANN003 + replacement_entered.set() + await release_replacement.wait() + return await real_replace(replacement, **kwargs) + + context.replace_history = delayed_replace # type: ignore[method-assign] + with patch("pythinker_code.soul.pythinkersoul.wire_send"): + compact = asyncio.create_task(soul.compact_context()) + try: + await asyncio.wait_for(replacement_entered.wait(), timeout=5.0) + concurrent = Message(role="user", content="concurrent append wins") + await context.append_message(concurrent) + finally: + release_replacement.set() + with pytest.raises(ContextGenerationConflictError): + await asyncio.wait_for(compact, timeout=5.0) + + assert context.history[-1] == concurrent + + +@pytest.mark.asyncio +async def test_compact_visible_commit_cancellation_settles_rearm_under_second_cancel( + runtime: Runtime, + tmp_path: Path, +) -> None: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "history-cancel.jsonl") + soul = PythinkerSoul(agent, context=context) + runtime.session.state.active_skills = [] + await context.append_message(Message(role="user", content="compact me")) + fake_result = CompactionResult( + messages=[Message(role="user", content="compacted")], + usage=None, + ) + soul._run_with_connection_recovery = AsyncMock(return_value=fake_result) # pyright: ignore[reportPrivateUsage] + soul._hook_engine.trigger = AsyncMock(return_value=[]) # pyright: ignore[reportPrivateUsage] + entered = asyncio.Event() + release = asyncio.Event() + blocking = _BlockingRearmProvider(entered, release) + recording = _RecordingRearmProvider() + soul._injection_providers = [blocking, recording] # pyright: ignore[reportPrivateUsage] + lifecycle_generation = soul._request_lifecycle.history_generation # pyright: ignore[reportPrivateUsage] + real_replace = context.replace_history + + async def commit_then_cancel(replacement, **kwargs): # noqa: ANN001, ANN003 + commit = await real_replace(replacement, **kwargs) + raise ContextCommittedCancellation(commit) + + context.replace_history = commit_then_cancel # type: ignore[method-assign] + with patch("pythinker_code.soul.pythinkersoul.wire_send"): + compact = asyncio.create_task(soul.compact_context()) + await asyncio.wait_for(entered.wait(), timeout=5.0) + compact.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await compact + + assert blocking.calls == 1 + assert recording.calls == 1 + assert soul._request_lifecycle.history_generation == lifecycle_generation + 1 # pyright: ignore[reportPrivateUsage] + assert context.history[-1].extract_text("") == "compacted" + + def test_display_path_skips_out_of_workspace_absolute_paths(tmp_path: Path) -> None: work = HostPath.unsafe_from_local_path(tmp_path) diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 9067e551..dfbe604f 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -2,8 +2,10 @@ from __future__ import annotations +import asyncio import json from pathlib import Path +from unittest.mock import AsyncMock import pytest from pythinker_core.message import Message, Role @@ -34,6 +36,21 @@ def _message_dict(role: Role, text: str) -> dict: ) +@pytest.fixture +def mixed_context_jsonl(tmp_path: Path) -> Path: + path = tmp_path / "mixed-context.jsonl" + path.write_text( + '{"role":"_system_prompt","content":"Frozen prompt"}\n' + '{"role":"user","content":"Before checkpoint"}\n' + '{"role":"_checkpoint","id":3}\n' + '{"role":"assistant","content":"Recorded answer"}\n' + '{"role":"_usage","token_count":144}\n' + '{"role":"user","content":"After usage"}\n', + encoding="utf-8", + ) + return path + + # --- write_system_prompt tests --- @@ -82,6 +99,23 @@ async def test_write_system_prompt_prepends_to_existing(tmp_path: Path) -> None: # --- restore tests --- +@pytest.mark.asyncio +async def test_restore_preserves_literal_mixed_record_contract(mixed_context_jsonl: Path) -> None: + ctx = Context(file_backend=mixed_context_jsonl) + + restored = await ctx.restore() + + assert restored is True + assert ctx.system_prompt == "Frozen prompt" + assert ctx.n_checkpoints == 4 + assert ctx.token_count == 144 + assert tuple(ctx.history) == ( + Message(role="user", content=[TextPart(text="Before checkpoint")]), + Message(role="assistant", content=[TextPart(text="Recorded answer")]), + Message(role="user", content=[TextPart(text="After usage")]), + ) + + @pytest.mark.asyncio async def test_restore_reads_system_prompt(tmp_path: Path) -> None: path = tmp_path / "context.jsonl" @@ -258,6 +292,55 @@ async def test_revert_preserves_system_prompt(tmp_path: Path) -> None: assert len(ctx.history) == 1 +@pytest.mark.parametrize("operation", ["revert", "clear"]) +@pytest.mark.parametrize("error", [OSError("disk full"), asyncio.CancelledError()]) +async def test_history_replacement_failure_preserves_exact_generation( + tmp_path: Path, + operation: str, + error: BaseException, +) -> None: + path = tmp_path / "context.jsonl" + _write_lines( + path, + [ + {"role": "_system_prompt", "content": "Preserved prompt"}, + _message_dict("user", "Before checkpoint"), + {"role": "_checkpoint", "id": 0}, + _message_dict("assistant", "After checkpoint"), + {"role": "_usage", "token_count": 23}, + {"role": "_checkpoint", "id": 1}, + ], + ) + ctx = Context(file_backend=path) + await ctx.restore() + before_bytes = path.read_bytes() + before_memory = ( + tuple(ctx.history), + ctx.system_prompt, + ctx.token_count, + ctx.token_count_with_pending, + ctx.n_checkpoints, + ) + replace_history = AsyncMock(side_effect=error) + ctx.replace_history = replace_history # type: ignore[method-assign] + + with pytest.raises(type(error)): + if operation == "revert": + await ctx.revert_to(1) + else: + await ctx.clear() + + replace_history.assert_awaited_once() + assert path.read_bytes() == before_bytes + assert ( + tuple(ctx.history), + ctx.system_prompt, + ctx.token_count, + ctx.token_count_with_pending, + ctx.n_checkpoints, + ) == before_memory + + @pytest.mark.asyncio async def test_revert_preserves_system_prompt_in_file(tmp_path: Path) -> None: path = tmp_path / "context.jsonl" diff --git a/tests/core/test_context_pruning.py b/tests/core/test_context_pruning.py index 2e5917c7..89f068be 100644 --- a/tests/core/test_context_pruning.py +++ b/tests/core/test_context_pruning.py @@ -8,6 +8,9 @@ from __future__ import annotations +import asyncio +from unittest.mock import AsyncMock + from pythinker_core.message import Message, TextPart from pythinker_code.soul.compaction import ( @@ -85,7 +88,7 @@ def test_should_prune_threshold() -> None: from pythinker_core.tooling.simple import SimpleToolset # noqa: E402 from pythinker_code.soul.agent import Agent, Runtime # noqa: E402 -from pythinker_code.soul.context import Context # noqa: E402 +from pythinker_code.soul.context import Context, ContextGenerationConflictError # noqa: E402 from pythinker_code.soul.pythinkersoul import PythinkerSoul # noqa: E402 @@ -111,6 +114,16 @@ async def test_prune_context_rewrites_history_preserving_structure(runtime, tmp_ Message(role="assistant", content=[TextPart(text="done")]), ] ) + legacy_clear = AsyncMock(side_effect=AssertionError("legacy clear path used")) + legacy_write_prompt = AsyncMock(side_effect=AssertionError("legacy prompt write used")) + legacy_checkpoint = AsyncMock(side_effect=AssertionError("legacy checkpoint path used")) + legacy_append = AsyncMock(side_effect=AssertionError("legacy rebuild append used")) + legacy_usage = AsyncMock(side_effect=AssertionError("legacy usage rollback used")) + context.clear = legacy_clear # type: ignore[method-assign] + context.write_system_prompt = legacy_write_prompt # type: ignore[method-assign] + context.checkpoint = legacy_checkpoint # type: ignore[method-assign] + context.append_message = legacy_append # type: ignore[method-assign] + context.update_token_count = legacy_usage # type: ignore[method-assign] did_prune = await soul.prune_context() @@ -123,6 +136,11 @@ async def test_prune_context_rewrites_history_preserving_structure(runtime, tmp_ assert "elided" in tool_msgs[0].extract_text("") # body replaced # Recent + non-tool messages untouched. assert history[-1].extract_text("") == "done" + legacy_clear.assert_not_awaited() + legacy_write_prompt.assert_not_awaited() + legacy_checkpoint.assert_not_awaited() + legacy_append.assert_not_awaited() + legacy_usage.assert_not_awaited() def _seed_prunable() -> list[Message]: @@ -136,33 +154,39 @@ def _seed_prunable() -> list[Message]: @pytest.mark.asyncio -async def test_prune_context_restores_history_when_rebuild_fails(runtime, tmp_path) -> None: - """If the rebuild after clear() fails, prune must restore prior history rather than - leave the context gutted to just the system prompt (data-loss guard).""" +@pytest.mark.parametrize("error", [OSError("disk full"), asyncio.CancelledError()]) +async def test_prune_context_replacement_failure_preserves_generation( + runtime, tmp_path, error: BaseException +) -> None: runtime.config.loop_control.prune_protect_last = 2 runtime.config.loop_control.prune_min_chars = 2000 context, soul = _make_soul(runtime, tmp_path) await context.write_system_prompt("sys") await context.append_message(_seed_prunable()) - before = list(context.history) - - # Fail the rebuild's append of the pruned body (it carries the "elided" placeholder); - # the restore re-appends the original snapshot, which must still succeed. - real_append = context.append_message - - async def flaky_append(message): - msgs = [message] if isinstance(message, Message) else list(message) - if any("elided" in m.extract_text("") for m in msgs): - raise RuntimeError("disk full") - return await real_append(message) - - context.append_message = flaky_append # type: ignore[method-assign] + await context.update_token_count(91) + before_bytes = context.file_backend.read_bytes() + before_memory = ( + tuple(context.history), + context.system_prompt, + context.token_count, + context.token_count_with_pending, + context.n_checkpoints, + ) + replace_history = AsyncMock(side_effect=error) + context.replace_history = replace_history # type: ignore[method-assign] - with pytest.raises(RuntimeError, match="disk full"): + with pytest.raises(type(error)): await soul.prune_context() - # History is restored intact — not left as just the system prompt. - assert list(context.history) == before + replace_history.assert_awaited_once() + assert context.file_backend.read_bytes() == before_bytes + assert ( + tuple(context.history), + context.system_prompt, + context.token_count, + context.token_count_with_pending, + context.n_checkpoints, + ) == before_memory @pytest.mark.asyncio @@ -232,3 +256,33 @@ async def test_prune_context_noop_when_nothing_stale(runtime, tmp_path) -> None: assert did_prune is False # protected + small → nothing to prune assert soul.context.history[-1].extract_text("") == "small" + + +@pytest.mark.asyncio +async def test_prune_context_rejects_stale_replacement_after_concurrent_append( + runtime, tmp_path +) -> None: + runtime.config.loop_control.prune_protect_last = 2 + runtime.config.loop_control.prune_min_chars = 2000 + context, soul = _make_soul(runtime, tmp_path) + await context.write_system_prompt("sys") + await context.append_message(_seed_prunable()) + replacement_entered = asyncio.Event() + release_replacement = asyncio.Event() + real_replace = context.replace_history + + async def delayed_replace(replacement, **kwargs): # noqa: ANN001, ANN003 + replacement_entered.set() + await release_replacement.wait() + return await real_replace(replacement, **kwargs) + + context.replace_history = delayed_replace # type: ignore[method-assign] + prune = asyncio.create_task(soul.prune_context()) + await replacement_entered.wait() + concurrent = Message(role="user", content="concurrent append wins") + await context.append_message(concurrent) + release_replacement.set() + + with pytest.raises(ContextGenerationConflictError): + await prune + assert context.history[-1] == concurrent diff --git a/tests/core/test_context_transactions.py b/tests/core/test_context_transactions.py new file mode 100644 index 00000000..99a1fc4f --- /dev/null +++ b/tests/core/test_context_transactions.py @@ -0,0 +1,1321 @@ +from __future__ import annotations + +import asyncio +import errno +import threading +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Any, cast + +import pytest +from pythinker_core.message import Message + +import pythinker_code.soul.context as context_module +from pythinker_code.soul.compaction import estimate_text_tokens +from pythinker_code.soul.context import Context +from pythinker_code.wire.types import TextPart + + +def _message(text: str) -> Message: + return Message(role="user", content=[TextPart(text=text)]) + + +def _memory(context: Context) -> tuple[object, ...]: + return ( + tuple(context.history), + context.token_count, + context.token_count_with_pending, + context.n_checkpoints, + context.system_prompt, + context._tail_repaired, + context._pending_messages, + ) + + +def _replacement( + *messages: Message, + system_prompt: str | None = "replacement prompt", + token_count: int = 37, + create_checkpoint: bool = True, + checkpoint_user_marker: bool = True, +) -> Any: + return context_module.ContextReplacement( + system_prompt=system_prompt, + messages=messages, + token_count=token_count, + create_checkpoint=create_checkpoint, + checkpoint_user_marker=checkpoint_user_marker, + ) + + +class _AppendFileProxy: + def __init__( + self, + wrapped: Any, + *, + failure_point: str | None = None, + entered: threading.Event | None = None, + release: threading.Event | None = None, + ) -> None: + self._wrapped = wrapped + self._failure_point = failure_point + self._entered = entered + self._release = release + + def __enter__(self) -> _AppendFileProxy: + self._wrapped.__enter__() + return self + + def __exit__(self, *args: object) -> object: + if self._failure_point == "close": + self._wrapped.__exit__(*args) + raise OSError("close failed") + if self._failure_point == "block_close": + assert self._entered is not None + assert self._release is not None + self._entered.set() + self._release.wait() + return self._wrapped.__exit__(*args) + + def write(self, payload: str) -> int: + if self._failure_point in {"block_write", "block_write_fail"}: + assert self._entered is not None + assert self._release is not None + self._entered.set() + self._release.wait() + if self._failure_point == "block_write_fail": + self._wrapped.write(payload[: len(payload) // 2]) + raise OSError("write failed after cancellation") + if self._failure_point == "write": + self._wrapped.write(payload[: len(payload) // 2]) + raise OSError("write failed") + return self._wrapped.write(payload) + + def flush(self) -> None: + if self._failure_point == "flush": + raise OSError("flush failed") + self._wrapped.flush() + + def __getattr__(self, name: str) -> Any: + return getattr(self._wrapped, name) + + +def _patch_append_open( + monkeypatch: pytest.MonkeyPatch, + context_path: Path, + failure_point: str, + *, + entered: threading.Event | None = None, + release: threading.Event | None = None, +) -> None: + real_open = cast(Callable[..., Any], Path.open) + injected = False + + def injected_open(path: Path, *args: object, **kwargs: object) -> Any: + nonlocal injected + wrapped = real_open(path, *args, **kwargs) + mode = args[0] if args else kwargs.get("mode", "r") + if path == context_path and mode == "a" and not injected: + injected = True + return _AppendFileProxy( + wrapped, + failure_point=failure_point, + entered=entered, + release=release, + ) + return wrapped + + monkeypatch.setattr(Path, "open", injected_open) + + +class _SyncFileProxy: + def __init__(self, wrapped: Any, failure_point: str) -> None: + self._wrapped = wrapped + self._failure_point = failure_point + + def __enter__(self) -> _SyncFileProxy: + self._wrapped.__enter__() + return self + + def __exit__(self, *args: object) -> object: + return self._wrapped.__exit__(*args) + + def write(self, payload: str) -> int: + if self._failure_point == "write": + self._wrapped.write(payload[: len(payload) // 2]) + raise OSError("system prompt write failed") + return self._wrapped.write(payload) + + def flush(self) -> None: + if self._failure_point == "flush": + raise OSError("system prompt flush failed") + self._wrapped.flush() + + def __getattr__(self, name: str) -> Any: + return getattr(self._wrapped, name) + + +@pytest.mark.asyncio +async def test_serialization_failure_leaves_exact_old_bytes_and_memory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + def fail_serialization(_records: Sequence[object]) -> str: + raise ValueError("cannot serialize") + + monkeypatch.setattr(context_module, "_serialize_context_records", fail_serialization) + + with pytest.raises(ValueError, match="cannot serialize"): + await context.append_messages((_message("new"),)) + + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + + +@pytest.mark.asyncio +async def test_real_open_failure_leaves_memory_unchanged_and_retry_appends_once( + tmp_path: Path, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + reminder = _message("required reminder") + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + context._file_backend = tmp_path / "missing" / "context.jsonl" + with pytest.raises(FileNotFoundError): + await context.append_message(reminder) + + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + + context._file_backend = context_path + await context.append_message(reminder) + + assert list(context.history).count(reminder) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_point", ["write", "flush"]) +async def test_append_boundary_failure_leaves_exact_old_bytes_and_memory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_point: str, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + _patch_append_open(monkeypatch, context_path, failure_point) + + with pytest.raises(OSError, match=f"{failure_point} failed"): + await context.append_messages((_message("new one"), _message("new two"))) + + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + + +@pytest.mark.asyncio +async def test_append_close_failure_rolls_back_exact_old_disk_and_memory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + committed = _message("committed before close failed") + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + _patch_append_open(monkeypatch, context_path, "close") + + with pytest.raises(OSError, match="close failed"): + await context.append_message(committed) + + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + + monkeypatch.undo() + await context.append_message(committed) + + assert list(context.history).count(committed) == 1 + + +@pytest.mark.asyncio +async def test_append_cancellation_during_close_keeps_new_disk_and_memory_coherent( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + committed = _message("committed before close cancellation") + close_entered = threading.Event() + release_close = threading.Event() + + _patch_append_open( + monkeypatch, + context_path, + "block_close", + entered=close_entered, + release=release_close, + ) + append = asyncio.create_task(context.append_message(committed)) + assert await asyncio.to_thread(close_entered.wait, 5) + + append.cancel() + release_close.set() + with pytest.raises(asyncio.CancelledError): + await append + + assert list(context.history)[-1] == committed + restored = Context(context_path) + assert await restored.restore() + assert list(restored.history) == list(context.history) + + +@pytest.mark.asyncio +async def test_cancelled_append_with_worker_failure_remains_cancelled_and_rolls_back( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + write_entered = threading.Event() + release_write = threading.Event() + + _patch_append_open( + monkeypatch, + context_path, + "block_write_fail", + entered=write_entered, + release=release_write, + ) + append = asyncio.create_task(context.append_message(_message("not committed"))) + assert await asyncio.to_thread(write_entered.wait, 5) + + append.cancel() + release_write.set() + with pytest.raises(asyncio.CancelledError) as cancellation: + await append + + assert append.cancelled() + assert isinstance(cancellation.value.__cause__, OSError) + assert str(cancellation.value.__cause__) == "write failed after cancellation" + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + + +@pytest.mark.asyncio +async def test_repeated_cancellation_settles_commit_before_final_cancellation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + committed = _message("committed despite repeated cancellation") + close_entered = threading.Event() + release_close = threading.Event() + + _patch_append_open( + monkeypatch, + context_path, + "block_close", + entered=close_entered, + release=release_close, + ) + append = asyncio.create_task(context.append_message(committed)) + assert await asyncio.to_thread(close_entered.wait, 5) + + append.cancel() + second_cancellation_sent = asyncio.Event() + + def cancel_again() -> None: + append.cancel() + second_cancellation_sent.set() + + asyncio.get_running_loop().call_soon(cancel_again) + await second_cancellation_sent.wait() + release_close.set() + with pytest.raises(asyncio.CancelledError): + await append + + assert append.cancelled() + assert list(context.history)[-1] == committed + restored = Context(context_path) + assert await restored.restore() + assert list(restored.history) == list(context.history) + + +@pytest.mark.asyncio +async def test_checkpoint_failure_keeps_id_and_optional_marker_uncommitted( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + _patch_append_open(monkeypatch, context_path, "write") + + with pytest.raises(OSError, match="write failed"): + await context.checkpoint(add_user_message=True) + + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + + monkeypatch.undo() + await context.checkpoint(add_user_message=True) + + assert context.n_checkpoints == 1 + assert context.history[-1].extract_text("") == "CHECKPOINT 0" + assert context_path.read_text(encoding="utf-8").count('"role": "_checkpoint", "id": 0') == 1 + + +@pytest.mark.asyncio +async def test_usage_failure_keeps_authoritative_and_pending_counts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + pending = _message("pending content") + await context.append_message(pending) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + _patch_append_open(monkeypatch, context_path, "flush") + + with pytest.raises(OSError, match="flush failed"): + await context.update_token_count(400) + + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert context.token_count_with_pending == estimate_text_tokens((pending,)) + + +@pytest.mark.asyncio +async def test_concurrent_appends_commit_in_lock_arrival_order( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + first_write_entered = threading.Event() + release_first_write = threading.Event() + + _patch_append_open( + monkeypatch, + context_path, + "block_write", + entered=first_write_entered, + release=release_first_write, + ) + first = asyncio.create_task(context.append_message(_message("first"))) + assert await asyncio.to_thread(first_write_entered.wait, 5) + second_started = asyncio.Event() + + async def append_second() -> None: + second_started.set() + await context.append_message(_message("second")) + + second = asyncio.create_task(append_second()) + await second_started.wait() + release_first_write.set() + await asyncio.gather(first, second) + + assert [message.extract_text("") for message in context.history] == ["first", "second"] + restored = Context(context_path) + assert await restored.restore() + assert list(restored.history) == list(context.history) + + +@pytest.mark.asyncio +async def test_cancelled_queued_writer_does_not_poison_lock_or_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + first_write_entered = threading.Event() + release_first_write = threading.Event() + + _patch_append_open( + monkeypatch, + context_path, + "block_write", + entered=first_write_entered, + release=release_first_write, + ) + first = asyncio.create_task(context.append_message(_message("first"))) + assert await asyncio.to_thread(first_write_entered.wait, 5) + queued_started = asyncio.Event() + + async def append_queued() -> None: + queued_started.set() + await context.append_message(_message("cancelled")) + + queued = asyncio.create_task(append_queued()) + await queued_started.wait() + queued.cancel() + with pytest.raises(asyncio.CancelledError): + await queued + + release_first_write.set() + await first + await context.append_message(_message("after cancellation")) + + assert [message.extract_text("") for message in context.history] == [ + "first", + "after cancellation", + ] + + +@pytest.mark.asyncio +async def test_system_prompt_open_failure_leaves_memory_unchanged(tmp_path: Path) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + context._file_backend = tmp_path / "missing" / "context.jsonl" + + with pytest.raises(FileNotFoundError): + await context.write_system_prompt("prompt") + + assert context.system_prompt is None + assert not context_path.exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_point", ["write", "flush"]) +async def test_system_prompt_failure_leaves_nonexistent_file_and_memory_unchanged( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_point: str, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + old_memory = _memory(context) + real_fdopen = cast(Callable[..., Any], context_module.os.fdopen) + + def failing_fdopen(*args: object, **kwargs: object) -> _SyncFileProxy: + return _SyncFileProxy(real_fdopen(*args, **kwargs), failure_point) + + monkeypatch.setattr(context_module.os, "fdopen", failing_fdopen) + + with pytest.raises(OSError, match=f"system prompt {failure_point} failed"): + await context.write_system_prompt("new prompt") + + assert not context_path.exists() + assert not list(tmp_path.glob(f"{context_path.name}*.tmp")) + assert _memory(context) == old_memory + + monkeypatch.undo() + await context.write_system_prompt("new prompt") + + assert context.system_prompt == "new prompt" + assert context_path.read_text(encoding="utf-8").count('"role": "_system_prompt"') == 1 + + +@pytest.mark.asyncio +async def test_system_prompt_cancellation_waits_for_commit_and_swaps_memory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + replace_entered = threading.Event() + release_replace = threading.Event() + replace_finished = threading.Event() + real_replace = Path.replace + + def blocking_replace(source: Path, target: Path) -> Path: + replace_entered.set() + release_replace.wait() + try: + return real_replace(source, target) + finally: + replace_finished.set() + + monkeypatch.setattr(Path, "replace", blocking_replace) + write_prompt = asyncio.create_task(context.write_system_prompt("committed prompt")) + assert await asyncio.to_thread(replace_entered.wait, 5) + + write_prompt.cancel() + release_replace.set() + with pytest.raises(asyncio.CancelledError): + await write_prompt + assert await asyncio.to_thread(replace_finished.wait, 5) + + assert context.system_prompt == "committed prompt" + restored = Context(context_path) + assert await restored.restore() + assert restored.system_prompt == context.system_prompt + + +class _ReplacementFileProxy: + def __init__(self, wrapped: Any, *, fail_write: int | None = None, fail_flush: bool = False): + self._wrapped = wrapped + self._fail_write = fail_write + self._fail_flush = fail_flush + self._writes = 0 + + def __enter__(self) -> _ReplacementFileProxy: + self._wrapped.__enter__() + return self + + def __exit__(self, *args: object) -> object: + return self._wrapped.__exit__(*args) + + def write(self, payload: str) -> int: + self._writes += 1 + if self._writes == self._fail_write: + raise OSError(f"record write {self._writes} failed") + return self._wrapped.write(payload) + + def flush(self) -> None: + if self._fail_flush: + raise OSError("replacement flush failed") + self._wrapped.flush() + + def __getattr__(self, name: str) -> Any: + return getattr(self._wrapped, name) + + +@pytest.mark.asyncio +async def test_replace_history_commits_compatible_records_and_exact_archive(tmp_path: Path) -> None: + context_path = tmp_path / "context.jsonl" + old_bytes = b'{"role":"user","content":"old"}\n{"torn":' + context_path.write_bytes(old_bytes) + context = Context(context_path) + context._history[:] = [_message("old")] + + commit = await context.replace_history(_replacement(_message("new"))) + + records = [ + context._parse_context_line(line, file_backend=context_path, line_no=index) + for index, line in enumerate(context_path.read_text(encoding="utf-8").splitlines(), 1) + ] + assert [record["role"] for record in records if record is not None] == [ + "_system_prompt", + "_checkpoint", + "user", + "user", + "_usage", + ] + assert commit.checkpoint_id == 0 + assert commit.rotated_file == tmp_path / "context_1.jsonl" + assert commit.rotated_file is not None + assert commit.rotated_file.read_bytes() == old_bytes + assert commit.message_count == 2 + assert context.system_prompt == "replacement prompt" + assert [message.extract_text("") for message in context.history] == [ + "CHECKPOINT 0", + "new", + ] + assert context.token_count == 37 + assert context.token_count_with_pending == 37 + assert context.n_checkpoints == 1 + assert not list(tmp_path.glob("context.jsonl*.tmp")) + + restored = Context(context_path) + assert await restored.restore() + assert _memory(restored) == _memory(context) + + +@pytest.mark.asyncio +async def test_replace_history_validates_before_filesystem_work( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context_path.write_bytes(b"old bytes") + context = Context(context_path) + old_memory = _memory(context) + + def unexpected_temp(*args: object, **kwargs: object) -> tuple[int, str]: + pytest.fail("validation touched the filesystem") + + monkeypatch.setattr(context_module.tempfile, "mkstemp", unexpected_temp) + with pytest.raises(ValueError, match="token_count"): + await context.replace_history(_replacement(token_count=-1)) + + assert context_path.read_bytes() == b"old bytes" + assert _memory(context) == old_memory + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure_point", "expected_category"), + [ + ("temp_create", "temporary_creation"), + ("write_1", "write"), + ("write_2", "write"), + ("write_3", "write"), + ("write_4", "write"), + ("write_5", "write"), + ("flush", "write"), + ("temp_fsync", "synchronization"), + ], +) +async def test_replace_history_preparation_failures_preserve_exact_old_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_point: str, + expected_category: str, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + if failure_point == "temp_create": + monkeypatch.setattr( + context_module.tempfile, + "mkstemp", + lambda *args, **kwargs: (_ for _ in ()).throw(OSError("temp create failed")), + ) + else: + real_fdopen = cast(Callable[..., Any], context_module.os.fdopen) + write_number = ( + int(failure_point.removeprefix("write_")) if "write_" in failure_point else None + ) + + def failing_fdopen(*args: object, **kwargs: object) -> _ReplacementFileProxy: + return _ReplacementFileProxy( + real_fdopen(*args, **kwargs), + fail_write=write_number, + fail_flush=failure_point == "flush", + ) + + monkeypatch.setattr(context_module.os, "fdopen", failing_fdopen) + if failure_point == "temp_fsync": + monkeypatch.setattr( + context_module.os, + "fsync", + lambda _fd: (_ for _ in ()).throw(OSError("temp fsync failed")), + ) + + with pytest.raises(context_module.ContextPersistenceError) as raised: + await context.replace_history(_replacement(_message("new"))) + + assert raised.value.operation == "replace_history" + assert raised.value.category == expected_category + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert not list(tmp_path.glob("context.jsonl*.tmp")) + assert not (tmp_path / "context_1.jsonl").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_point", ["archive_write", "archive_flush", "archive_fsync"]) +async def test_replace_history_archive_failures_preserve_old_bytes_and_memory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_point: str, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + b'{"torn":' + context_path.write_bytes(old_bytes) + old_memory = _memory(context) + real_open = cast(Callable[..., Any], Path.open) + + def failing_open(path: Path, *args: object, **kwargs: object) -> Any: + wrapped = real_open(path, *args, **kwargs) + mode = args[0] if args else kwargs.get("mode", "r") + if path.name == "context_1.jsonl" and mode == "wb": + return _ReplacementFileProxy( + wrapped, + fail_write=1 if failure_point == "archive_write" else None, + fail_flush=failure_point == "archive_flush", + ) + return wrapped + + monkeypatch.setattr(Path, "open", failing_open) + if failure_point == "archive_fsync": + real_fsync = context_module.os.fsync + calls = 0 + + def fail_second_fsync(fd: int) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("archive fsync failed") + real_fsync(fd) + + monkeypatch.setattr(context_module.os, "fsync", fail_second_fsync) + + with pytest.raises(context_module.ContextPersistenceError) as raised: + await context.replace_history(_replacement(_message("new"))) + + assert raised.value.category == "rotation_archive" + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert not list(tmp_path.glob("context.jsonl*.tmp")) + assert not (tmp_path / "context_1.jsonl").exists() + + +@pytest.mark.asyncio +async def test_replace_history_atomic_replace_failure_preserves_old_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + monkeypatch.setattr( + context_module.os, + "replace", + lambda *_args: (_ for _ in ()).throw(OSError("replace failed")), + ) + + with pytest.raises(context_module.ContextPersistenceError) as raised: + await context.replace_history(_replacement(_message("new"))) + + assert raised.value.category == "atomic_replacement" + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert (tmp_path / "context_1.jsonl").read_bytes() == old_bytes + assert not list(tmp_path.glob("context.jsonl*.tmp")) + + +@pytest.mark.asyncio +async def test_directory_sync_failure_reports_coherent_visible_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + monkeypatch.setattr( + context_module, + "_sync_parent_directory", + lambda _parent: (_ for _ in ()).throw(OSError("directory fsync failed")), + ) + + with pytest.raises(context_module.ContextPersistenceError) as raised: + await context.replace_history(_replacement(_message("committed"))) + + assert raised.value.category == "visible_commit_durability" + assert context.history[-1] == _message("committed") + restored = Context(context_path) + assert await restored.restore() + assert _memory(restored) == _memory(context) + + +@pytest.mark.asyncio +async def test_unsupported_directory_sync_is_not_reported_as_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context_path.write_bytes(b"old") + context = Context(context_path) + monkeypatch.setattr(context_module, "_sync_parent_directory", lambda _parent: False) + + commit = await context.replace_history(_replacement(_message("new"))) + + assert commit.rotated_file is not None + assert context.history[-1] == _message("new") + + +@pytest.mark.asyncio +async def test_cancellation_during_atomic_replace_settles_coherent_new_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + replace_entered = threading.Event() + release_replace = threading.Event() + real_replace = context_module.os.replace + + def blocking_replace(source: Path, target: Path) -> None: + replace_entered.set() + release_replace.wait() + real_replace(source, target) + + monkeypatch.setattr(context_module.os, "replace", blocking_replace) + replacement = asyncio.create_task(context.replace_history(_replacement(_message("new")))) + assert await asyncio.to_thread(replace_entered.wait, 5) + + replacement.cancel() + replacement.cancel() + release_replace.set() + with pytest.raises(asyncio.CancelledError): + await replacement + + assert context.history[-1] == _message("new") + restored = Context(context_path) + assert await restored.restore() + assert _memory(restored) == _memory(context) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("boundary", ["temporary_write", "archive_write"]) +async def test_precommit_cancellation_preserves_old_generation_and_cleans_artifacts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + boundary: str, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + entered = threading.Event() + release = threading.Event() + if boundary == "temporary_write": + original = context_module._prepare_replacement_file + + def block_temporary_write(path: Path, records: Sequence[str]) -> Path: + entered.set() + release.wait() + return original(path, records) + + monkeypatch.setattr(context_module, "_prepare_replacement_file", block_temporary_write) + else: + original_archive = context_module._write_rotation_archive + + def block_archive_write(path: Path, content: bytes) -> None: + entered.set() + release.wait() + original_archive(path, content) + + monkeypatch.setattr(context_module, "_write_rotation_archive", block_archive_write) + + replacement = asyncio.create_task(context.replace_history(_replacement(_message("new")))) + assert await asyncio.to_thread(entered.wait, 5) + replacement.cancel() + release.set() + + with pytest.raises(asyncio.CancelledError): + await replacement + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert not list(tmp_path.glob("context.jsonl*.tmp")) + assert not (tmp_path / "context_1.jsonl").exists() + + +@pytest.mark.asyncio +async def test_cleanup_failure_retains_atomic_replace_error_as_primary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + real_unlink = Path.unlink + + monkeypatch.setattr( + context_module.os, + "replace", + lambda *_args: (_ for _ in ()).throw(OSError("replace failed")), + ) + + def fail_temp_cleanup(path: Path, missing_ok: bool = False) -> None: + if path.name.endswith(".tmp"): + raise OSError("cleanup failed") + real_unlink(path, missing_ok=missing_ok) + + monkeypatch.setattr(Path, "unlink", fail_temp_cleanup) + + with pytest.raises(context_module.ContextPersistenceError) as raised: + await context.replace_history(_replacement(_message("new"))) + + assert raised.value.category == "atomic_replacement" + assert "Cleanup also failed" in "\n".join(raised.value.__notes__) + + +@pytest.mark.asyncio +async def test_replace_history_serializes_with_concurrent_append( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + replace_entered = threading.Event() + release_replace = threading.Event() + real_replace = context_module.os.replace + + def blocking_replace(source: Path, target: Path) -> None: + replace_entered.set() + release_replace.wait() + real_replace(source, target) + + monkeypatch.setattr(context_module.os, "replace", blocking_replace) + replacement = asyncio.create_task( + context.replace_history(_replacement(_message("replacement"))) + ) + assert await asyncio.to_thread(replace_entered.wait, 5) + append = asyncio.create_task(context.append_message(_message("after replacement"))) + release_replace.set() + await asyncio.gather(replacement, append) + + assert [message.extract_text("") for message in context.history] == [ + "CHECKPOINT 0", + "replacement", + "after replacement", + ] + restored = Context(context_path) + assert await restored.restore() + assert list(restored.history) == list(context.history) + + +@pytest.mark.asyncio +async def test_rotation_reservation_failure_is_categorized_and_cleans_temp( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + + async def fail_reservation(_path: Path) -> Path | None: + raise OSError("listdir failed") + + monkeypatch.setattr(context_module, "next_available_rotation", fail_reservation) + + with pytest.raises(context_module.ContextPersistenceError) as raised: + await context.replace_history(_replacement(_message("new"))) + + assert raised.value.category == "rotation_archive" + assert isinstance(raised.value.__cause__, OSError) + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert not list(tmp_path.glob("context.jsonl*.tmp")) + assert not (tmp_path / "context_1.jsonl").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field", "invalid_value", "message"), + [ + ("system_prompt", 7, "system_prompt"), + ("messages", [_message("list")], "messages"), + ("messages", ({"role": "_usage", "token_count": 0},), "messages"), + ("token_count", True, "token_count"), + ("token_count", "7", "token_count"), + ("token_count", -1, "token_count"), + ("create_checkpoint", 1, "create_checkpoint"), + ("checkpoint_user_marker", 1, "checkpoint_user_marker"), + ], +) +async def test_replacement_exact_type_validation_precedes_lock_and_filesystem( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, + invalid_value: object, + message: str, +) -> None: + context_path = tmp_path / "context.jsonl" + context_path.write_bytes(b"old generation") + context = Context(context_path) + values: dict[str, object] = { + "system_prompt": "prompt", + "messages": (_message("valid"),), + "token_count": 1, + "create_checkpoint": True, + "checkpoint_user_marker": False, + } + values[field] = invalid_value + replacement = context_module.ContextReplacement( + system_prompt=cast(Any, values["system_prompt"]), + messages=cast(Any, values["messages"]), + token_count=cast(Any, values["token_count"]), + create_checkpoint=cast(Any, values["create_checkpoint"]), + checkpoint_user_marker=cast(Any, values["checkpoint_user_marker"]), + ) + + monkeypatch.setattr( + context_module.tempfile, + "mkstemp", + lambda *args, **kwargs: pytest.fail("invalid input touched filesystem"), + ) + async with context._mutation_lock: + with pytest.raises((TypeError, ValueError), match=message): + await asyncio.wait_for(context.replace_history(replacement), timeout=0.1) + + assert context_path.read_bytes() == b"old generation" + + +@pytest.mark.asyncio +async def test_checkpoint_marker_relationship_validation_precedes_filesystem( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(tmp_path / "context.jsonl") + monkeypatch.setattr( + context_module.tempfile, + "mkstemp", + lambda *args, **kwargs: pytest.fail("invalid input touched filesystem"), + ) + + with pytest.raises(ValueError, match="checkpoint_user_marker"): + await context.replace_history( + _replacement(create_checkpoint=False, checkpoint_user_marker=True) + ) + + +def test_context_persistence_error_renders_only_safe_path() -> None: + absolute_path = Path("/private/session-secret/context.jsonl") + error = context_module.ContextPersistenceError( + "replace_history", "rotation_archive", absolute_path + ) + + assert str(absolute_path) not in str(error) + assert "session-secret" not in str(error) + assert "context.jsonl" in str(error) + + +def test_visible_commit_durability_error_is_explicit() -> None: + error = context_module.ContextPersistenceError( + "replace_history", + "visible_commit_durability", + Path("/private/session-secret/context.jsonl"), + ) + + assert "new generation is visible" in str(error) + assert "power-loss durability is uncertain" in str(error) + + +@pytest.mark.asyncio +async def test_cancellation_before_archive_reservation_keeps_old_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + read_entered = threading.Event() + release_read = threading.Event() + original_read = context_module._read_live_bytes + + def blocking_read(path: Path) -> bytes | None: + read_entered.set() + release_read.wait() + return original_read(path) + + monkeypatch.setattr(context_module, "_read_live_bytes", blocking_read) + replacement = asyncio.create_task(context.replace_history(_replacement(_message("new")))) + assert await asyncio.to_thread(read_entered.wait, 5) + replacement.cancel() + release_read.set() + + with pytest.raises(asyncio.CancelledError): + await replacement + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert not list(tmp_path.glob("context.jsonl*.tmp")) + assert not (tmp_path / "context_1.jsonl").exists() + + +@pytest.mark.asyncio +async def test_cancellation_immediately_before_replace_keeps_old_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + old_bytes = context_path.read_bytes() + old_memory = _memory(context) + checkpoint_entered = asyncio.Event() + release_checkpoint = asyncio.Event() + + async def pause_before_replace() -> None: + checkpoint_entered.set() + await release_checkpoint.wait() + + monkeypatch.setattr(context_module, "_before_replacement_commit", pause_before_replace) + replacement = asyncio.create_task(context.replace_history(_replacement(_message("new")))) + await checkpoint_entered.wait() + replacement.cancel() + release_checkpoint.set() + + with pytest.raises(asyncio.CancelledError): + await replacement + assert context_path.read_bytes() == old_bytes + assert _memory(context) == old_memory + assert not list(tmp_path.glob("context.jsonl*.tmp")) + assert not (tmp_path / "context_1.jsonl").exists() + + +@pytest.mark.asyncio +async def test_cancellation_during_directory_sync_propagates_after_coherent_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + sync_entered = threading.Event() + release_sync = threading.Event() + original_sync = context_module._sync_parent_directory + + def blocking_sync(parent: Path) -> bool: + sync_entered.set() + release_sync.wait() + return original_sync(parent) + + monkeypatch.setattr(context_module, "_sync_parent_directory", blocking_sync) + replacement = asyncio.create_task(context.replace_history(_replacement(_message("new")))) + assert await asyncio.to_thread(sync_entered.wait, 5) + replacement.cancel() + release_sync.set() + + with pytest.raises(asyncio.CancelledError): + await replacement + assert context.history[-1] == _message("new") + restored = Context(context_path) + assert await restored.restore() + assert _memory(restored) == _memory(context) + + +@pytest.mark.asyncio +async def test_second_cancellation_after_commit_settlement_has_started_is_settled( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_path = tmp_path / "context.jsonl" + context = Context(context_path) + await context.append_message(_message("existing")) + replace_entered = threading.Event() + release_replace = threading.Event() + settlement_entered = asyncio.Event() + real_replace = context_module.os.replace + real_wait = context_module.asyncio.wait + + def blocking_replace(source: Path, target: Path) -> None: + replace_entered.set() + release_replace.wait() + real_replace(source, target) + + async def tracked_wait(fs: Any, **kwargs: Any) -> Any: + settlement_entered.set() + return await real_wait(fs, **kwargs) + + monkeypatch.setattr(context_module.os, "replace", blocking_replace) + monkeypatch.setattr(context_module.asyncio, "wait", tracked_wait) + replacement = asyncio.create_task(context.replace_history(_replacement(_message("new")))) + assert await asyncio.to_thread(replace_entered.wait, 5) + replacement.cancel() + await settlement_entered.wait() + replacement.cancel() + release_replace.set() + + with pytest.raises(asyncio.CancelledError): + await replacement + assert context.history[-1] == _message("new") + restored = Context(context_path) + assert await restored.restore() + assert _memory(restored) == _memory(context) + + +def test_parent_directory_sync_non_posix_is_explicitly_unsupported( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(context_module.os, "name", "nt") + monkeypatch.setattr( + context_module.os, + "open", + lambda *args, **kwargs: pytest.fail("non-POSIX directory sync opened a directory"), + ) + + assert context_module._sync_parent_directory(tmp_path) is False + + +@pytest.mark.asyncio +async def test_expected_generation_conflict_precedes_replacement_io( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(tmp_path / "context.jsonl") + expected_generation = context.mutation_generation + await context.append_message(_message("concurrent")) + before_bytes = context.file_backend.read_bytes() + monkeypatch.setattr( + context_module.tempfile, + "mkstemp", + lambda *args, **kwargs: pytest.fail("generation conflict touched replacement I/O"), + ) + + with pytest.raises(context_module.ContextGenerationConflictError): + await context.replace_history( + _replacement(_message("stale")), + expected_generation=expected_generation, + ) + + assert context.file_backend.read_bytes() == before_bytes + assert context.history[-1] == _message("concurrent") + + +@pytest.mark.asyncio +async def test_revert_generation_conflicts_stop_after_bounded_attempts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(tmp_path / "context.jsonl") + await context.append_message(_message("before")) + await context.checkpoint(add_user_message=False) + attempts = 0 + + async def conflict_then_forbidden(*_args: object, **_kwargs: object) -> Any: + nonlocal attempts + attempts += 1 + if attempts <= 3: + raise context_module.ContextGenerationConflictError(attempts - 1, attempts) + raise AssertionError("revert retried past its conflict budget") + + monkeypatch.setattr(context, "replace_history", conflict_then_forbidden) + + with pytest.raises(context_module.ContextGenerationConflictError): + await context.revert_to(0) + + assert attempts == 3 + + +@pytest.mark.parametrize("unsupported_errno", [errno.EINVAL, errno.ENOTSUP]) +def test_parent_directory_sync_treats_known_posix_errors_as_unsupported( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + unsupported_errno: int, +) -> None: + monkeypatch.setattr(context_module.os, "name", "posix") + monkeypatch.setattr(context_module.os, "open", lambda *_args: 17) + monkeypatch.setattr( + context_module.os, + "fsync", + lambda _fd: (_ for _ in ()).throw(OSError(unsupported_errno, "unsupported")), + ) + closed: list[int] = [] + monkeypatch.setattr(context_module.os, "close", closed.append) + + assert context_module._sync_parent_directory(tmp_path) is False + assert closed == [17] diff --git a/tests/core/test_dynamic_injection_budget.py b/tests/core/test_dynamic_injection_budget.py index 18937603..8746964e 100644 --- a/tests/core/test_dynamic_injection_budget.py +++ b/tests/core/test_dynamic_injection_budget.py @@ -3,6 +3,8 @@ from types import SimpleNamespace from typing import cast +import pytest + from pythinker_code.config import Config, LLMModel from pythinker_code.soul.agent import Runtime from pythinker_code.soul.dynamic_injection import ( @@ -25,6 +27,16 @@ def test_collect_within_budget_orders_by_priority_and_caps(): assert sum(item.token_estimate or 0 for item in out) <= 20 +def test_collect_within_budget_preserves_equal_priority_order_past_single_digits(): + candidates = [ + InjectionCandidate(type=f"candidate-{index}", content="x" * 4) for index in range(12) + ] + + out = collect_within_budget(candidates, budget_tokens=20) + + assert [candidate.type for candidate in out] == [candidate.type for candidate in candidates] + + def test_collect_within_budget_truncates_deterministically(): out = collect_within_budget( [InjectionCandidate(type="x", content="alpha\nbeta\ngamma" * 100, priority=10)], @@ -35,6 +47,53 @@ def test_collect_within_budget_truncates_deterministically(): assert (out[0].token_estimate or 0) <= 10 +def test_collect_within_budget_recomputes_untrusted_candidate_estimate(): + out = collect_within_budget( + [ + InjectionCandidate( + type="understated", + content="x" * 40, + priority=10, + token_estimate=1, + ) + ], + budget_tokens=5, + ) + + assert len(out) == 1 + assert out[0].content.endswith("…") + assert out[0].token_estimate == 5 + + +@pytest.mark.parametrize( + ("budget_tokens", "expected_count"), + [(0, 0), (1, 1)], +) +def test_collect_within_budget_preserves_empty_candidate_at_minimum_budget( + budget_tokens: int, expected_count: int +): + candidate = InjectionCandidate(type="empty", content="", token_estimate=0) + + out = collect_within_budget([candidate], budget_tokens=budget_tokens) + + assert len(out) == expected_count + if out: + assert out[0].content == "" + assert out[0].token_estimate == 1 + + +def test_failed_first_truncation_attempt_prevents_lower_candidate_truncation(): + out = collect_within_budget( + [ + InjectionCandidate(type="empty-prefix", content="\n" * 20, priority=20), + InjectionCandidate(type="lower", content="lower candidate" * 10, priority=10), + ], + budget_tokens=1, + ) + + assert out == [] + + def test_context_budget_uses_ceiling_and_available_context(): assert ( ContextBudget( diff --git a/tests/core/test_dynamic_injection_hooks.py b/tests/core/test_dynamic_injection_hooks.py index 8c1a1689..176eafdb 100644 --- a/tests/core/test_dynamic_injection_hooks.py +++ b/tests/core/test_dynamic_injection_hooks.py @@ -2,14 +2,18 @@ from __future__ import annotations +import asyncio from pathlib import Path -from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, patch +import pytest +from pythinker_core.message import Message from pythinker_core.tooling.empty import EmptyToolset +import pythinker_code.soul.context as context_module from pythinker_code.soul.agent import Agent, Runtime -from pythinker_code.soul.context import Context +from pythinker_code.soul.compaction import CompactionResult +from pythinker_code.soul.context import Context, ContextGenerationConflictError, ContextReplacement from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider from pythinker_code.soul.pythinkersoul import PythinkerSoul @@ -17,10 +21,14 @@ class _BoomProvider(DynamicInjectionProvider): """Buggy provider that raises from both hooks.""" + def __init__(self) -> None: + self.on_context_compacted_calls = 0 + async def get_injections(self, history, soul) -> list[DynamicInjection]: # noqa: ARG002 raise RuntimeError("boom") async def on_context_compacted(self) -> None: + self.on_context_compacted_calls += 1 raise RuntimeError("boom-compact") @@ -39,6 +47,24 @@ async def on_context_compacted(self) -> None: self.on_context_compacted_calls += 1 +class _BlockingProvider(_RecordingProvider): + def __init__(self, entered: asyncio.Event, release: asyncio.Event) -> None: + super().__init__() + self._entered = entered + self._release = release + + async def on_context_compacted(self) -> None: + self.on_context_compacted_calls += 1 + self._entered.set() + await self._release.wait() + + +class _SelfCancellingProvider(_RecordingProvider): + async def on_context_compacted(self) -> None: + self.on_context_compacted_calls += 1 + raise asyncio.CancelledError() + + async def test_compacted_hook_isolates_provider_failures(runtime: Runtime, tmp_path: Path) -> None: """A buggy provider must not abort compaction notification of later providers.""" agent = Agent( @@ -52,65 +78,146 @@ async def test_compacted_hook_isolates_provider_failures(runtime: Runtime, tmp_p recorder = _RecordingProvider() soul._injection_providers = [_BoomProvider(), recorder] # pyright: ignore[reportPrivateUsage] - await soul._notify_injection_providers_compacted() # pyright: ignore[reportPrivateUsage] + await soul.notify_history_rebuilt() + + assert recorder.on_context_compacted_calls == 1 + + +async def test_compacted_hook_isolates_provider_originated_cancellation( + runtime: Runtime, tmp_path: Path +) -> None: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + soul = PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + cancelling = _SelfCancellingProvider() + recorder = _RecordingProvider() + soul._injection_providers = [cancelling, recorder] # pyright: ignore[reportPrivateUsage] + + await soul.notify_history_rebuilt() + + assert cancelling.on_context_compacted_calls == 1 + assert recorder.on_context_compacted_calls == 1 + +async def test_stale_operation_does_not_claim_concurrent_replacement_commit( + runtime: Runtime, tmp_path: Path +) -> None: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "concurrent.jsonl") + soul = PythinkerSoul(agent, context=context) + recorder = _RecordingProvider() + soul._injection_providers = [recorder] # pyright: ignore[reportPrivateUsage] + expected_generation = context.mutation_generation + stale_entered = asyncio.Event() + release_stale = asyncio.Event() + + async def stale_replacement(): + stale_entered.set() + await release_stale.wait() + return await context.replace_history( + ContextReplacement(None, (Message(role="user", content="stale"),), 1, False), + expected_generation=expected_generation, + ) + + stale = asyncio.create_task(soul._complete_history_replacement(stale_replacement())) # pyright: ignore[reportPrivateUsage] + try: + await asyncio.wait_for(stale_entered.wait(), timeout=5.0) + await soul._complete_history_replacement( # pyright: ignore[reportPrivateUsage] + context.replace_history( + ContextReplacement(None, (Message(role="user", content="winner"),), 1, False), + expected_generation=expected_generation, + ) + ) + finally: + release_stale.set() + + with pytest.raises(ContextGenerationConflictError): + await asyncio.wait_for(stale, timeout=5.0) + assert recorder.on_context_compacted_calls == 1 + assert soul._request_lifecycle.history_generation == 1 # pyright: ignore[reportPrivateUsage] + + +async def test_revert_visible_durability_error_rearms_all_providers_before_propagating( + runtime: Runtime, + tmp_path: Path, + monkeypatch, +) -> None: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "history.jsonl") + soul = PythinkerSoul(agent, context=context) + boom = _BoomProvider() + recorder = _RecordingProvider() + soul._injection_providers = [boom, recorder] # pyright: ignore[reportPrivateUsage] + await context.append_message(Message(role="user", content="before")) + await context.checkpoint(add_user_message=False) + await context.append_message(Message(role="assistant", content="after")) + lifecycle_generation = soul._request_lifecycle.history_generation # pyright: ignore[reportPrivateUsage] + + def fail_directory_sync(_path: Path) -> bool: + raise OSError("fsync failed") + + monkeypatch.setattr( + context_module, + "_sync_parent_directory", + fail_directory_sync, + ) + + with pytest.raises( + context_module.ContextPersistenceError, + match="power-loss durability is uncertain", + ): + await soul._revert_context_to(0) # pyright: ignore[reportPrivateUsage] + + assert [message.extract_text("") for message in context.history] == ["before"] + assert soul._request_lifecycle.history_generation == lifecycle_generation + 1 # pyright: ignore[reportPrivateUsage] + assert boom.on_context_compacted_calls == 1 assert recorder.on_context_compacted_calls == 1 -def _make_compactable_soul() -> Any: - """Minimal PythinkerSoul bypassing __init__, just enough for compact_context(). +async def _make_compactable_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: + """Build a real soul through its constructor and mock only the compaction LLM boundary. - Mirrors the pattern used in tests/telemetry/test_instrumentation.py. + Exercising the production constructor (rather than ``object.__new__`` with hand-assigned + private fields) keeps the test honest against constructor drift: a field the compaction + path starts reading is wired by ``__init__`` here instead of silently defaulting to a mock. """ - soul = object.__new__(PythinkerSoul) - - runtime = MagicMock() - runtime.llm = MagicMock() - runtime.session.id = "test-session" - runtime.role = "non-root" # skip active-task-snapshot branch - runtime.background_tasks = MagicMock() - soul._runtime = runtime - - ctx = MagicMock() - ctx.token_count = 10_000 - ctx.history = [] - ctx.clear = AsyncMock() - ctx.write_system_prompt = AsyncMock() - ctx.append_message = AsyncMock() - ctx.update_token_count = AsyncMock() - soul._context = ctx - - soul._hook_engine = MagicMock() - soul._hook_engine.trigger = AsyncMock() - - soul._compaction = MagicMock() - - soul._agent = MagicMock() - soul._agent.system_prompt = "sys" - - loop_control = MagicMock() - loop_control.max_retries_per_step = 1 - soul._loop_control = loop_control - - soul._checkpoint = AsyncMock() - - fake_result = MagicMock() - # Non-empty to satisfy the post-compaction guard against producing no - # messages; the exact contents do not matter for the injection-hook test. - fake_result.messages = [MagicMock()] - fake_result.estimated_token_count = 2_000 - # No usage, so the cumulative-usage accumulation (subagent-2) is skipped — - # this harness bypasses __init__. - fake_result.usage = None - soul._run_with_connection_recovery = AsyncMock(return_value=fake_result) - - soul._injection_providers = [] + runtime.session.state.active_skills = [] + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "history.jsonl") + await context.append_message(Message(role="user", content="compact me")) + soul = PythinkerSoul(agent, context=context) + result = CompactionResult( + messages=[Message(role="user", content="compacted")], + usage=None, + ) + soul._run_with_connection_recovery = AsyncMock(return_value=result) # pyright: ignore[reportPrivateUsage] return soul -async def test_compact_context_notifies_injection_providers() -> None: +async def test_compact_context_notifies_injection_providers( + runtime: Runtime, tmp_path: Path +) -> None: """compact_context() must await on_context_compacted on every registered provider.""" - soul = _make_compactable_soul() + soul = await _make_compactable_soul(runtime, tmp_path) provider_a = _RecordingProvider() provider_b = _RecordingProvider() soul.add_injection_provider(provider_a) @@ -123,9 +230,11 @@ async def test_compact_context_notifies_injection_providers() -> None: assert provider_b.on_context_compacted_calls == 1 -async def test_compact_context_notifies_surviving_providers_after_failure() -> None: +async def test_compact_context_notifies_surviving_providers_after_failure( + runtime: Runtime, tmp_path: Path +) -> None: """A provider raising in its hook must not prevent later providers from being notified.""" - soul = _make_compactable_soul() + soul = await _make_compactable_soul(runtime, tmp_path) boom = _BoomProvider() recorder = _RecordingProvider() soul.add_injection_provider(boom) diff --git a/tests/core/test_load_agent.py b/tests/core/test_load_agent.py index be63cb39..c191e802 100644 --- a/tests/core/test_load_agent.py +++ b/tests/core/test_load_agent.py @@ -23,6 +23,7 @@ from pythinker_code.soul.approval import Approval from pythinker_code.soul.denwarenji import DenwaRenji from pythinker_code.soul.toolset import PythinkerToolset +from pythinker_code.subagents.models import AgentTypeDefinition, ToolPolicy from pythinker_code.utils.environment import Environment @@ -77,11 +78,29 @@ async def test_render_agent_system_prompt_builds_args_without_runtime( assert "${PYTHINKER_" not in prompt +@pytest.mark.asyncio +async def test_system_prompt_skill_policy_is_static_and_omits_catalogue_paths( + temp_work_dir: HostPath, + config: Config, +) -> None: + from pythinker_code.agentspec import DEFAULT_AGENT_FILE + from pythinker_code.soul.agent import render_agent_system_prompt + + first = await render_agent_system_prompt(DEFAULT_AGENT_FILE, temp_work_dir, config) + second = await render_agent_system_prompt(DEFAULT_AGENT_FILE, temp_work_dir, config) + + first_skills = first.split("## 12. Skills", 1)[1] + second_skills = second.split("## 12. Skills", 1)[1] + assert first_skills == second_skills + assert "Task-relevant skill candidates arrive with each request" in first_skills + assert "Path:" not in first_skills + + def test_render_agents_md_reminder_present(builtin_args: BuiltinSystemPromptArgs): """The merged AGENTS.md renders as an authoritative, fenced body. AGENTS.md is delivered as a session-start preamble (a user-role system-reminder), - not baked into the system prompt — see render_agents_md_reminder / _with_agents_md_preamble. + not baked into the system prompt — see render_agents_md_reminder / RequestAssembler. """ from pythinker_code.soul.agent import render_agents_md_reminder @@ -522,6 +541,65 @@ async def test_load_agent_registers_builtin_subagent_types(runtime: Runtime): assert builtin_type.agent_file.samefile(builtin_type_yaml) +@pytest.fixture +def agent_projection_files(tmp_path: Path) -> tuple[Path, Path]: + (tmp_path / "root-system.md").write_text("Root prompt", encoding="utf-8") + (tmp_path / "child-system.md").write_text("Child prompt", encoding="utf-8") + child_file = tmp_path / "child.yaml" + child_file.write_text( + "version: 1\n" + "agent:\n" + ' name: "Child"\n' + " system_prompt_path: ./child-system.md\n" + ' tools: ["pythinker_code.tools.think:Think"]\n' + ' allowed_tools: ["pythinker_code.tools.think:Think"]\n' + ' model: "characterized-model"\n' + ' when_to_use: "Use for exact contract tests."\n' + " hidden: true\n", + encoding="utf-8", + ) + root_file = tmp_path / "root.yaml" + root_file.write_text( + "version: 1\n" + "agent:\n" + ' name: "Root"\n' + " system_prompt_path: ./root-system.md\n" + ' tools: ["pythinker_code.tools.think:Think"]\n' + " subagents:\n" + " analyst:\n" + " path: ./child.yaml\n" + ' description: "Literal projected agent"\n', + encoding="utf-8", + ) + return root_file, child_file + + +async def test_load_agent_preserves_literal_type_projection_and_toolset_facade( + runtime: Runtime, + agent_projection_files: tuple[Path, Path], +) -> None: + root_file, child_file = agent_projection_files + + agent = await load_agent(root_file, runtime, mcp_configs=[]) + + assert runtime.labor_market.require_builtin_type("analyst") == AgentTypeDefinition( + name="analyst", + description="Literal projected agent", + agent_file=child_file, + when_to_use="Use for exact contract tests.", + default_model="characterized-model", + tool_policy=ToolPolicy( + mode="allowlist", + tools=("pythinker_code.tools.think:Think",), + ), + supports_background=False, + required_mcp_servers=(), + ) + assert isinstance(agent.toolset, PythinkerToolset) + assert runtime.mcp_status == agent.toolset.mcp_status_snapshot + assert agent.toolset.find("Think") is not None + + async def test_load_agent_starts_mcp_in_background(runtime: Runtime, monkeypatch): called: dict[str, bool] = {} @@ -610,3 +688,67 @@ def system_prompt_file() -> Generator[Path, Any, Any]: system_md.write_text("Test system prompt with ${PYTHINKER_NOW} and ${CUSTOM_ARG}") yield system_md + + +def test_extend_escaping_agent_roots_is_rejected(tmp_path: Path) -> None: + # An `extend:` that traverses outside the spec's own directory (and the + # built-in agents dir) is rejected fail-closed as defense-in-depth, since + # the resolved path is otherwise loaded directly. + from pythinker_code.agentspec import AgentSpecError, load_agent_spec + + (tmp_path / "outside-system.md").write_text("outside", encoding="utf-8") + (tmp_path / "outside.yaml").write_text( + "version: 1\nagent:\n name: outside\n" + " system_prompt_path: ./outside-system.md\n tools: []\n", + encoding="utf-8", + ) + agents = tmp_path / "agents" + agents.mkdir() + (agents / "system.md").write_text("child", encoding="utf-8") + (agents / "child.yaml").write_text( + "version: 1\nagent:\n name: child\n" + " system_prompt_path: ./system.md\n tools: []\n" + " extend: ../outside.yaml\n", + encoding="utf-8", + ) + + with pytest.raises(AgentSpecError, match="outside the permitted"): + load_agent_spec(agents / "child.yaml") + + +def test_subagent_path_escaping_agent_roots_is_rejected(tmp_path: Path) -> None: + from pythinker_code.agentspec import AgentSpecError, load_agent_spec + + (tmp_path / "outside.yaml").write_text("version: 1\nagent:\n name: x\n", encoding="utf-8") + agents = tmp_path / "agents" + agents.mkdir() + (agents / "system.md").write_text("root", encoding="utf-8") + (agents / "root.yaml").write_text( + "version: 1\nagent:\n name: root\n" + " system_prompt_path: ./system.md\n tools: []\n" + " subagents:\n analyst:\n path: ../outside.yaml\n" + ' description: "d"\n', + encoding="utf-8", + ) + + with pytest.raises(AgentSpecError, match="outside the permitted"): + load_agent_spec(agents / "root.yaml") + + +def test_sibling_extend_within_agent_root_still_loads(tmp_path: Path) -> None: + # Regression guard: the containment check must not reject the normal + # `./sibling.yaml` shape every shipped spec uses. + from pythinker_code.agentspec import load_agent_spec + + (tmp_path / "base-system.md").write_text("base", encoding="utf-8") + (tmp_path / "base.yaml").write_text( + "version: 1\nagent:\n name: base\n system_prompt_path: ./base-system.md\n tools: []\n", + encoding="utf-8", + ) + (tmp_path / "child.yaml").write_text( + "version: 1\nagent:\n name: child\n extend: ./base.yaml\n", + encoding="utf-8", + ) + + resolved = load_agent_spec(tmp_path / "child.yaml") + assert resolved.name == "child" diff --git a/tests/core/test_mcp_cleanup.py b/tests/core/test_mcp_cleanup.py index 284b7b11..35b448c0 100644 --- a/tests/core/test_mcp_cleanup.py +++ b/tests/core/test_mcp_cleanup.py @@ -57,3 +57,28 @@ async def test_cleanup_times_out_hung_close(monkeypatch: pytest.MonkeyPatch) -> await asyncio.wait_for(ts.cleanup(), timeout=2.0) # completes fast despite the hang assert closed == ["good"] + + +async def test_cleanup_cancels_background_load_before_closing_clients() -> None: + entered = asyncio.Event() + cancelled = asyncio.Event() + closed: list[str] = [] + ts = PythinkerToolset() + ts._mcp_servers["alpha"] = _info(_GoodClient(closed, "alpha")) + + async def load() -> None: + entered.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + + loading = asyncio.create_task(load()) + ts._mcp_loading_task = loading + await entered.wait() + + await ts.cleanup() + + assert loading.cancelled() + assert cancelled.is_set() + assert closed == ["alpha"] diff --git a/tests/core/test_mcp_lifecycle.py b/tests/core/test_mcp_lifecycle.py index f692aadd..105a048f 100644 --- a/tests/core/test_mcp_lifecycle.py +++ b/tests/core/test_mcp_lifecycle.py @@ -14,6 +14,7 @@ MCPServerInfo, MCPTool, PythinkerToolset, + _discover_optional_capability, _make_mcp_live_refresh_handler, ) @@ -307,3 +308,235 @@ async def _connect( assert toolset.find("GoodTool") is good_tool assert runtime.mcp_tools["mcp__good__GoodTool"] is good_tool + + +@pytest.mark.asyncio +async def test_optional_inventory_distinguishes_method_not_found_and_transient_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from mcp.shared.exceptions import McpError + from mcp.types import METHOD_NOT_FOUND, ErrorData + + import pythinker_code.soul.toolset as toolset_mod + + debug_messages: list[str] = [] + warning_messages: list[str] = [] + + def capture_debug(message: str, **_kwargs: object) -> None: + debug_messages.append(message) + + def capture_warning(message: str, **_kwargs: object) -> None: + warning_messages.append(message) + + monkeypatch.setattr(toolset_mod.logger, "debug", capture_debug) + monkeypatch.setattr(toolset_mod.logger, "warning", capture_warning) + + async def unsupported() -> list[object]: + raise McpError(ErrorData(code=METHOD_NOT_FOUND, message="not supported")) + + async def transient() -> list[object]: + raise ConnectionError("inventory transport reset") + + assert await _discover_optional_capability("alpha", "resources", unsupported) == [] + assert debug_messages == ["MCP server {name} does not support {cap}"] + assert warning_messages == [] + + assert await _discover_optional_capability("alpha", "prompts", transient) == [] + assert warning_messages == [ + "MCP server {name} failed listing {cap} (transient?); treating as empty: {error}" + ] + + +@pytest.mark.asyncio +async def test_list_change_storm_completes_every_refresh_without_orphan_tasks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import mcp.types + + toolset = PythinkerToolset() + runtime = _runtime() + toolset._mcp_servers["alpha"] = MCPServerInfo( + status="connected", + client=cast(Any, SimpleNamespace()), + tools=[], + resources=[], + prompts=[], + ) + entered = asyncio.Semaphore(0) + release = asyncio.Event() + completed: list[int] = [] + + async def refresh(_server_name: str, _runtime: Any) -> None: + index = len(completed) + entered.release() + await release.wait() + completed.append(index) + + monkeypatch.setattr(toolset, "refresh_mcp_server", refresh) + + class _FakeClient: + pass + + handler = _make_mcp_live_refresh_handler(_FakeClient(), toolset, runtime, "alpha") + tasks = [ + asyncio.create_task(handler.on_tool_list_changed(mcp.types.ToolListChangedNotification())) + for _ in range(5) + ] + for _ in tasks: + await entered.acquire() + release.set() + await asyncio.gather(*tasks) + + assert len(completed) == 5 + assert all(task.done() for task in tasks) + + +@pytest.mark.asyncio +async def test_refresh_racing_disconnect_cannot_republish_disconnected_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + toolset = PythinkerToolset() + runtime = _runtime() + old_tool = _fake_mcp_tool("alpha", "OldTool") + new_tool = _fake_mcp_tool("alpha", "NewTool") + inventory_entered = asyncio.Event() + release_inventory = asyncio.Event() + info = MCPServerInfo( + status="connected", + client=cast(Any, SimpleNamespace(close=AsyncMock())), + tools=[old_tool], + resources=[], + prompts=[], + server_config={"command": "echo"}, + ) + toolset._mcp_servers["alpha"] = info + toolset._publish_connected_mcp_tools(runtime) + + async def inventory( + _server_name: str, _server_info: MCPServerInfo, _runtime: Any + ) -> tuple[list[MCPTool[Any]], list[Any], list[Any]]: + inventory_entered.set() + await release_inventory.wait() + return [new_tool], [], [] + + monkeypatch.setattr(toolset, "_inventory_mcp_server", inventory) + refresh_task = asyncio.create_task(toolset.refresh_mcp_server("alpha", runtime)) + await inventory_entered.wait() + await toolset.disconnect_mcp_server("alpha", runtime) + release_inventory.set() + await refresh_task + + assert info.status == "failed" + assert toolset.find("OldTool") is None + assert toolset.find("NewTool") is None + assert runtime.mcp_tools == {} + + +@pytest.mark.asyncio +async def test_rebuild_rolls_back_after_partial_publication_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + toolset = PythinkerToolset() + runtime = _runtime() + original = _fake_mcp_tool("original", "Original") + alpha = _fake_mcp_tool("alpha", "Alpha") + beta = _fake_mcp_tool("beta", "Beta") + toolset.add(original) + runtime.mcp_tools["mcp__original__Original"] = original + toolset._mcp_servers = { + "alpha": MCPServerInfo( + status="connected", + client=cast(Any, SimpleNamespace()), + tools=[alpha], + resources=[], + prompts=[], + ), + "beta": MCPServerInfo( + status="connected", + client=cast(Any, SimpleNamespace()), + tools=[beta], + resources=[], + prompts=[], + ), + } + original_register = toolset._register_mcp_tools + + def fail_second_server(server_name: str, tools: list[MCPTool[Any]]) -> None: + if server_name == "beta": + raise RuntimeError("publication failed") + original_register(server_name, tools) + + monkeypatch.setattr(toolset, "_register_mcp_tools", fail_second_server) + + with pytest.raises(RuntimeError, match="publication failed"): + toolset._rebuild_published_mcp_tools(runtime) + + assert toolset.find("Original") is original + assert toolset.find("Alpha") is None + assert toolset.find("Beta") is None + assert runtime.mcp_tools == {"mcp__original__Original": original} + + +@pytest.mark.asyncio +async def test_hung_connect_reports_timeout_without_publishing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + toolset = PythinkerToolset() + runtime = _runtime() + runtime.config.mcp.client.startup_timeout_ms = 0 + info = MCPServerInfo( + status="pending", + client=cast(Any, SimpleNamespace()), + tools=[], + resources=[], + prompts=[], + ) + + async def hung_inventory( + _server_name: str, _server_info: MCPServerInfo, _runtime: Any + ) -> tuple[list[MCPTool[Any]], list[Any], list[Any]]: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + monkeypatch.setattr(toolset, "_inventory_mcp_server", hung_inventory) + + server_name, error = await toolset._connect_mcp_server("alpha", info, runtime) + + assert server_name == "alpha" + assert isinstance(error, TimeoutError) + assert info.status == "failed" + assert info.error is not None and "startup timed out" in info.error + assert info.tools == [] + + +@pytest.mark.asyncio +async def test_duplicate_server_name_connects_once_with_last_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from fastmcp.mcp_config import MCPConfig + + toolset = PythinkerToolset() + runtime = _runtime() + connected: list[str] = [] + + async def connect( + server_name: str, server_info: MCPServerInfo, _runtime: Any + ) -> tuple[str, Exception | None]: + connected.append(server_name) + server_info.status = "connected" + return server_name, None + + monkeypatch.setattr(toolset, "_connect_mcp_server", connect) + monkeypatch.setattr( + "pythinker_code.soul.toolset._configure_mcp_client_handlers", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr("fastmcp.Client", lambda *_args, **_kwargs: SimpleNamespace()) + first = MCPConfig.model_validate({"mcpServers": {"alpha": {"command": "first-command"}}}) + second = MCPConfig.model_validate({"mcpServers": {"alpha": {"command": "second-command"}}}) + + await toolset.load_mcp_tools([first, second], runtime, in_background=False) + + assert connected == ["alpha"] + assert tuple(toolset.mcp_servers) == ("alpha",) + assert toolset.mcp_servers["alpha"].server_config.command == "second-command" diff --git a/tests/core/test_notifications.py b/tests/core/test_notifications.py index f10ed18c..c06e2060 100644 --- a/tests/core/test_notifications.py +++ b/tests/core/test_notifications.py @@ -91,6 +91,7 @@ def _runtime_with_llm(runtime: Runtime, llm: LLM) -> Runtime: environment=runtime.environment, notifications=runtime.notifications, background_tasks=runtime.background_tasks, + skill_catalog=runtime.skill_catalog, skills=runtime.skills, oauth=runtime.oauth, additional_dirs=runtime.additional_dirs, diff --git a/tests/core/test_plan_mode.py b/tests/core/test_plan_mode.py index 60a282a5..192a70e0 100644 --- a/tests/core/test_plan_mode.py +++ b/tests/core/test_plan_mode.py @@ -13,6 +13,7 @@ from pythinker_code.soul.agent import Agent, Runtime from pythinker_code.soul.approval import Approval from pythinker_code.soul.context import Context +from pythinker_code.soul.dynamic_injections.plan_mode import PlanModeInjectionProvider from pythinker_code.soul.pythinkersoul import PythinkerSoul from pythinker_code.soul.toolset import PythinkerToolset from pythinker_code.tools.file.replace import StrReplaceFile @@ -235,7 +236,12 @@ async def test_manual_toggle_defers_activation_to_injection( assert soul._pending_plan_activation_injection is True assert soul.context.history == [] - injections = await soul._collect_injections() + provider = next( + provider + for provider in soul._injection_providers + if isinstance(provider, PlanModeInjectionProvider) + ) + injections = await provider.prepare_injections(soul.context.history, soul) plan_injections = [i for i in injections if i.type.startswith("plan_mode")] assert len(plan_injections) == 1 @@ -259,7 +265,12 @@ async def test_manual_exit_clears_pending_activation_injection( assert soul.plan_mode is False assert soul._pending_plan_activation_injection is False - injections = await soul._collect_injections() + provider = next( + provider + for provider in soul._injection_providers + if isinstance(provider, PlanModeInjectionProvider) + ) + injections = await provider.prepare_injections(soul.context.history, soul) assert [i for i in injections if i.type.startswith("plan_mode")] == [] async def test_tool_toggle_does_not_queue_manual_activation_injection( diff --git a/tests/core/test_prompt_manifest_slash.py b/tests/core/test_prompt_manifest_slash.py new file mode 100644 index 00000000..f3bb3b16 --- /dev/null +++ b/tests/core/test_prompt_manifest_slash.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Awaitable +from pathlib import Path +from typing import cast + +import pytest +from pythinker_core.tooling.empty import EmptyToolset + +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul +from pythinker_code.soul.request_assembly import ( + FragmentOutcome, + FragmentPersistence, + FragmentRequirement, + FragmentStatus, + RequestManifest, + RequestStatus, +) +from pythinker_code.soul.slash import registry as soul_slash_registry +from pythinker_code.telemetry import metrics +from pythinker_code.wire.types import TextPart + + +def _make_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: + agent = Agent( + name="manifest test", + system_prompt="Static system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + return PythinkerSoul(agent, context=Context(file_backend=tmp_path / "context.jsonl")) + + +async def _run_prompt_manifest(soul: PythinkerSoul) -> None: + command = soul_slash_registry.find_command("prompt-manifest") + assert command is not None + pending = command.func(soul, "") + if isinstance(pending, Awaitable): + await pending + + +def _manifest( + status: RequestStatus, + *, + reason_code: str | None = None, + outcome_status: FragmentStatus = FragmentStatus.INCLUDED, + source: str = "permissions_state", + key: str = "permissions", + outcome_reason: str | None = None, +) -> RequestManifest: + return RequestManifest( + status=status, + reason_code=reason_code, + outcomes=( + FragmentOutcome( + key=key, + source=source, + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.REQUEST_ONLY, + status=outcome_status, + estimated_tokens=12, + admitted_tokens=8, + reason_code=outcome_reason, + ), + ), + budget_tokens=128, + budgeted_admitted_tokens=8, + non_budgeted_estimated_tokens=4, + ) + + +def _opaque_identifier(identifier: str) -> str: + identifier_bytes = identifier.encode(encoding="utf-8") + return f"id:{hashlib.sha256(identifier_bytes).hexdigest()[:16]}" + + +@pytest.fixture +def sent_text(monkeypatch: pytest.MonkeyPatch) -> list[str]: + captured: list[str] = [] + monkeypatch.setattr( + "pythinker_code.soul.slash.wire_send", + lambda part: captured.append(cast(TextPart, part).text), + ) + return captured + + +@pytest.mark.asyncio +async def test_prompt_manifest_before_first_assembly_reports_no_data( + runtime: Runtime, + tmp_path: Path, + sent_text: list[str], +) -> None: + soul = _make_soul(runtime, tmp_path) + + await _run_prompt_manifest(soul) + + assert sent_text == ["No request has been assembled in this session."] + + +@pytest.mark.parametrize( + ("status", "reason_code", "outcome_status", "outcome_reason"), + [ + (RequestStatus.SUCCEEDED, None, FragmentStatus.INCLUDED, None), + ( + RequestStatus.DEGRADED, + "optional_source_degraded", + FragmentStatus.DEGRADED, + "source_unavailable", + ), + ( + RequestStatus.FAILED, + "required_source_failed", + FragmentStatus.FAILED, + "permissions_state_unavailable", + ), + ], +) +@pytest.mark.asyncio +async def test_prompt_manifest_renders_safe_status_and_accounting( + runtime: Runtime, + tmp_path: Path, + sent_text: list[str], + status: RequestStatus, + reason_code: str | None, + outcome_status: FragmentStatus, + outcome_reason: str | None, +) -> None: + soul = _make_soul(runtime, tmp_path) + soul.latest_request_manifest = _manifest( + status, + reason_code=reason_code, + outcome_status=outcome_status, + outcome_reason=outcome_reason, + ) + + await _run_prompt_manifest(soul) + + rendered = sent_text[0] + assert status.value.upper() in rendered + assert f"{_opaque_identifier('permissions')} " in rendered + assert f"[{_opaque_identifier('permissions_state')}]" in rendered + assert "permissions [permissions_state]" not in rendered + assert "required" in rendered + assert "request_only" in rendered + assert outcome_status.value in rendered + assert "estimated=12" in rendered + assert "admitted=8" in rendered + assert "limit=128" in rendered + assert "budgeted_admitted=8" in rendered + assert "non_budgeted_estimated=4" in rendered + if reason_code is not None: + assert reason_code in rendered + if outcome_reason is not None: + assert outcome_reason in rendered + + +@pytest.mark.asyncio +async def test_prompt_manifest_redacts_untrusted_identifiers( + runtime: Runtime, + tmp_path: Path, + sent_text: list[str], +) -> None: + soul = _make_soul(runtime, tmp_path) + secret = "sk-proj-" + "a" * 24 + private_path = "/Users/alice/private/request.txt" + soul.latest_request_manifest = _manifest( + RequestStatus.FAILED, + reason_code=f"traceback:{private_path}", + source=private_path, + key=secret, + outcome_status=FragmentStatus.FAILED, + outcome_reason=f"credential={secret}", + ) + + await _run_prompt_manifest(soul) + + rendered = sent_text[0] + assert private_path not in rendered + assert secret not in rendered + assert "traceback" not in rendered + assert "" in rendered + + +@pytest.mark.asyncio +async def test_prompt_manifest_redacts_embedded_credentials_from_reason_codes( + runtime: Runtime, + tmp_path: Path, + sent_text: list[str], +) -> None: + embedded_secret = "prefix-sk-proj-" + "e" * 24 + "-suffix" + soul = _make_soul(runtime, tmp_path) + soul.latest_request_manifest = _manifest( + RequestStatus.FAILED, + reason_code=embedded_secret, + outcome_status=FragmentStatus.FAILED, + outcome_reason=embedded_secret, + ) + + await _run_prompt_manifest(soul) + + rendered = sent_text[0] + assert embedded_secret not in rendered + assert rendered.count("") == 2 + + +@pytest.mark.parametrize( + ("key", "source"), + [ + ( + "plugin-sk-proj-" + "a" * 24 + "-suffix", + "prefix-ghp_" + "b" * 24 + "-suffix", + ), + ("user_alice_request_20260711", "provider_controlled_plugin_alpha"), + ], +) +@pytest.mark.asyncio +async def test_prompt_manifest_uses_opaque_ids_for_provider_controlled_metadata( + runtime: Runtime, + tmp_path: Path, + sent_text: list[str], + key: str, + source: str, +) -> None: + soul = _make_soul(runtime, tmp_path) + soul.latest_request_manifest = _manifest( + RequestStatus.SUCCEEDED, + key=key, + source=source, + ) + + await _run_prompt_manifest(soul) + + rendered = sent_text[0] + assert key not in rendered + assert source not in rendered + assert _opaque_identifier(key) in rendered + assert _opaque_identifier(source) in rendered + + +def test_request_manifest_metrics_record_only_sanitized_aggregates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + records: list[tuple[float, dict[str, object]]] = [] + + class RecordingHistogram: + def record(self, amount: float, attributes: dict[str, object]) -> None: + records.append((amount, attributes)) + + monkeypatch.setattr( + metrics, + "request_assembly_duration_seconds", + RecordingHistogram(), + raising=False, + ) + record_manifest = getattr(metrics, "record_request_assembly", None) + assert record_manifest is not None + embedded_secret_source = "plugin-sk-proj-" + "c" * 24 + "-suffix" + user_derived_source = "user_alice_request_20260711" + suffixed_secret_source = "prefix-ghp_" + "d" * 24 + "-suffix" + manifest = RequestManifest( + status=RequestStatus.DEGRADED, + reason_code="optional_source_degraded", + outcomes=( + FragmentOutcome( + key="permissions", + source=embedded_secret_source, + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.REQUEST_ONLY, + status=FragmentStatus.INCLUDED, + estimated_tokens=12, + admitted_tokens=12, + reason_code=None, + ), + FragmentOutcome( + key="plan", + source=user_derived_source, + requirement=FragmentRequirement.BEST_EFFORT, + persistence=FragmentPersistence.HISTORY, + status=FragmentStatus.TRUNCATED, + estimated_tokens=20, + admitted_tokens=5, + reason_code="budget_truncated", + ), + FragmentOutcome( + key="git", + source=suffixed_secret_source, + requirement=FragmentRequirement.BEST_EFFORT, + persistence=FragmentPersistence.REQUEST_ONLY, + status=FragmentStatus.OMITTED_BUDGET, + estimated_tokens=18, + admitted_tokens=0, + reason_code="budget_exhausted", + ), + ), + budget_tokens=32, + budgeted_admitted_tokens=17, + non_budgeted_estimated_tokens=7, + ) + + record_manifest(manifest, duration_seconds=0.25) + + assert records == [ + ( + 0.25, + { + "source_ids": ( + _opaque_identifier(embedded_secret_source), + _opaque_identifier(user_derived_source), + _opaque_identifier(suffixed_secret_source), + ), + "required_count": 1, + "optional_count": 2, + "included_count": 1, + "omitted_count": 1, + "truncated_count": 1, + "degraded_count": 0, + "failed_count": 0, + "budget_limit": 32, + }, + ) + ] + + +@pytest.mark.asyncio +async def test_telemetry_failure_does_not_change_successful_assembly( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + soul = _make_soul(runtime, tmp_path) + telemetry_attempted = False + + def fail_telemetry(*_args: object, **_kwargs: object) -> None: + nonlocal telemetry_attempted + telemetry_attempted = True + raise RuntimeError("telemetry unavailable") + + monkeypatch.setattr(metrics, "record_request_assembly", fail_telemetry, raising=False) + + prepared = await soul._assemble_request("Inspect the repository") + + assert telemetry_attempted is True + assert prepared.assembled.manifest.status is RequestStatus.SUCCEEDED diff --git a/tests/core/test_provider_handoff_contract.py b/tests/core/test_provider_handoff_contract.py new file mode 100644 index 00000000..29f18cdc --- /dev/null +++ b/tests/core/test_provider_handoff_contract.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import dataclasses +from collections.abc import Sequence +from pathlib import Path + +import pytest +import pythinker_core +from pythinker_core import StepResult +from pythinker_core.message import Message, TextPart +from pythinker_core.tooling.empty import EmptyToolset + +import pythinker_code.soul.pythinkersoul as pythinkersoul_module +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider +from pythinker_code.soul.dynamic_injections.permissions_state import PermissionsInjectionProvider +from pythinker_code.soul.pythinkersoul import PythinkerSoul + +_AGENTS_REMINDER = ( + "\n" + "The merged `AGENTS.md` project instructions below are authoritative and already " + "assembled: every file from the project root down to the working directory, deeper " + "(more specific) files overriding shallower ones, each governing its own directory " + "and everything beneath it. Treat them with the same authority as your system " + "instructions.\n\n" + "`````````\n" + "Project rule.\n" + "`````````\n" + "" +) + + +class _StaticInjectionProvider(DynamicInjectionProvider): + def __init__(self, injection_type: str, content: str) -> None: + self._injection = DynamicInjection(type=injection_type, content=content) + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + return [self._injection] + + +@pytest.mark.asyncio +async def test_agent_step_has_one_characterized_provider_handoff( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + builtin_args = dataclasses.replace( + runtime.builtin_args, + PYTHINKER_AGENTS_MD="Project rule.", + ) + runtime = dataclasses.replace( + runtime, + builtin_args=builtin_args, + role="subagent", + subagent_id="contract-agent", + ) + toolset = EmptyToolset() + agent = Agent( + name="Contract Agent", + system_prompt="Static provider prompt.\n", + toolset=toolset, + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "history.jsonl") + soul = PythinkerSoul(agent, context=context) + await context.append_message( + [ + Message(role="user", content=[TextPart(text="Original question")]), + Message(role="assistant", content=[TextPart(text="Original answer")]), + Message(role="user", content=[TextPart(text="Latest request")]), + ] + ) + + soul.add_injection_provider(_StaticInjectionProvider("first", "First reminder")) + soul.add_injection_provider(_StaticInjectionProvider("second", "Second reminder")) + + captured: list[tuple[object, str, object, tuple[Message, ...]]] = [] + + async def capture(provider, system_prompt, provider_toolset, history, **_kwargs): + captured.append((provider, system_prompt, provider_toolset, tuple(history))) + return StepResult( + id="characterized-step", + message=Message(role="assistant", content=[TextPart(text="Done")]), + usage=None, + tool_calls=[], + _tool_result_futures={}, + ) + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + permission_content = (await PermissionsInjectionProvider().prepare_injections([], soul))[ + 0 + ].content + dynamic_reminders = ( + f"\n{permission_content}\n\n" + "\nFirst reminder\n\n" + "\nSecond reminder\n" + ) + + await soul._step() + + assert len(captured) == 1 + provider, system_prompt, provider_toolset, effective_history = captured[0] + assert runtime.llm is not None + assert provider is runtime.llm.chat_provider + assert system_prompt.encode("utf-8") == b"Static provider prompt.\n" + assert provider_toolset is toolset + assert effective_history == ( + Message( + role="user", + content=[ + TextPart(text=_AGENTS_REMINDER), + TextPart(text="Original question"), + ], + ), + Message(role="assistant", content=[TextPart(text="Original answer")]), + Message( + role="user", + content=[ + TextPart(text="Latest request"), + TextPart(text=dynamic_reminders), + ], + ), + ) + assert tuple(context.history) == ( + Message(role="user", content=[TextPart(text="Original question")]), + Message(role="assistant", content=[TextPart(text="Original answer")]), + Message(role="user", content=[TextPart(text="Latest request")]), + Message( + role="user", + content=[TextPart(text=dynamic_reminders)], + ), + Message(role="assistant", content=[TextPart(text="Done")]), + ) diff --git a/tests/core/test_pythinkersoul_ralph_loop.py b/tests/core/test_pythinkersoul_ralph_loop.py index 11c0c5d3..d9b3d84c 100644 --- a/tests/core/test_pythinkersoul_ralph_loop.py +++ b/tests/core/test_pythinkersoul_ralph_loop.py @@ -142,6 +142,7 @@ def _runtime_with_llm(runtime: Runtime, llm: LLM) -> Runtime: environment=runtime.environment, notifications=runtime.notifications, background_tasks=runtime.background_tasks, + skill_catalog=runtime.skill_catalog, skills=runtime.skills, oauth=runtime.oauth, additional_dirs=runtime.additional_dirs, diff --git a/tests/core/test_pythinkersoul_retry_recovery.py b/tests/core/test_pythinkersoul_retry_recovery.py index c506e3f0..07f3e9fa 100644 --- a/tests/core/test_pythinkersoul_retry_recovery.py +++ b/tests/core/test_pythinkersoul_retry_recovery.py @@ -356,6 +356,7 @@ def _runtime_with_llm(runtime: Runtime, llm: LLM) -> Runtime: environment=runtime.environment, notifications=runtime.notifications, background_tasks=runtime.background_tasks, + skill_catalog=runtime.skill_catalog, skills=runtime.skills, oauth=runtime.oauth, additional_dirs=runtime.additional_dirs, diff --git a/tests/core/test_pythinkersoul_skill_projection.py b/tests/core/test_pythinkersoul_skill_projection.py new file mode 100644 index 00000000..3b2906c7 --- /dev/null +++ b/tests/core/test_pythinkersoul_skill_projection.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest +import pythinker_core +from pythinker_core import StepResult +from pythinker_core.message import Message, TextPart +from pythinker_core.tooling.empty import EmptyToolset +from pythinker_host.path import HostPath + +import pythinker_code.soul.pythinkersoul as pythinkersoul_module +from pythinker_code.skill import Skill +from pythinker_code.skill.catalog import ( + SkillCatalog, + SkillProjectionOutcome, + SkillProjectionStatus, +) +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.message import system_reminder +from pythinker_code.soul.pythinkersoul import PythinkerSoul + + +def _skill(tmp_path: Path, name: str, description: str) -> Skill: + path = tmp_path / name / "SKILL.md" + return Skill( + name=name, + description=description, + dir=HostPath.unsafe_from_local_path(path.parent), + skill_md_file=HostPath.unsafe_from_local_path(path), + scope="user", + ) + + +async def _capture_one_step( + runtime: Runtime, + context: Context, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[tuple[Message, ...], PythinkerSoul, str]: + runtime = dataclasses.replace(runtime, role="subagent", subagent_id="projection-test") + soul = PythinkerSoul( + Agent( + name="projection", + system_prompt="static", + toolset=EmptyToolset(), + runtime=runtime, + ), + context=context, + ) + captured: list[tuple[Message, ...]] = [] + system_prompts: list[str] = [] + + async def capture(_provider, prompt, _toolset, history, **_kwargs): + system_prompts.append(prompt) + captured.append(tuple(history)) + return StepResult( + id="projection", + message=Message(role="assistant", content=[TextPart(text="done")]), + usage=None, + tool_calls=[], + _tool_result_futures={}, + ) + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + await soul._step() + return captured[0], soul, system_prompts[0] + + +@pytest.mark.asyncio +async def test_candidates_are_request_only_and_use_latest_real_user_task( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + skills = { + name: _skill(tmp_path, name, description) + for name, description in ( + ("first-explicit", "explicit workflow"), + ("second-explicit", "explicit workflow"), + ("old-active", "active workflow"), + ("new-active", "active workflow"), + ("spreadsheet", "build workbook tables"), + ) + } + runtime.skill_catalog = SkillCatalog(skills, ()) + runtime.skills = dict(runtime.skill_catalog.exhaustive_mapping()) + runtime.session.state.active_skills = ["old-active", "new-active"] + context = Context(file_backend=tmp_path / "history.jsonl") + task = "Build workbook tables with $first-explicit then /skill:second-explicit" + await context.append_message( + [ + Message(role="user", content=[TextPart(text=task)]), + Message( + role="user", + content=[system_reminder("Injected reminder that must not become the task")], + ), + ] + ) + + effective_history, soul, _ = await _capture_one_step(runtime, context, monkeypatch) + + rendered = effective_history[-1].extract_text("\n").split("Task-relevant skills:", 1)[1] + assert rendered.index("`first-explicit`") < rendered.index("`second-explicit`") + assert rendered.index("`second-explicit`") < rendered.index("`new-active`") + assert rendered.index("`new-active`") < rendered.index("`old-active`") + assert "`spreadsheet`" in rendered + assert not any( + "Task-relevant skills:" in message.extract_text("\n") for message in context.history + ) + assert soul.latest_skill_projection_outcome is not None + assert soul.latest_skill_projection_outcome.status is SkillProjectionStatus.READY + + +class _FailedProjectionCatalog(SkillCatalog): + def prompt_view(self, query: str, *, max_characters: int, explicit_names=(), active_names=()): + del query, max_characters, explicit_names, active_names + return SkillProjectionOutcome( + status=SkillProjectionStatus.FAILED, + view=None, + reason_code="safe_projection_failure", + ) + + +@pytest.mark.asyncio +async def test_projection_failure_is_recorded_without_unbounded_fallback( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + skills = { + f"skill-{index}": _skill(tmp_path, f"skill-{index}", "description") for index in range(100) + } + runtime.skill_catalog = _FailedProjectionCatalog(skills, ()) + runtime.skills = dict(runtime.skill_catalog.exhaustive_mapping()) + context = Context(file_backend=tmp_path / "history.jsonl") + await context.append_message(Message(role="user", content=[TextPart(text="current task")])) + + effective_history, soul, _ = await _capture_one_step(runtime, context, monkeypatch) + + rendered = "\n".join(message.extract_text("\n") for message in effective_history) + assert "skill-99" not in rendered + assert soul.latest_skill_projection_outcome is not None + assert soul.latest_skill_projection_outcome.status is SkillProjectionStatus.FAILED + assert soul.latest_skill_projection_outcome.reason_code == "safe_projection_failure" + + +@pytest.mark.asyncio +async def test_plugin_qualified_mentions_resolve_in_left_to_right_priority_order( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + skills = { + name: _skill(tmp_path, name, "ü" * 2_000) + for name in ("gh-fix-ci", "gh-address-comments", "implicit") + } + runtime.skill_catalog = SkillCatalog(skills, ()) + runtime.skills = runtime.skill_catalog.exhaustive_mapping() + context = Context(file_backend=tmp_path / "qualified.jsonl") + await context.append_message( + Message( + role="user", + content=[ + TextPart( + text=( + "Use $github:gh-fix-ci then /skill:github:gh-address-comments " + "for implicit work" + ) + ) + ], + ) + ) + + history, _, _ = await _capture_one_step(runtime, context, monkeypatch) + + candidates = history[-1].extract_text("\n").split("Task-relevant skills:", 1)[1] + assert candidates.index("`gh-fix-ci`") < candidates.index("`gh-address-comments`") + assert candidates.index("`gh-address-comments`") < candidates.index("`implicit`") + + +@pytest.mark.parametrize("cap", [7_999, 8_000, 8_001]) +@pytest.mark.asyncio +async def test_provider_visible_candidate_wrapper_respects_exact_cap_boundaries( + cap: int, + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + skills = { + f"技能-{index:03d}-{'界' * 80}": _skill( + tmp_path, f"技能-{index:03d}-{'界' * 80}", "説明🙂" * 300 + ) + for index in range(150) + } + runtime.skill_catalog = SkillCatalog(skills, ()) + runtime.skills = runtime.skill_catalog.exhaustive_mapping() + runtime.config.memory.injection_ceiling_tokens = 4_096 + monkeypatch.setattr(pythinkersoul_module, "SKILL_PROMPT_MAX_CHARACTERS", cap) + context = Context(file_backend=tmp_path / f"cap-{cap}.jsonl") + await context.append_message(Message(role="user", content=[TextPart(text="説明")])) + + history, _, _ = await _capture_one_step(runtime, context, monkeypatch) + + candidate_parts = [ + part.text + for message in history + for part in message.content + if isinstance(part, TextPart) and "Task-relevant skills:" in part.text + ] + assert len(candidate_parts) == 1 + assert len(candidate_parts[0]) <= cap + assert "omitted" in candidate_parts[0] + omitted = int(candidate_parts[0].split("; ", 1)[1].split(" omitted", 1)[0]) + assert omitted >= 10 + + +@pytest.mark.asyncio +async def test_different_tasks_keep_static_prompt_identical_but_change_candidates( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + skills = { + "spreadsheets": _skill(tmp_path, "spreadsheets", "workbook tables"), + "review-pr": _skill(tmp_path, "review-pr", "review pull requests"), + } + runtime.skill_catalog = SkillCatalog(skills, ()) + runtime.skills = runtime.skill_catalog.exhaustive_mapping() + first = Context(file_backend=tmp_path / "first.jsonl") + second = Context(file_backend=tmp_path / "second.jsonl") + await first.append_message( + Message(role="user", content=[TextPart(text="Create workbook tables")]) + ) + await second.append_message( + Message(role="user", content=[TextPart(text="Review this pull request")]) + ) + + first_history, _, first_prompt = await _capture_one_step(runtime, first, monkeypatch) + second_history, _, second_prompt = await _capture_one_step(runtime, second, monkeypatch) + + assert first_prompt == second_prompt + assert first_history[-1] != second_history[-1] diff --git a/tests/core/test_pythinkersoul_slash_commands.py b/tests/core/test_pythinkersoul_slash_commands.py index 14ab2fbf..9ed3d75f 100644 --- a/tests/core/test_pythinkersoul_slash_commands.py +++ b/tests/core/test_pythinkersoul_slash_commands.py @@ -4,14 +4,18 @@ from unittest.mock import AsyncMock import pytest +from pythinker_core.message import Message from pythinker_core.tooling.empty import EmptyToolset from pythinker_host.path import HostPath +import pythinker_code.soul.context as context_module import pythinker_code.soul.pythinkersoul as pythinkersoul_module +import pythinker_code.soul.slash as slash_module from pythinker_code.skill import Skill from pythinker_code.skill.flow import Flow, FlowEdge, FlowNode from pythinker_code.soul.agent import Agent, Runtime from pythinker_code.soul.context import Context +from pythinker_code.soul.dynamic_injection import DynamicInjectionProvider from pythinker_code.soul.pythinkersoul import PythinkerSoul from pythinker_code.utils.slashcmd import SlashCommand @@ -28,6 +32,17 @@ def _make_flow() -> Flow: return Flow(nodes=nodes, outgoing=outgoing, begin_id="BEGIN", end_id="END") +class _RearmProvider(DynamicInjectionProvider): + def __init__(self) -> None: + self.calls = 0 + + async def get_injections(self, history, soul): # noqa: ANN001 + return [] + + async def on_context_compacted(self) -> None: + self.calls += 1 + + def test_flow_skill_registers_skill_and_flow_commands(runtime: Runtime, tmp_path: Path) -> None: flow = _make_flow() skill_dir = tmp_path / "flow-skill" @@ -174,3 +189,70 @@ async def test_flow_slash_run_does_not_auto_generate_session_title( await soul.run("/flow:demo-flow") assert runtime.session.state.custom_title is None + + +@pytest.mark.asyncio +async def test_clear_slash_notifies_lifecycle_only_after_coherent_reset( + runtime: Runtime, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + agent = Agent( + name="Test Agent", + system_prompt="Current system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "history.jsonl") + soul = PythinkerSoul(agent, context=context) + await context.write_system_prompt("Current system prompt.") + before_bytes = context.file_backend.read_bytes() + notify = AsyncMock() + soul.notify_history_rebuilt = notify # type: ignore[method-assign] + context.replace_history = AsyncMock(side_effect=OSError("disk full")) # type: ignore[method-assign] + monkeypatch.setattr(slash_module, "wire_send", lambda _message: None) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + with pytest.raises(OSError, match="disk full"): + await soul.run("/clear") + + assert context.file_backend.read_bytes() == before_bytes + notify.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_clear_visible_durability_error_rearms_before_propagating( + runtime: Runtime, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + agent = Agent( + name="Test Agent", + system_prompt="Current system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + context = Context(file_backend=tmp_path / "history-durability.jsonl") + soul = PythinkerSoul(agent, context=context) + provider = _RearmProvider() + soul._injection_providers = [provider] # pyright: ignore[reportPrivateUsage] + await context.append_message(Message(role="user", content="old")) + lifecycle_generation = soul._request_lifecycle.history_generation # pyright: ignore[reportPrivateUsage] + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + monkeypatch.setattr(slash_module, "wire_send", lambda _message: None) + + def fail_directory_sync(_path: Path) -> bool: + raise OSError("fsync failed") + + monkeypatch.setattr( + context_module, + "_sync_parent_directory", + fail_directory_sync, + ) + + with pytest.raises( + context_module.ContextPersistenceError, + match="power-loss durability is uncertain", + ): + await soul.run("/clear") + + assert context.system_prompt == "Current system prompt." + assert context.history == [] + assert soul._request_lifecycle.history_generation == lifecycle_generation + 1 # pyright: ignore[reportPrivateUsage] + assert provider.calls == 1 diff --git a/tests/core/test_pythinkersoul_steer.py b/tests/core/test_pythinkersoul_steer.py index f8a924a6..8624b9ee 100644 --- a/tests/core/test_pythinkersoul_steer.py +++ b/tests/core/test_pythinkersoul_steer.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +from collections.abc import Sequence from pathlib import Path import pytest @@ -24,9 +25,10 @@ _current_deliberation_scope, ) from pythinker_code.soul.context import Context -from pythinker_code.soul.dynamic_injection import DynamicInjection +from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider from pythinker_code.soul.message import is_system_reminder_message from pythinker_code.soul.pythinkersoul import PythinkerSoul +from pythinker_code.soul.request_lifecycle import RequestLifecycle from pythinker_code.utils.aioqueue import QueueShutDown from pythinker_code.wire import Wire from pythinker_code.wire.types import ( @@ -39,6 +41,16 @@ ) +class _StaticInjectionProvider(DynamicInjectionProvider): + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + return [DynamicInjection(type="plan_mode", content="Internal reminder")] + + @pytest.fixture def approval() -> Approval: """Override global yolo=True fixture; steer tests don't need yolo.""" @@ -67,6 +79,7 @@ def _runtime_with_llm(runtime: Runtime, llm: LLM) -> Runtime: environment=runtime.environment, notifications=runtime.notifications, background_tasks=runtime.background_tasks, + skill_catalog=runtime.skill_catalog, skills=runtime.skills, oauth=runtime.oauth, additional_dirs=runtime.additional_dirs, @@ -371,33 +384,26 @@ async def fake_pythinker_core_step(chat_provider, system_prompt, toolset, histor _tool_result_futures={}, ) - async def fake_collect_injections() -> list[DynamicInjection]: - return [DynamicInjection(type="plan_mode", content="Internal reminder")] - - monkeypatch.setattr( - soul, - "_collect_injections", - fake_collect_injections, - ) + soul._injection_providers.append(_StaticInjectionProvider()) + soul._request_lifecycle = RequestLifecycle(soul._injection_providers) monkeypatch.setattr(pythinkersoul_module.pythinker_core, "step", fake_pythinker_core_step) monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _msg: None) outcome = await soul._step() assert outcome is not None - assert soul.context.history[-3:] == [ - Message(role="user", content=[TextPart(text="Follow user note")]), - Message( - role="user", - content=[TextPart(text="\nInternal reminder\n")], - ), - Message(role="assistant", content=[TextPart(text="done")]), - ] + assert soul.context.history[-3] == Message( + role="user", content=[TextPart(text="Follow user note")] + ) + assert _is_permissions_state_injection(soul.context.history[-2]) + assert "Internal reminder" in soul.context.history[-2].extract_text(" ") + assert soul.context.history[-1] == Message(role="assistant", content=[TextPart(text="done")]) assert captured_history[-1].role == "user" - assert captured_history[-1].content == [ - TextPart(text="Follow user note"), - TextPart(text="\nInternal reminder\n"), - ] + assert captured_history[-1].content[0] == TextPart(text="Follow user note") + injection = captured_history[-1].content[1] + assert isinstance(injection, TextPart) + assert "Permissions state:" in injection.text + assert "Internal reminder" in injection.text @pytest.mark.asyncio diff --git a/tests/core/test_pythinkersoul_stuck_loop.py b/tests/core/test_pythinkersoul_stuck_loop.py index 73898079..aaa729ab 100644 --- a/tests/core/test_pythinkersoul_stuck_loop.py +++ b/tests/core/test_pythinkersoul_stuck_loop.py @@ -23,7 +23,7 @@ from pythinker_code.llm import LLM from pythinker_code.soul import run_soul -from pythinker_code.soul.agent import Agent, BuiltinSystemPromptArgs, Runtime +from pythinker_code.soul.agent import Agent, Runtime from pythinker_code.soul.context import Context from pythinker_code.soul.pythinkersoul import PythinkerSoul, TurnStopReason from pythinker_code.soul.toolset import PythinkerToolset @@ -140,6 +140,7 @@ def _rebuild_runtime_with_llm(runtime: Runtime, llm: LLM) -> Runtime: environment=runtime.environment, notifications=runtime.notifications, background_tasks=runtime.background_tasks, + skill_catalog=runtime.skill_catalog, skills=runtime.skills, oauth=runtime.oauth, additional_dirs=runtime.additional_dirs, @@ -313,59 +314,6 @@ def test_user_message_with_hook_context() -> None: assert "should be ignored" not in blocked.extract_text(" ") -def test_with_agents_md_preamble_prepends_authoritative_reminder( - builtin_args: BuiltinSystemPromptArgs, -) -> None: - """The merged AGENTS.md is prepended as a leading user-role , ahead of - the conversation, WITHOUT mutating context history — assembled fresh each step so it - survives compaction (never persisted) and the injection budget (not a dynamic injection).""" - from pythinker_code.soul.message import is_system_reminder_message - from pythinker_code.soul.pythinkersoul import _with_agents_md_preamble - - history = [Message(role="user", content=[TextPart(text="hello")])] - result = _with_agents_md_preamble(history, builtin_args) - - # A leading reminder is prepended; the original history follows it, by identity. - assert len(result) == 2 - assert is_system_reminder_message(result[0]) - reminder_part = result[0].content[0] - assert isinstance(reminder_part, TextPart) - assert "Test agents content" in reminder_part.text - assert result[1] is history[0] - # The input list is never mutated (the preamble must not leak into context.history). - assert history == [Message(role="user", content=[TextPart(text="hello")])] - - -def test_with_agents_md_preamble_absent_returns_history_unchanged( - builtin_args: BuiltinSystemPromptArgs, -) -> None: - """No AGENTS.md → history passes through unchanged (no empty preamble is injected).""" - from dataclasses import replace - - from pythinker_code.soul.pythinkersoul import _with_agents_md_preamble - - empty = replace(builtin_args, PYTHINKER_AGENTS_MD="") - history = [Message(role="user", content=[TextPart(text="hi")])] - result = _with_agents_md_preamble(history, empty) - assert result == history - - -def test_with_agents_md_preamble_normalizes_to_lead_the_first_user_turn( - builtin_args: BuiltinSystemPromptArgs, -) -> None: - """After history normalization the AGENTS.md reminder leads the first user message — - a stable position-0 prefix (good for prompt-cache keying), not a stray extra turn.""" - from pythinker_code.soul.dynamic_injection import normalize_history - from pythinker_code.soul.pythinkersoul import _with_agents_md_preamble - - history = [Message(role="user", content=[TextPart(text="first prompt")])] - normalized = normalize_history(_with_agents_md_preamble(history, builtin_args)) - - assert len(normalized) == 1 - text = "".join(p.text for p in normalized[0].content if isinstance(p, TextPart)) - assert text.index("Test agents content") < text.index("first prompt") - - @pytest.mark.asyncio async def test_agents_md_reaches_llm_but_is_never_persisted( runtime: Runtime, tmp_path: Path diff --git a/tests/core/test_pythinkersoul_turn_balance.py b/tests/core/test_pythinkersoul_turn_balance.py index 930a11bb..a31724a6 100644 --- a/tests/core/test_pythinkersoul_turn_balance.py +++ b/tests/core/test_pythinkersoul_turn_balance.py @@ -15,7 +15,6 @@ from pythinker_code.soul.agent import Agent, Runtime from pythinker_code.soul.approval import Approval from pythinker_code.soul.context import Context -from pythinker_code.soul.dynamic_injection import DynamicInjection from pythinker_code.soul.pythinkersoul import PythinkerSoul, TurnOutcome from pythinker_code.wire.types import StepBegin, StepInterrupted, TextPart, TurnBegin, TurnEnd @@ -36,6 +35,16 @@ def _make_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: return PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) +class _EnteredFuture(asyncio.Future[ToolResult]): + def __init__(self) -> None: + super().__init__() + self.entered = asyncio.Event() + + def __await__(self): + self.entered.set() + return super().__await__() + + @pytest.mark.asyncio async def test_run_emits_turn_end_when_step_interrupts( runtime: Runtime, @@ -225,7 +234,7 @@ async def test_step_persists_assistant_message_when_tool_results_cancelled( id="call-cancel-1", function=ToolCall.FunctionBody(name="Noop", arguments="{}"), ) - pending_future: asyncio.Future[ToolResult] = asyncio.get_event_loop().create_future() + pending_future = _EnteredFuture() async def fake_pythinker_core_step(chat_provider, system_prompt, toolset, history, **kwargs): return StepResult( @@ -236,19 +245,11 @@ async def fake_pythinker_core_step(chat_provider, system_prompt, toolset, histor _tool_result_futures={"call-cancel-1": pending_future}, ) - async def fake_collect_injections() -> list[DynamicInjection]: - return [] - - monkeypatch.setattr(soul, "_collect_injections", fake_collect_injections) monkeypatch.setattr(pythinkersoul_module.pythinker_core, "step", fake_pythinker_core_step) monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _msg: None) - # Run _step in a task and cancel it while it is blocked in tool_results() step_task = asyncio.create_task(soul._step()) - # Yield enough times for the task to reach `await result.tool_results()` which - # then blocks on the pending_future. - for _ in range(10): - await asyncio.sleep(0) + await pending_future.entered.wait() step_task.cancel() with pytest.raises(asyncio.CancelledError): @@ -284,7 +285,7 @@ async def test_step_persists_markers_when_cancelled_twice( id="call-cancel-2", function=ToolCall.FunctionBody(name="Noop", arguments="{}"), ) - pending_future: asyncio.Future[ToolResult] = asyncio.get_event_loop().create_future() + pending_future = _EnteredFuture() async def fake_pythinker_core_step(chat_provider, system_prompt, toolset, history, **kwargs): return StepResult( @@ -295,9 +296,6 @@ async def fake_pythinker_core_step(chat_provider, system_prompt, toolset, histor _tool_result_futures={"call-cancel-2": pending_future}, ) - async def fake_collect_injections() -> list[DynamicInjection]: - return [] - real_grow = soul._grow_context write_started = asyncio.Event() @@ -307,14 +305,12 @@ async def slow_grow(result, results): await asyncio.sleep(0) await real_grow(result, results) - monkeypatch.setattr(soul, "_collect_injections", fake_collect_injections) monkeypatch.setattr(soul, "_grow_context", slow_grow) monkeypatch.setattr(pythinkersoul_module.pythinker_core, "step", fake_pythinker_core_step) monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _msg: None) step_task = asyncio.create_task(soul._step()) - for _ in range(10): - await asyncio.sleep(0) + await pending_future.entered.wait() step_task.cancel() await write_started.wait() step_task.cancel() # second interrupt lands mid marker-write diff --git a/tests/core/test_request_assembly.py b/tests/core/test_request_assembly.py new file mode 100644 index 00000000..1151fcf1 --- /dev/null +++ b/tests/core/test_request_assembly.py @@ -0,0 +1,693 @@ +from __future__ import annotations + +from collections.abc import Sequence + +import pytest +from pythinker_core.message import Message + +from pythinker_code.soul.request_assembly import ( + AGENTS_MD_SOURCE_POLICY, + FragmentBudgetClass, + FragmentPersistence, + FragmentRequirement, + FragmentStatus, + FragmentTruncation, + RequestAssembler, + RequestAssemblyError, + RequestAssemblyInput, + RequestFragment, + RequestSourceResult, + RequestStatus, + SourceApplicability, + SourceResultStatus, + TrustedSourcePolicy, +) + + +def _policy( + source: str, + key: str, + *, + requirement: FragmentRequirement = FragmentRequirement.BEST_EFFORT, + persistence: FragmentPersistence = FragmentPersistence.HISTORY, + priority: int = 100, + budget_class: FragmentBudgetClass = FragmentBudgetClass.BUDGETED, + truncation: FragmentTruncation = FragmentTruncation.FORBIDDEN, + applicability: SourceApplicability = SourceApplicability.ALWAYS, + failure_reason_codes: tuple[str, ...] = (), +) -> TrustedSourcePolicy: + return TrustedSourcePolicy( + source=source, + key=key, + requirement=requirement, + persistence=persistence, + priority=priority, + budget_class=budget_class, + truncation=truncation, + applicability=applicability, + failure_reason_codes=failure_reason_codes, + ) + + +def _provided(policy: TrustedSourcePolicy, content: str) -> RequestSourceResult: + return RequestSourceResult( + source=policy.source, + key=policy.key, + status=SourceResultStatus.PROVIDED, + fragment=RequestFragment( + key=policy.key, + content=content, + source=policy.source, + requirement=policy.requirement, + persistence=policy.persistence, + priority=policy.priority, + truncatable=policy.truncation is FragmentTruncation.ALLOWED, + ), + reason_code=None, + ) + + +def _failed(policy: TrustedSourcePolicy, reason_code: str) -> RequestSourceResult: + return RequestSourceResult( + source=policy.source, + key=policy.key, + status=SourceResultStatus.FAILED, + fragment=None, + reason_code=reason_code, + ) + + +def _not_applicable(policy: TrustedSourcePolicy) -> RequestSourceResult: + return RequestSourceResult( + source=policy.source, + key=policy.key, + status=SourceResultStatus.NOT_APPLICABLE, + fragment=None, + reason_code=None, + ) + + +def _request(*, budget_tokens: int, history: tuple[Message, ...] = ()) -> RequestAssemblyInput: + return RequestAssemblyInput( + system_prompt="stable system prompt", + persisted_history=history, + current_task="implement request assembly", + budget_tokens=budget_tokens, + ) + + +def _satisfied(policy: TrustedSourcePolicy, content: str, generation: int) -> RequestSourceResult: + provided = _provided(policy, content) + return RequestSourceResult( + source=provided.source, + key=provided.key, + status=SourceResultStatus.ALREADY_SATISFIED, + fragment=provided.fragment, + reason_code=None, + history_generation=generation, + ) + + +@pytest.mark.asyncio +async def test_already_satisfied_requires_matching_history_generation_proof() -> None: + permissions = _policy( + "permissions", + "state", + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.HISTORY, + ) + request = RequestAssemblyInput( + system_prompt="stable", + persisted_history=(Message(role="user", content="persisted"),), + current_task="task", + budget_tokens=100, + history_generation=7, + ) + + assembled = await RequestAssembler( + (permissions,), + (_satisfied(permissions, "current permissions", 7),), + ).assemble(request) + + assert assembled.history_appends == () + assert assembled.provider_history == request.persisted_history + + +@pytest.mark.parametrize( + ("persistence", "proof_generation"), + [ + (FragmentPersistence.REQUEST_ONLY, 7), + (FragmentPersistence.HISTORY, 6), + (FragmentPersistence.HISTORY, None), + ], +) +@pytest.mark.asyncio +async def test_already_satisfied_rejects_request_only_or_stale_proof( + persistence: FragmentPersistence, + proof_generation: int | None, +) -> None: + policy = _policy( + "permissions", + "state", + requirement=FragmentRequirement.REQUIRED, + persistence=persistence, + ) + provided = _provided(policy, "current permissions") + result = RequestSourceResult( + source=provided.source, + key=provided.key, + status=SourceResultStatus.ALREADY_SATISFIED, + fragment=provided.fragment, + reason_code=None, + history_generation=proof_generation, + ) + request = RequestAssemblyInput( + system_prompt="stable", + persisted_history=(), + current_task="task", + budget_tokens=100, + history_generation=7, + ) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((policy,), (result,)).assemble(request) + + assert caught.value.reason_code == "source_history_proof_invalid" + + +@pytest.mark.asyncio +async def test_required_fragments_are_reserved_before_higher_priority_optional_content() -> None: + required = _policy( + "permissions", + "permission", + requirement=FragmentRequirement.REQUIRED, + priority=1, + ) + optional = _policy("plan", "plan", priority=1_000) + + assembled = await RequestAssembler( + (required, optional), (_provided(optional, "o" * 20), _provided(required, "r" * 20)) + ).assemble(_request(budget_tokens=5)) + + assert [outcome.key for outcome in assembled.manifest.outcomes] == ["permission", "plan"] + assert [outcome.status for outcome in assembled.manifest.outcomes] == [ + FragmentStatus.INCLUDED, + FragmentStatus.OMITTED_BUDGET, + ] + assert assembled.manifest.budgeted_admitted_tokens == 5 + + +@pytest.mark.asyncio +async def test_agents_preamble_is_required_visible_and_non_budgeted() -> None: + plan = _policy("plan", "plan") + assembled = await RequestAssembler( + (AGENTS_MD_SOURCE_POLICY, plan), + (_provided(plan, "p" * 20), _provided(AGENTS_MD_SOURCE_POLICY, "a" * 40)), + ).assemble(_request(budget_tokens=5)) + + assert assembled.manifest.status is RequestStatus.SUCCEEDED + assert assembled.manifest.budgeted_admitted_tokens == 5 + assert assembled.manifest.non_budgeted_estimated_tokens == 10 + assert [outcome.admitted_tokens for outcome in assembled.manifest.outcomes] == [10, 5] + assert len(assembled.history_appends) == 1 + assert "p" * 20 in assembled.history_appends[0].extract_text("") + assert "a" * 40 in assembled.provider_history[-1].extract_text("") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("budget_tokens", [0, 5]) +async def test_non_budgeted_agents_and_exact_required_budget_succeed( + budget_tokens: int, +) -> None: + policies = [AGENTS_MD_SOURCE_POLICY] + source_results = [_provided(AGENTS_MD_SOURCE_POLICY, "a" * 20)] + if budget_tokens: + permission = _policy( + "permissions", + "permission", + requirement=FragmentRequirement.REQUIRED, + ) + policies.append(permission) + source_results.append(_provided(permission, "p" * 20)) + + assembled = await RequestAssembler(tuple(policies), tuple(source_results)).assemble( + _request(budget_tokens=budget_tokens) + ) + + assert assembled.manifest.status is RequestStatus.SUCCEEDED + assert assembled.manifest.budgeted_admitted_tokens == budget_tokens + assert assembled.manifest.non_budgeted_estimated_tokens == 5 + + +@pytest.mark.asyncio +async def test_negative_budget_raises_categorized_error() -> None: + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((), ()).assemble(_request(budget_tokens=-1)) + + assert caught.value.reason_code == "invalid_budget" + assert caught.value.manifest.status is RequestStatus.FAILED + assert caught.value.manifest.reason_code == caught.value.reason_code + + +@pytest.mark.asyncio +async def test_optional_source_failure_degrades_without_prompt_content() -> None: + plan = _policy("plan", "plan", failure_reason_codes=("plan_unavailable",)) + + assembled = await RequestAssembler((plan,), (_failed(plan, "plan_unavailable"),)).assemble( + _request(budget_tokens=10) + ) + + assert assembled.provider_history == () + assert assembled.history_appends == () + assert assembled.manifest.status is RequestStatus.DEGRADED + assert assembled.manifest.outcomes[0].status is FragmentStatus.DEGRADED + assert assembled.manifest.outcomes[0].reason_code == "plan_unavailable" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("budget_tokens", "expected_status", "expected_admitted", "expected_appends"), + [ + (0, FragmentStatus.OMITTED_BUDGET, 0, 0), + (1, FragmentStatus.INCLUDED, 1, 1), + ], +) +async def test_provided_empty_optional_content_uses_minimum_token_accounting( + budget_tokens: int, + expected_status: FragmentStatus, + expected_admitted: int, + expected_appends: int, +) -> None: + plan = _policy("plan", "plan") + + assembled = await RequestAssembler((plan,), (_provided(plan, ""),)).assemble( + _request(budget_tokens=budget_tokens) + ) + + outcome = assembled.manifest.outcomes[0] + assert outcome.status is expected_status + assert outcome.estimated_tokens == 1 + assert outcome.admitted_tokens == expected_admitted + assert assembled.manifest.budgeted_admitted_tokens == expected_admitted + assert len(assembled.history_appends) == expected_appends + + +@pytest.mark.asyncio +async def test_provided_empty_required_content_remains_fail_closed() -> None: + permission = _policy("permissions", "permission", requirement=FragmentRequirement.REQUIRED) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((permission,), (_provided(permission, ""),)).assemble( + _request(budget_tokens=1) + ) + + assert caught.value.reason_code == "required_source_invalid" + assert caught.value.manifest.status is RequestStatus.FAILED + assert caught.value.manifest.outcomes[0].status is FragmentStatus.FAILED + + +@pytest.mark.asyncio +async def test_required_source_failure_raises_with_matching_safe_reason() -> None: + permission = _policy( + "permissions", + "permission", + requirement=FragmentRequirement.REQUIRED, + failure_reason_codes=("permission_state_unavailable",), + ) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler( + (permission,), (_failed(permission, "permission_state_unavailable"),) + ).assemble(_request(budget_tokens=10)) + + assert caught.value.reason_code == "permission_state_unavailable" + assert caught.value.manifest.reason_code == caught.value.reason_code + assert caught.value.manifest.outcomes[0].status is FragmentStatus.FAILED + + +@pytest.mark.asyncio +async def test_multiple_required_fragments_include_when_estimated_sum_equals_budget() -> None: + permission = _policy("permissions", "permission", requirement=FragmentRequirement.REQUIRED) + defense = _policy("model_defense", "model_defense", requirement=FragmentRequirement.REQUIRED) + + assembled = await RequestAssembler( + (permission, defense), + (_provided(permission, "p" * 20), _provided(defense, "d" * 20)), + ).assemble(_request(budget_tokens=10)) + + assert [outcome.status for outcome in assembled.manifest.outcomes] == [ + FragmentStatus.INCLUDED, + FragmentStatus.INCLUDED, + ] + assert assembled.manifest.budgeted_admitted_tokens == 10 + + +@pytest.mark.asyncio +async def test_multiple_required_fragments_fail_when_budget_is_one_below_sum() -> None: + permission = _policy("permissions", "permission", requirement=FragmentRequirement.REQUIRED) + defense = _policy("model_defense", "model_defense", requirement=FragmentRequirement.REQUIRED) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler( + (permission, defense), + (_provided(permission, "p" * 20), _provided(defense, "d" * 20)), + ).assemble(_request(budget_tokens=9)) + + assert caught.value.reason_code == "required_content_exceeds_budget" + assert caught.value.manifest.status is RequestStatus.FAILED + assert all( + outcome.status is FragmentStatus.FAILED for outcome in caught.value.manifest.outcomes + ) + + +@pytest.mark.asyncio +async def test_not_applicable_source_is_successful_and_visible() -> None: + defense = _policy( + "model_defense", + "model_defense", + requirement=FragmentRequirement.REQUIRED, + applicability=SourceApplicability.MAY_BE_NOT_APPLICABLE, + ) + + assembled = await RequestAssembler((defense,), (_not_applicable(defense),)).assemble( + _request(budget_tokens=0) + ) + + assert assembled.manifest.status is RequestStatus.SUCCEEDED + assert assembled.manifest.outcomes[0].status is FragmentStatus.NOT_APPLICABLE + + +@pytest.mark.asyncio +async def test_required_source_cannot_claim_not_applicable_without_policy_permission() -> None: + permission = _policy("permissions", "permission", requirement=FragmentRequirement.REQUIRED) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((permission,), (_not_applicable(permission),)).assemble( + _request(budget_tokens=0) + ) + + assert caught.value.reason_code == "source_policy_mismatch" + assert caught.value.manifest.outcomes == () + + +@pytest.mark.asyncio +async def test_equal_priority_sources_follow_policy_order_for_all_result_permutations() -> None: + plan = _policy("plan", "plan", persistence=FragmentPersistence.REQUEST_ONLY, priority=100) + git = _policy("git", "git", persistence=FragmentPersistence.REQUEST_ONLY, priority=100) + plan_result = _provided(plan, "plan") + git_result = _provided(git, "git") + + first = await RequestAssembler((plan, git), (git_result, plan_result)).assemble( + _request(budget_tokens=10) + ) + second = await RequestAssembler((plan, git), (plan_result, git_result)).assemble( + _request(budget_tokens=10) + ) + + assert [outcome.key for outcome in first.manifest.outcomes] == ["plan", "git"] + assert first.provider_history == second.provider_history + + +@pytest.mark.asyncio +async def test_equal_priority_same_source_keys_follow_policy_key_rank() -> None: + alpha = _policy("reminders", "alpha", persistence=FragmentPersistence.REQUEST_ONLY) + beta = _policy("reminders", "beta", persistence=FragmentPersistence.REQUEST_ONLY) + alpha_result = _provided(alpha, "alpha") + beta_result = _provided(beta, "beta") + + first = await RequestAssembler((beta, alpha), (alpha_result, beta_result)).assemble( + _request(budget_tokens=10) + ) + second = await RequestAssembler((alpha, beta), (beta_result, alpha_result)).assemble( + _request(budget_tokens=10) + ) + + assert [outcome.key for outcome in first.manifest.outcomes] == ["alpha", "beta"] + assert first.provider_history == second.provider_history + + +@pytest.mark.asyncio +async def test_oversized_optional_fragment_is_truncated_only_when_policy_allows() -> None: + fixed = _policy("fixed", "fixed") + flexible = _policy("flexible", "flexible", truncation=FragmentTruncation.ALLOWED) + + assembled = await RequestAssembler( + (fixed, flexible), + (_provided(fixed, "f" * 40), _provided(flexible, "alpha\nbeta\ngamma" * 10)), + ).assemble(_request(budget_tokens=5)) + + outcomes = {outcome.key: outcome for outcome in assembled.manifest.outcomes} + assert outcomes["fixed"].status is FragmentStatus.OMITTED_BUDGET + assert outcomes["flexible"].status is FragmentStatus.TRUNCATED + assert outcomes["flexible"].admitted_tokens <= 5 + + +@pytest.mark.asyncio +async def test_unicode_truncation_keeps_complete_codepoints_and_line_boundary() -> None: + flexible = _policy("flexible", "unicode", truncation=FragmentTruncation.ALLOWED) + + assembled = await RequestAssembler( + (flexible,), (_provided(flexible, "alpha🙂beta\nsecond🙂line"),) + ).assemble(_request(budget_tokens=3)) + + rendered = assembled.provider_history[-1].extract_text("") + assert "alpha🙂beta\n…" in rendered + assert "second" not in rendered + assert "�" not in rendered + assert assembled.manifest.outcomes[0].admitted_tokens == 3 + + +@pytest.mark.asyncio +async def test_request_only_fragment_never_appears_in_history_appends() -> None: + persisted = (Message(role="assistant", content="previous"),) + skills = _policy("skills", "skills", persistence=FragmentPersistence.REQUEST_ONLY) + + assembled = await RequestAssembler((skills,), (_provided(skills, "candidate"),)).assemble( + _request(budget_tokens=10, history=persisted) + ) + + assert assembled.history_appends == () + assert assembled.provider_history[0] == persisted[0] + assert "candidate" in assembled.provider_history[-1].extract_text("") + + +@pytest.mark.asyncio +async def test_empty_source_registry_preserves_request_and_zero_accounting() -> None: + persisted = (Message(role="user", content="task"),) + + assembled = await RequestAssembler((), ()).assemble( + _request(budget_tokens=0, history=persisted) + ) + + assert assembled.system_prompt == "stable system prompt" + assert assembled.provider_history == persisted + assert assembled.history_appends == () + assert assembled.manifest.status is RequestStatus.SUCCEEDED + assert assembled.manifest.outcomes == () + assert assembled.manifest.budgeted_admitted_tokens == 0 + assert assembled.manifest.non_budgeted_estimated_tokens == 0 + + +@pytest.mark.asyncio +async def test_required_content_over_budget_raises_sanitized_categorized_error() -> None: + permission = _policy( + "permissions", + "permission", + requirement=FragmentRequirement.REQUIRED, + truncation=FragmentTruncation.ALLOWED, + ) + secret = "credential=do-not-leak /Users/private/project" + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((permission,), (_provided(permission, secret * 10),)).assemble( + _request(budget_tokens=1) + ) + + error = caught.value + assert error.reason_code == "required_content_exceeds_budget" + assert error.manifest.status is RequestStatus.FAILED + assert error.manifest.reason_code == error.reason_code + assert error.manifest.outcomes[0].status is FragmentStatus.FAILED + assert secret not in repr(error.manifest) + assert "/Users/private" not in str(error) + + +@pytest.mark.asyncio +async def test_failed_manifest_counts_observed_non_budgeted_agents() -> None: + permission = _policy("permissions", "permission", requirement=FragmentRequirement.REQUIRED) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler( + (AGENTS_MD_SOURCE_POLICY, permission), + ( + _provided(AGENTS_MD_SOURCE_POLICY, "a" * 40), + _provided(permission, "p" * 40), + ), + ).assemble(_request(budget_tokens=5)) + + assert caught.value.manifest.non_budgeted_estimated_tokens == 10 + agents_outcome = caught.value.manifest.outcomes[0] + assert agents_outcome.source == "agents_md" + assert agents_outcome.admitted_tokens == 10 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("spoofed_policy", "reason_code"), + [ + ( + _policy( + "agents_md", + "agents_preamble", + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.HISTORY, + budget_class=FragmentBudgetClass.NON_BUDGETED, + ), + "invalid_agents_policy", + ), + ( + _policy( + "agents_md", + "agents_preamble", + requirement=FragmentRequirement.BEST_EFFORT, + persistence=FragmentPersistence.REQUEST_ONLY, + budget_class=FragmentBudgetClass.NON_BUDGETED, + ), + "invalid_agents_policy", + ), + ], +) +async def test_agents_policy_rejects_history_and_requirement_spoofs( + spoofed_policy: TrustedSourcePolicy, reason_code: str +) -> None: + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((spoofed_policy,), (_provided(spoofed_policy, "content"),)).assemble( + _request(budget_tokens=10) + ) + + assert caught.value.reason_code == reason_code + assert caught.value.manifest.outcomes == () + + +@pytest.mark.asyncio +async def test_agents_fragment_cannot_override_request_only_persistence() -> None: + mismatched = RequestSourceResult( + source=AGENTS_MD_SOURCE_POLICY.source, + key=AGENTS_MD_SOURCE_POLICY.key, + status=SourceResultStatus.PROVIDED, + fragment=RequestFragment( + key=AGENTS_MD_SOURCE_POLICY.key, + content="agents content", + source=AGENTS_MD_SOURCE_POLICY.source, + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.HISTORY, + priority=AGENTS_MD_SOURCE_POLICY.priority, + truncatable=False, + ), + reason_code=None, + ) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((AGENTS_MD_SOURCE_POLICY,), (mismatched,)).assemble( + _request(budget_tokens=0) + ) + + assert caught.value.reason_code == "source_policy_mismatch" + assert caught.value.manifest.outcomes == () + + +@pytest.mark.asyncio +async def test_only_agents_policy_can_be_non_budgeted() -> None: + spoofed = _policy( + "plan", + "plan", + budget_class=FragmentBudgetClass.NON_BUDGETED, + ) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((spoofed,), (_provided(spoofed, "content"),)).assemble( + _request(budget_tokens=0) + ) + + assert caught.value.reason_code == "invalid_non_budgeted_policy" + assert caught.value.manifest.outcomes == () + + +@pytest.mark.asyncio +async def test_unknown_credential_shaped_source_is_rejected_before_manifest() -> None: + plan = _policy("plan", "plan") + credential_source = "api_token_abcd1234" + unknown = RequestSourceResult( + source=credential_source, + key="user_alice_request", + status=SourceResultStatus.NOT_APPLICABLE, + fragment=None, + reason_code=None, + ) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((plan,), (unknown,)).assemble(_request(budget_tokens=10)) + + assert caught.value.reason_code == "unknown_source_result" + assert caught.value.manifest.outcomes == () + assert credential_source not in repr(caught.value.manifest) + assert "user_alice_request" not in repr(caught.value.manifest) + + +@pytest.mark.asyncio +async def test_fragment_metadata_must_match_trusted_policy() -> None: + plan = _policy("plan", "plan", persistence=FragmentPersistence.REQUEST_ONLY) + mismatched = RequestSourceResult( + source="plan", + key="plan", + status=SourceResultStatus.PROVIDED, + fragment=RequestFragment( + key="plan", + content="content", + source="plan", + requirement=FragmentRequirement.REQUIRED, + persistence=FragmentPersistence.HISTORY, + priority=plan.priority, + truncatable=True, + ), + reason_code=None, + ) + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((plan,), (mismatched,)).assemble(_request(budget_tokens=10)) + + assert caught.value.reason_code == "source_policy_mismatch" + assert caught.value.manifest.outcomes == () + + +@pytest.mark.asyncio +async def test_normalization_failure_is_categorized_and_preserves_cause() -> None: + plan = _policy("plan", "plan") + failure = RuntimeError("normalizer details must not surface") + + def fail_normalization(_history: Sequence[Message]) -> list[Message]: + raise failure + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler( + (plan,), (_provided(plan, "content"),), history_normalizer=fail_normalization + ).assemble(_request(budget_tokens=10)) + + assert caught.value.reason_code == "history_normalization_failed" + assert caught.value.manifest.status is RequestStatus.FAILED + assert caught.value.manifest.reason_code == caught.value.reason_code + assert caught.value.__cause__ is failure + assert "normalizer details" not in str(caught.value) + + +@pytest.mark.asyncio +async def test_duplicate_policy_is_categorized_as_internal_invariant_failure() -> None: + plan = _policy("plan", "plan") + + with pytest.raises(RequestAssemblyError) as caught: + await RequestAssembler((plan, plan), (_provided(plan, "content"),)).assemble( + _request(budget_tokens=10) + ) + + assert caught.value.reason_code == "internal_invariant_violation" + assert caught.value.manifest.status is RequestStatus.FAILED diff --git a/tests/core/test_request_assembly_providers.py b/tests/core/test_request_assembly_providers.py new file mode 100644 index 00000000..ed7ea961 --- /dev/null +++ b/tests/core/test_request_assembly_providers.py @@ -0,0 +1,521 @@ +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +import pytest +import pythinker_core +from pythinker_core import StepResult +from pythinker_core.message import Message, TextPart +from pythinker_core.tooling.empty import EmptyToolset + +import pythinker_code.soul.pythinkersoul as pythinkersoul_module +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.dynamic_injection import ( + DynamicInjection, + DynamicInjectionProvider, + PreparedInjection, +) +from pythinker_code.soul.dynamic_injections.model_defense import ( + ModelDefenseFragment, + ModelDefenseInjectionProvider, +) +from pythinker_code.soul.dynamic_injections.permissions_state import PermissionsInjectionProvider +from pythinker_code.soul.pythinkersoul import PythinkerSoul +from pythinker_code.soul.request_assembly import ( + FragmentRequirement, + FragmentStatus, + RequestSourceError, + RequestStatus, +) +from pythinker_code.soul.request_lifecycle import RequestLifecycleError + + +class _StaticProvider(DynamicInjectionProvider): + def __init__(self, injection_type: str = "plugin_reminder") -> None: + self.injection_type = injection_type + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + return [DynamicInjection(type=self.injection_type, content="Plugin reminder")] + + +class _FailingProvider(DynamicInjectionProvider): + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + raise RuntimeError("provider unavailable") + + +class _CountingProvider(DynamicInjectionProvider): + def __init__(self, injection_type: str = "counted") -> None: + self.calls = 0 + self.injection_type = injection_type + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + self.calls += 1 + return [DynamicInjection(type=self.injection_type, content=f"Reminder {self.calls}")] + + +class _EmptyPermissionsProvider(PermissionsInjectionProvider): + async def prepare_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[PreparedInjection]: + del history, soul + return [] + + +class _RepeatedTypeProvider(DynamicInjectionProvider): + def __init__(self) -> None: + self.acknowledged: list[tuple[str, ...]] = [] + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + return [ + DynamicInjection(type="repeat", content="A"), + DynamicInjection(type="repeat", content="B" * 4_000), + DynamicInjection(type="repeat", content="C" * 4_000), + ] + + def acknowledge_injections(self, keys: Sequence[str]) -> None: + self.acknowledged.append(tuple(keys)) + super().acknowledge_injections(keys) + + +class _BrokenIdentityProvider(_StaticProvider): + def injection_identity(self, injection: DynamicInjection, index: int) -> str: + del injection, index + raise RuntimeError("identity adapter failed") + + +def _soul(runtime: Runtime, context: Context) -> PythinkerSoul: + return PythinkerSoul( + Agent( + name="request assembly", + system_prompt="stable prompt", + toolset=EmptyToolset(), + runtime=runtime, + ), + context=context, + ) + + +def _successful_step() -> StepResult: + return StepResult( + id="assembled-step", + message=Message(role="assistant", content=[TextPart(text="Done")]), + usage=None, + tool_calls=[], + _tool_result_futures={}, + ) + + +@pytest.mark.asyncio +async def test_model_defense_preparation_is_replayed_until_acknowledged( + runtime: Runtime, + tmp_path: Path, +) -> None: + soul = _soul(runtime, Context(file_backend=tmp_path / "model-defense.jsonl")) + provider = ModelDefenseInjectionProvider( + (ModelDefenseFragment(name="mock", patterns=("mock",), content="Defense"),) + ) + + first = await provider.prepare_injections([], soul) + second = await provider.prepare_injections([], soul) + + assert first == second + assert len(first) == 1 + provider.acknowledge_injections((first[0].identity,)) + replay = await provider.prepare_injections([], soul) + assert replay[0].identity == first[0].identity + + +@pytest.mark.asyncio +async def test_registered_providers_have_explicit_trusted_outcomes( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "registered.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + manifest = soul.latest_request_manifest + assert manifest is not None + assert manifest.status is RequestStatus.SUCCEEDED + requirements = {outcome.source: outcome.requirement for outcome in manifest.outcomes} + assert requirements["agents_md"] is FragmentRequirement.REQUIRED + assert requirements["permissions_state"] is FragmentRequirement.REQUIRED + assert requirements["model_defense"] is FragmentRequirement.REQUIRED + assert ( + next(outcome for outcome in manifest.outcomes if outcome.source == "model_defense").status + is FragmentStatus.NOT_APPLICABLE + ) + optional_sources = { + type(provider).__name__ + for provider in soul._injection_providers + if not isinstance(provider, (PermissionsInjectionProvider, ModelDefenseInjectionProvider)) + } + assert optional_sources <= { + outcome.source + for outcome in manifest.outcomes + if outcome.requirement is FragmentRequirement.BEST_EFFORT + } + + +@pytest.mark.asyncio +async def test_optional_provider_exception_degrades_and_still_invokes_model( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "degraded.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + soul._injection_providers = [ + ModelDefenseInjectionProvider(), + PermissionsInjectionProvider(), + _FailingProvider(), + ] + model_calls = 0 + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + nonlocal model_calls + model_calls += 1 + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + assert model_calls == 1 + manifest = soul.latest_request_manifest + assert manifest is not None + assert manifest.status is RequestStatus.DEGRADED + failure = next(outcome for outcome in manifest.outcomes if outcome.source == "FailingProvider") + assert failure.status is FragmentStatus.DEGRADED + assert failure.reason_code == "provider_failed" + + +@pytest.mark.asyncio +async def test_disabled_optional_bus_keeps_required_security_sources( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime.config.memory.injection_bus = False + context = Context(file_backend=tmp_path / "disabled.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + soul._injection_providers = [ + _StaticProvider(), + ModelDefenseInjectionProvider(), + PermissionsInjectionProvider(), + ] + captured_history: tuple[Message, ...] = () + + async def capture( + _provider: object, + _system_prompt: str, + _toolset: object, + history: Sequence[Message], + **_kwargs: object, + ) -> StepResult: + nonlocal captured_history + captured_history = tuple(history) + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + rendered = "\n".join(message.extract_text() for message in captured_history) + assert "Permissions state:" in rendered + assert "Plugin reminder" not in rendered + manifest = soul.latest_request_manifest + assert manifest is not None + assert ( + next(outcome for outcome in manifest.outcomes if outcome.source == "StaticProvider").status + is FragmentStatus.NOT_APPLICABLE + ) + + +@pytest.mark.asyncio +async def test_empty_permissions_fail_closed_before_optional_provider_or_model( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "empty-permissions.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + optional = _CountingProvider() + soul._injection_providers = [ + optional, + ModelDefenseInjectionProvider(), + _EmptyPermissionsProvider(), + ] + model_calls = 0 + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + nonlocal model_calls + model_calls += 1 + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + with pytest.raises(RequestSourceError, match="permissions_state_invalid"): + await soul._step() + + assert optional.calls == 0 + assert model_calls == 0 + assert len(context.history) == 1 + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.FAILED + assert soul.latest_request_manifest.reason_code == "permissions_state_invalid" + + +@pytest.mark.asyncio +async def test_required_sources_run_in_security_order_before_legacy_optional_order( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "provider-order.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + observed: list[str] = [] + + class OrderedPermissions(PermissionsInjectionProvider): + async def prepare_injections(self, history, soul): + observed.append("permissions") + return await super().prepare_injections(history, soul) + + class OrderedDefense(ModelDefenseInjectionProvider): + async def prepare_injections(self, history, soul): + observed.append("model_defense") + return await super().prepare_injections(history, soul) + + class OrderedOptional(_StaticProvider): + def __init__(self, name: str) -> None: + super().__init__(name) + self.name = name + + async def get_injections(self, history, soul): + observed.append(self.name) + return await super().get_injections(history, soul) + + soul._injection_providers = [ + OrderedOptional("first"), + OrderedDefense(), + OrderedOptional("second"), + OrderedPermissions(), + ] + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + assert observed == ["permissions", "model_defense", "first", "second"] + + +@pytest.mark.asyncio +async def test_duplicate_provider_classes_receive_collision_free_registration_sources( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "duplicate-providers.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider("first"), _StaticProvider("second")] + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + assert soul.latest_request_manifest is not None + sources = [ + outcome.source + for outcome in soul.latest_request_manifest.outcomes + if outcome.key in {"first", "second"} + ] + assert len(sources) == 2 + assert len(set(sources)) == 2 + assert any( + outcome.source == "permissions_state" + and outcome.requirement is FragmentRequirement.REQUIRED + for outcome in soul.latest_request_manifest.outcomes + ) + + +@pytest.mark.asyncio +async def test_repeated_types_acknowledge_only_exact_admitted_identity( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime.config.memory.injection_ceiling_tokens = 200 + context = Context(file_backend=tmp_path / "exact-ack.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + provider = _RepeatedTypeProvider() + soul._injection_providers = [provider] + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + assert provider.acknowledged == [("repeat", "repeat:0001")] + persisted = "\n".join(message.extract_text() for message in context.history) + assert "A" in persisted + assert "C" * 4_000 not in persisted + + runtime.config.memory.injection_ceiling_tokens = 2_000 + await soul._step() + + assert provider.acknowledged == [ + ("repeat", "repeat:0001"), + ("repeat:0002",), + ] + persisted = "\n".join(message.extract_text() for message in context.history) + assert "C" * 4_000 in persisted + + +@pytest.mark.asyncio +async def test_multiple_admitted_identities_are_acknowledged_once_as_one_batch( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime.config.memory.injection_ceiling_tokens = 2_000 + context = Context(file_backend=tmp_path / "batch-ack.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + provider = _RepeatedTypeProvider() + soul._injection_providers = [provider] + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + assert provider.acknowledged == [("repeat", "repeat:0001", "repeat:0002")] + + +@pytest.mark.asyncio +async def test_optional_identity_adapter_failure_replaces_prior_manifest_with_degraded( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "adapter-failure.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + soul._injection_providers = [_StaticProvider()] + await soul._step() + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.SUCCEEDED + + soul._injection_providers = [_BrokenIdentityProvider()] + await soul._step() + + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.DEGRADED + failure = next( + outcome + for outcome in soul.latest_request_manifest.outcomes + if outcome.source == "BrokenIdentityProvider" + ) + assert failure.reason_code == "provider_failed" + + +@pytest.mark.parametrize( + "duplicates", + [ + [PermissionsInjectionProvider(), PermissionsInjectionProvider()], + [ModelDefenseInjectionProvider(), ModelDefenseInjectionProvider()], + ], +) +@pytest.mark.asyncio +async def test_ambiguous_required_provider_roles_fail_before_optional_or_model( + duplicates: list[DynamicInjectionProvider], + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "ambiguous-required.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + optional = _CountingProvider() + soul._injection_providers = [*duplicates, optional] + model_calls = 0 + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + nonlocal model_calls + model_calls += 1 + return _successful_step() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + with pytest.raises(RequestLifecycleError, match="ambiguous_required_provider"): + await soul._step() + + assert optional.calls == 0 + assert model_calls == 0 + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.FAILED + assert soul.latest_request_manifest.reason_code == "ambiguous_required_provider" diff --git a/tests/core/test_request_assembly_soul.py b/tests/core/test_request_assembly_soul.py new file mode 100644 index 00000000..b8766287 --- /dev/null +++ b/tests/core/test_request_assembly_soul.py @@ -0,0 +1,713 @@ +from __future__ import annotations + +import asyncio +import inspect +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import cast + +import pytest +import pythinker_core +from pythinker_core import StepResult +from pythinker_core.message import Message, TextPart +from pythinker_core.tooling.empty import EmptyToolset +from pythinker_host.path import HostPath + +import pythinker_code.soul.pythinkersoul as pythinkersoul_module +from pythinker_code.skill import Skill, SkillCatalog +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.btw import execute_side_question +from pythinker_code.soul.context import Context +from pythinker_code.soul.dynamic_injection import ( + DynamicInjection, + DynamicInjectionProvider, + PreparedInjection, +) +from pythinker_code.soul.dynamic_injections.model_defense import ( + ModelDefenseFragment, + ModelDefenseInjectionProvider, +) +from pythinker_code.soul.dynamic_injections.permissions_state import PermissionsInjectionProvider +from pythinker_code.soul.pythinkersoul import PythinkerSoul +from pythinker_code.soul.request_assembly import RequestSourceError, RequestStatus +from pythinker_code.soul.request_lifecycle import RequestLifecycleError +from pythinker_code.soul.slash import clear as clear_context + + +class _StaticProvider(DynamicInjectionProvider): + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + return [DynamicInjection(type="stable_plugin", content="Stable plugin reminder")] + + +class _FailingPermissionsProvider(PermissionsInjectionProvider): + async def prepare_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[PreparedInjection]: + del history, soul + raise RuntimeError("permission state unavailable") + + +class _BarrierProvider(DynamicInjectionProvider): + def __init__(self) -> None: + self.calls = 0 + self.entered = asyncio.Event() + self.release = asyncio.Event() + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + del history, soul + self.calls += 1 + self.entered.set() + await self.release.wait() + return [DynamicInjection(type="barrier", content="Concurrent reminder")] + + +class _FailingAckProvider(_StaticProvider): + def _on_injections_acknowledged(self, injections: Sequence[DynamicInjection]) -> None: + del injections + raise RuntimeError("ack failed") + + +def _soul(runtime: Runtime, context: Context) -> PythinkerSoul: + return PythinkerSoul( + Agent( + name="request assembly", + system_prompt="static provider prompt\n", + toolset=EmptyToolset(), + runtime=runtime, + ), + context=context, + ) + + +def _step_result(text: str = "Done") -> StepResult: + return StepResult( + id="assembled-step", + message=Message(role="assistant", content=[TextPart(text=text)]), + usage=None, + tool_calls=[], + _tool_result_futures={}, + ) + + +@pytest.mark.asyncio +async def test_required_provider_failure_skips_model_and_records_failed_manifest( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "required-failure.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + soul._injection_providers = [ + ModelDefenseInjectionProvider(), + _FailingPermissionsProvider(), + ] + model_calls = 0 + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + nonlocal model_calls + model_calls += 1 + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + with pytest.raises(RequestSourceError, match="permissions_state_unavailable"): + await soul._step() + + assert model_calls == 0 + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.FAILED + assert soul.latest_request_manifest.reason_code == "permissions_state_unavailable" + + +@pytest.mark.asyncio +async def test_history_handoff_is_byte_equivalent_and_request_only_sources_are_not_persisted( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + skill = Skill( + name="deploy-check", + description="deployment validation", + dir=HostPath.unsafe_from_local_path(tmp_path / "deploy-check"), + skill_md_file=HostPath.unsafe_from_local_path(tmp_path / "deploy-check" / "SKILL.md"), + scope="project", + ) + runtime.skill_catalog = SkillCatalog({skill.name: skill}, ()) + runtime.skills = runtime.skill_catalog.exhaustive_mapping() + context = Context(file_backend=tmp_path / "equivalent.jsonl") + await context.append_message( + [ + Message(role="user", content="Original question"), + Message(role="assistant", content="Original answer"), + Message(role="user", content="Validate this deployment"), + ] + ) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider()] + captured_history: list[Message] = [] + + async def capture( + _provider: object, + _system_prompt: str, + _toolset: object, + history: Sequence[Message], + **_kwargs: object, + ) -> StepResult: + captured_history.extend(history) + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + + assert len(captured_history) == 3 + assert captured_history[0].role == "user" + assert "Test agents content" in captured_history[0].extract_text() + assert "Original question" in captured_history[0].extract_text() + assert captured_history[1] == Message(role="assistant", content="Original answer") + final_text = captured_history[-1].extract_text("\n") + assert "Validate this deployment" in final_text + assert "Stable plugin reminder" in final_text + assert "Task-relevant skills:" in final_text + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + assert "Task-relevant skills:" not in persisted + assert "Test agents content" not in persisted + + +@pytest.mark.asyncio +async def test_persistence_failure_skips_model_and_retries_same_identity( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context_file = tmp_path / "persistence.jsonl" + context = Context(file_backend=context_file) + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider()] + model_calls = 0 + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + nonlocal model_calls + model_calls += 1 + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + append_message = context.append_message + persistence_attempts = 0 + + async def fail_once(message: Message | Sequence[Message]) -> None: + nonlocal persistence_attempts + persistence_attempts += 1 + if persistence_attempts == 1: + raise PermissionError("context unavailable") + await append_message(message) + + monkeypatch.setattr(context, "append_message", fail_once) + + with pytest.raises(RequestLifecycleError, match="context_persistence_failed"): + await soul._step() + + assert model_calls == 0 + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.FAILED + assert soul.latest_request_manifest.reason_code == "context_persistence_failed" + + await soul._step() + + assert model_calls == 1 + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + + +@pytest.mark.asyncio +async def test_stable_provider_key_prevents_duplicate_history_on_later_steps( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "dedupe.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider()] + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + await soul._step() + + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + + +@pytest.mark.asyncio +async def test_model_failure_after_persistence_keeps_committed_reminder( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime.config.loop_control.max_retries_per_step = 1 + context = Context(file_backend=tmp_path / "provider-failure.jsonl") + await context.append_message(Message(role="user", content="Do the task")) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider()] + + async def fail(*_args: object, **_kwargs: object) -> StepResult: + raise RuntimeError("model unavailable") + + monkeypatch.setattr(pythinker_core, "step", fail) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + with pytest.raises(RuntimeError, match="model unavailable"): + await soul._step() + + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + + +@pytest.mark.asyncio +async def test_default_btw_uses_request_assembler_without_persisting_side_question( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "btw.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider()] + captured_history: list[Message] = [] + + async def capture( + _provider: object, + _system_prompt: str, + _toolset: object, + history: Sequence[Message], + **kwargs: object, + ) -> StepResult: + captured_history.extend(history) + on_message_part = cast(Callable[[TextPart], None], kwargs["on_message_part"]) + on_message_part(TextPart(text="Side answer")) + return _step_result("Side answer") + + monkeypatch.setattr("pythinker_code.soul.btw.pythinker_core.step", capture) + + response, error = await execute_side_question(soul, "What changed?") + + assert response == "Side answer" + assert error is None + rendered = "\n".join(message.extract_text() for message in captured_history) + assert "Test agents content" in rendered + assert "Stable plugin reminder" in rendered + assert "What changed?" in rendered + persisted = "\n".join(message.extract_text() for message in context.history) + assert "Stable plugin reminder" not in persisted + assert "What changed?" not in persisted + + +@pytest.mark.asyncio +async def test_concurrent_main_and_btw_share_one_stateful_preparation( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "concurrent-prepare.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + provider = _BarrierProvider() + soul._injection_providers = [provider] + + async def capture(*_args: object, **kwargs: object) -> StepResult: + if "on_message_part" in kwargs: + on_message_part = cast(Callable[[TextPart], None], kwargs["on_message_part"]) + on_message_part(TextPart(text="Side answer")) + return _step_result("Side answer") + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr("pythinker_code.soul.btw.pythinker_core.step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + main_task = asyncio.create_task(soul._step()) + await provider.entered.wait() + btw_task = asyncio.create_task(execute_side_question(soul, "What changed?")) + await asyncio.sleep(0) + provider.release.set() + await asyncio.gather(main_task, btw_task) + + assert provider.calls == 1 + + +@pytest.mark.asyncio +async def test_cancelled_preparer_releases_lock_and_next_request_retries( + runtime: Runtime, + tmp_path: Path, +) -> None: + context = Context(file_backend=tmp_path / "cancelled-prepare.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + provider = _BarrierProvider() + soul._injection_providers = [provider] + + first = asyncio.create_task(soul.assemble_side_request("one", "side")) + await provider.entered.wait() + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + + provider.release.set() + assembled = await asyncio.wait_for(soul.assemble_side_request("two", "side"), timeout=1) + + assert provider.calls == 2 + assert "Concurrent reminder" in "\n".join( + message.extract_text() for message in assembled.provider_history + ) + + +@pytest.mark.asyncio +async def test_revert_rearms_both_required_security_sources( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "security-generation.jsonl") + await context.append_message(Message(role="user", content="Main task")) + await context.checkpoint(add_user_message=False) + soul = _soul(runtime, context) + defense = ModelDefenseInjectionProvider( + (ModelDefenseFragment(name="mock", patterns=("mock",), content="Defense"),) + ) + permissions = PermissionsInjectionProvider() + soul._injection_providers = [defense, permissions] + captured: list[str] = [] + + async def capture( + _provider: object, + _system_prompt: str, + _toolset: object, + history: Sequence[Message], + **_kwargs: object, + ) -> StepResult: + captured.append("\n".join(message.extract_text() for message in history)) + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + await soul._revert_context_to(0) + await context.append_message(Message(role="user", content="Retry task")) + await soul._step() + + assert all("Permissions state:" in item for item in captured) + assert all("Defense" in item for item in captured) + + +@pytest.mark.asyncio +async def test_compaction_rebuild_rearms_both_required_security_sources( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "security-compaction.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + soul._injection_providers = [ + ModelDefenseInjectionProvider( + (ModelDefenseFragment(name="mock", patterns=("mock",), content="Defense"),) + ), + PermissionsInjectionProvider(), + ] + captured: list[str] = [] + + async def capture( + _provider: object, + _system_prompt: str, + _toolset: object, + history: Sequence[Message], + **_kwargs: object, + ) -> StepResult: + captured.append("\n".join(message.extract_text() for message in history)) + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + await context.clear() + await context.append_message(Message(role="user", content="Compacted task")) + await soul.notify_history_rebuilt() + await soul._step() + + assert len(captured) == 2 + assert all("Permissions state:" in item for item in captured) + assert all("Defense" in item for item in captured) + rebuilt = "\n".join(message.extract_text() for message in context.history) + assert rebuilt.count("Permissions state:") == 1 + assert rebuilt.count("Defense") == 1 + + +@pytest.mark.asyncio +async def test_cancellation_during_commit_finishes_commit_and_dedupes_retry( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "cancel-commit.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider()] + entered = asyncio.Event() + release = asyncio.Event() + real_append = context.append_message + + async def blocked_append(message: Message | Sequence[Message]) -> None: + entered.set() + await release.wait() + await real_append(message) + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _step_result() + + monkeypatch.setattr(context, "append_message", blocked_append) + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + step = asyncio.create_task(soul._step()) + await entered.wait() + step.cancel() + await asyncio.sleep(0) + step.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await step + + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.FAILED + assert soul.latest_request_manifest.reason_code == "context_persistence_cancelled" + monkeypatch.setattr(context, "append_message", real_append) + await soul._step() + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + + +@pytest.mark.asyncio +async def test_clear_rearms_required_security_sources_for_next_turn( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "clear-generation.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + soul._injection_providers = [ + PermissionsInjectionProvider(), + ModelDefenseInjectionProvider( + (ModelDefenseFragment(name="mock", patterns=("mock",), content="Defense"),) + ), + ] + captured: list[str] = [] + + async def capture( + _provider: object, + _system_prompt: str, + _toolset: object, + history: Sequence[Message], + **_kwargs: object, + ) -> StepResult: + captured.append("\n".join(message.extract_text() for message in history)) + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + monkeypatch.setattr("pythinker_code.soul.slash.wire_send", lambda _message: None) + + await soul._step() + await clear_context(soul, "") # type: ignore[reportGeneralTypeIssues] + await context.append_message(Message(role="user", content="New task")) + await soul._step() + + assert len(captured) == 2 + assert all("Permissions state:" in request for request in captured) + assert all("Defense" in request for request in captured) + rebuilt = "\n".join(message.extract_text() for message in context.history) + assert rebuilt.count("Permissions state:") == 1 + assert rebuilt.count("Defense") == 1 + + +@pytest.mark.asyncio +async def test_history_rebuild_discards_obsolete_committed_identity_generations( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "bounded-generations.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + assert soul._request_lifecycle.committed_identity_count > 0 + for _ in range(20): + soul._request_lifecycle.context_rebuilt() + assert soul._request_lifecycle.committed_identity_count == 0 + + +@pytest.mark.asyncio +async def test_ack_failure_replaces_manifest_and_retry_does_not_duplicate( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "ack-failure.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + soul._injection_providers = [_FailingAckProvider()] + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + with pytest.raises(RequestLifecycleError, match="provider_finalization_failed"): + await soul._step() + + assert soul.latest_request_manifest is not None + assert soul.latest_request_manifest.status is RequestStatus.FAILED + assert soul.latest_request_manifest.reason_code == "provider_finalization_failed" + await soul._step() + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + + +@pytest.mark.asyncio +async def test_permission_posture_rearm_commits_new_identity_once( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "permission-rearm.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + soul._injection_providers = [PermissionsInjectionProvider(), ModelDefenseInjectionProvider()] + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + await soul._step() + await soul._step() + assert soul.latest_request_manifest is not None + satisfied = next( + outcome + for outcome in soul.latest_request_manifest.outcomes + if outcome.source.startswith("permissions_state") + ) + assert satisfied.reason_code == "already_satisfied" + initial_yolo = runtime.approval.is_yolo() + runtime.approval.set_yolo(not initial_yolo) + soul.rearm_injection("permissions_state") + await soul._step() + await soul._step() + + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Permissions state:") == 2 + expected = "yolo off" if initial_yolo else "yolo on" + assert persisted.count(expected) == 1 + + +@pytest.mark.asyncio +async def test_cancellation_before_commit_keeps_history_and_provider_uncommitted( + runtime: Runtime, + tmp_path: Path, +) -> None: + context = Context(file_backend=tmp_path / "cancel-before-commit.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + provider = _BarrierProvider() + soul._injection_providers = [provider] + + step = asyncio.create_task(soul._step()) + await provider.entered.wait() + step.cancel() + with pytest.raises(asyncio.CancelledError): + await step + + assert len(context.history) == 1 + provider.release.set() + assembled = await soul.assemble_side_request("retry", "side") + assert "Concurrent reminder" in "\n".join( + message.extract_text() for message in assembled.provider_history + ) + + +@pytest.mark.asyncio +async def test_cancellation_after_finalize_keeps_one_committed_reminder( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = Context(file_backend=tmp_path / "cancel-after-finalize.jsonl") + await context.append_message(Message(role="user", content="Main task")) + soul = _soul(runtime, context) + soul._injection_providers = [_StaticProvider()] + provider_entered = asyncio.Event() + provider_release = asyncio.Event() + + async def blocked_model(*_args: object, **_kwargs: object) -> StepResult: + provider_entered.set() + await provider_release.wait() + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", blocked_model) + monkeypatch.setattr(pythinkersoul_module, "wire_send", lambda _message: None) + + step = asyncio.create_task(soul._step()) + await provider_entered.wait() + step.cancel() + with pytest.raises(asyncio.CancelledError): + await step + + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + + provider_release.set() + + async def capture(*_args: object, **_kwargs: object) -> StepResult: + return _step_result() + + monkeypatch.setattr(pythinker_core, "step", capture) + await soul._step() + persisted = "\n".join(message.extract_text() for message in context.history) + assert persisted.count("Stable plugin reminder") == 1 + + +def test_request_lifecycle_owns_dynamic_dedupe_state() -> None: + assert "_committed_injection_keys" not in inspect.getsource(PythinkerSoul) + assert "_dynamic_request_sources" not in PythinkerSoul.__dict__ diff --git a/tests/core/test_runtime_auto_state.py b/tests/core/test_runtime_auto_state.py index 55d09678..e1cedf48 100644 --- a/tests/core/test_runtime_auto_state.py +++ b/tests/core/test_runtime_auto_state.py @@ -10,6 +10,7 @@ import pythinker_code.soul.agent as agent_module from pythinker_code.auth.oauth import OAuthManager +from pythinker_code.skill import SkillCatalog from pythinker_code.soul.agent import Runtime from pythinker_code.wire.types import ToolCall @@ -26,10 +27,11 @@ def lightweight_runtime_create(monkeypatch: pytest.MonkeyPatch, environment) -> monkeypatch.setattr(agent_module, "list_directory", AsyncMock(return_value="")) monkeypatch.setattr(agent_module, "load_agents_md", AsyncMock(return_value=None)) monkeypatch.setattr(agent_module.Environment, "detect", AsyncMock(return_value=environment)) - monkeypatch.setattr(agent_module, "resolve_skills_roots", AsyncMock(return_value=[])) - monkeypatch.setattr(agent_module, "discover_skills_from_roots", AsyncMock(return_value=[])) - monkeypatch.setattr(agent_module, "index_skills", lambda _skills: {}) - monkeypatch.setattr(agent_module, "format_skills_for_prompt", lambda _skills: None) + monkeypatch.setattr( + agent_module, + "discover_runtime_skill_catalog", + AsyncMock(return_value=(SkillCatalog({}, ()), [])), + ) @pytest.mark.asyncio diff --git a/tests/core/test_skill_catalog.py b/tests/core/test_skill_catalog.py new file mode 100644 index 00000000..6a7471ac --- /dev/null +++ b/tests/core/test_skill_catalog.py @@ -0,0 +1,528 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +from pythinker_host.path import HostPath + +from pythinker_code.skill import ScopedSkillsRoot, SkillScope +from pythinker_code.skill import SkillCatalog as PublicSkillCatalog +from pythinker_code.skill.catalog import ( + SkillCatalog, + SkillDiagnosticCategory, + SkillProjectionStatus, + SkillRelevanceTier, + SkillSourceKind, + render_skill_prompt_view, +) + + +def _write_skill(root: Path, directory: str, *, name: str, description: str) -> None: + skill_dir = root / directory + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + f"---\nname: {name}\ndescription: {description}\n---\n", + encoding="utf-8", + ) + + +def _root(path: Path, scope: SkillScope = "user") -> ScopedSkillsRoot: + return ScopedSkillsRoot( + root=HostPath.unsafe_from_local_path(path), + scope=scope, + ) + + +def test_skill_catalog_is_exported_from_skill_package() -> None: + assert PublicSkillCatalog is SkillCatalog + + +@pytest.mark.asyncio +async def test_resolve_finds_exact_skill_name(tmp_path: Path) -> None: + skills_root = tmp_path / "skills" + _write_skill(skills_root, "deploy", name="deploy", description="Deploy applications") + + catalog = await SkillCatalog.discover([_root(skills_root)]) + + skill = catalog.resolve("deploy") + assert skill is not None + assert skill.description == "Deploy applications" + + +@pytest.mark.asyncio +async def test_resolve_is_case_insensitive(tmp_path: Path) -> None: + skills_root = tmp_path / "skills" + _write_skill(skills_root, "deploy", name="Deploy", description="Deploy applications") + + catalog = await SkillCatalog.discover([_root(skills_root)]) + + assert catalog.resolve("dEpLoY") is not None + + +@pytest.mark.asyncio +async def test_resolve_accepts_plugin_style_alias(tmp_path: Path) -> None: + skills_root = tmp_path / "skills" + _write_skill( + skills_root, + "designer-skill", + name="designer-skill", + description="Design interfaces", + ) + + catalog = await SkillCatalog.discover([_root(skills_root)]) + + assert catalog.resolve("designer-skill:designer-skill") is not None + + +@pytest.mark.asyncio +async def test_discover_keeps_first_root_winner_in_exhaustive_mapping(tmp_path: Path) -> None: + project_root = tmp_path / "project" + user_root = tmp_path / "user" + _write_skill(project_root, "shared", name="shared", description="Project version") + _write_skill(user_root, "shared", name="shared", description="User version") + + catalog = await SkillCatalog.discover( + [_root(project_root, "project"), _root(user_root, "user")] + ) + + assert catalog.exhaustive_mapping()["shared"].description == "Project version" + assert tuple(catalog.exhaustive_mapping()) == ("shared",) + + +@pytest.mark.asyncio +async def test_exhaustive_mapping_uses_legacy_global_skill_name_order(tmp_path: Path) -> None: + first_root = tmp_path / "first-root" + second_root = tmp_path / "second-root" + _write_skill(first_root, "zeta", name="zeta", description="Zeta workflow") + _write_skill(second_root, "alpha", name="alpha", description="Alpha workflow") + + forward = await SkillCatalog.discover([_root(first_root), _root(second_root)]) + reversed_catalog = await SkillCatalog.discover([_root(second_root), _root(first_root)]) + + expected_names = ("alpha", "zeta") + assert tuple(forward.exhaustive_mapping()) == expected_names + assert tuple(reversed_catalog.exhaustive_mapping()) == expected_names + assert tuple(skill.name for skill in forward.exhaustive_mapping().values()) == expected_names + assert ( + tuple(skill.name for skill in reversed_catalog.exhaustive_mapping().values()) + == expected_names + ) + + +@pytest.mark.asyncio +async def test_discovery_retains_unavailable_diagnostic_for_malformed_source( + tmp_path: Path, +) -> None: + skills_root = tmp_path / "skills" + malformed_dir = skills_root / "broken" + malformed_dir.mkdir(parents=True) + malformed_path = malformed_dir / "SKILL.md" + malformed_path.write_text( + "---\nname: broken\ntype: unsupported\nsecret: do-not-leak\n---\n", + encoding="utf-8", + ) + + catalog = await SkillCatalog.discover([_root(skills_root)]) + + assert catalog.resolve("broken") is None + assert len(catalog.diagnostics) == 1 + diagnostic = catalog.diagnostics[0] + assert diagnostic.name == "broken" + assert diagnostic.source_kind is SkillSourceKind.DIRECTORY + assert diagnostic.category is SkillDiagnosticCategory.UNAVAILABLE + assert diagnostic.source_id == "broken/SKILL.md" + assert str(tmp_path) not in diagnostic.source_id + assert not Path(diagnostic.source_id).is_absolute() + assert diagnostic.reason_code == "invalid_skill_metadata" + assert "do-not-leak" not in diagnostic.safe_reason + + +@pytest.mark.asyncio +async def test_unreadable_flat_source_retains_unavailable_diagnostic(tmp_path: Path) -> None: + skills_root = tmp_path / "skills" + skills_root.mkdir() + (skills_root / "unreadable.md").symlink_to(skills_root / "missing-target.md") + + catalog = await SkillCatalog.discover([_root(skills_root)]) + + assert catalog.resolve("unreadable") is None + assert len(catalog.diagnostics) == 1 + diagnostic = catalog.diagnostics[0] + assert diagnostic.source_id == "unreadable.md" + assert diagnostic.reason_code == "unreadable_skill_source" + + +@pytest.mark.asyncio +async def test_search_ranks_relevance_before_scope_with_stable_ties(tmp_path: Path) -> None: + skills_root = tmp_path / "skills" + _write_skill(skills_root, "exact", name="deploy", description="unrelated") + _write_skill(skills_root, "phrase", name="release-deploy", description="unrelated") + _write_skill(skills_root, "tokens", name="deploy-service", description="unrelated") + _write_skill(skills_root, "description", name="alpha", description="deploy service safely") + catalog = await SkillCatalog.discover([_root(skills_root, "builtin")]) + + assert [match.skill.name for match in catalog.search("deploy", limit=10)] == [ + "deploy", + "deploy-service", + "release-deploy", + "alpha", + ] + + +def test_search_tiers_cannot_be_crossed_by_more_than_one_hundred_token_overlaps( + tmp_path: Path, +) -> None: + tokens = [f"token{index}" for index in range(120)] + query = "target " + " ".join(tokens) + catalog = SkillCatalog( + { + "target-tool": _skill_for_catalog( + tmp_path / "target-tool" / "SKILL.md", "target-tool", "unrelated" + ), + "description-heavy": _skill_for_catalog( + tmp_path / "description-heavy" / "SKILL.md", + "description-heavy", + " ".join(tokens), + ), + }, + (), + ) + + matches = catalog.search(query, limit=2) + + assert [match.skill.name for match in matches] == ["target-tool", "description-heavy"] + assert matches[0].tier is SkillRelevanceTier.NAME_TOKEN + assert matches[1].tier is SkillRelevanceTier.DESCRIPTION_TOKEN + + +def test_name_phrase_requires_contiguous_normalized_tokens_not_substrings(tmp_path: Path) -> None: + catalog = SkillCatalog( + { + "art": _skill_for_catalog(tmp_path / "art" / "SKILL.md", "art", "drawing"), + "cartography": _skill_for_catalog( + tmp_path / "cartography" / "SKILL.md", "cartography", "maps" + ), + }, + (), + ) + + assert [match.skill.name for match in catalog.search("cartography", limit=10)] == [ + "cartography" + ] + + +@pytest.mark.parametrize( + "query", + ["$github:gh-fix-ci", "/skill:github:gh-fix-ci", "github:gh-fix-ci"], +) +def test_search_promotes_complete_qualified_alias_to_exact_name( + query: str, + tmp_path: Path, +) -> None: + catalog = SkillCatalog( + { + "gh-fix-ci": _skill_for_catalog( + tmp_path / "gh-fix-ci" / "SKILL.md", "gh-fix-ci", "GitHub CI" + ), + "github-helper": _skill_for_catalog( + tmp_path / "github-helper" / "SKILL.md", + "github-helper", + "Fix GitHub CI failures", + ), + }, + (), + ) + + matches = catalog.search(query, limit=2) + + assert [match.skill.name for match in matches] == ["gh-fix-ci", "github-helper"] + assert matches[0].tier is SkillRelevanceTier.EXACT_NAME + assert matches[0].reasons == ("exact_alias",) + + +@pytest.mark.parametrize("query", ["deploy", "release deploy safely"]) +def test_relevance_beats_scope_with_limit_one(query: str, tmp_path: Path) -> None: + catalog = SkillCatalog( + { + "deploy": _skill_for_catalog( + tmp_path / "builtin" / "SKILL.md", + "deploy", + "Ship releases", + "builtin", + ), + "project-helper": _skill_for_catalog( + tmp_path / "project" / "SKILL.md", + "project-helper", + "deploy release safely", + "project", + ), + }, + (), + ) + + match = catalog.search(query, limit=1)[0] + + assert match.skill.name == "deploy" + assert match.tier in (SkillRelevanceTier.EXACT_NAME, SkillRelevanceTier.NAME_PHRASE) + + +@pytest.mark.asyncio +async def test_search_is_stable_under_reversed_insertion_order(tmp_path: Path) -> None: + skills = { + "zeta": _skill_for_catalog(tmp_path / "zeta" / "SKILL.md", "zeta", "release deploy"), + "alpha": _skill_for_catalog(tmp_path / "alpha" / "SKILL.md", "alpha", "release deploy"), + } + + forward = SkillCatalog(skills, ()) + reversed_catalog = SkillCatalog(dict(reversed(tuple(skills.items()))), ()) + + assert forward.search("release deploy", limit=10) == reversed_catalog.search( + "release deploy", limit=10 + ) + + +def _skill_for_catalog(path: Path, name: str, description: str, scope: SkillScope = "user"): + from pythinker_code.skill import Skill + + return Skill( + name=name, + description=description, + dir=HostPath.unsafe_from_local_path(path.parent), + skill_md_file=HostPath.unsafe_from_local_path(path), + scope=scope, + ) + + +def test_prompt_view_prioritizes_explicit_then_newest_active_and_respects_hard_cap( + tmp_path: Path, +) -> None: + skills = { + name: _skill_for_catalog( + tmp_path / name / "SKILL.md", + name, + "x" * 4_000, + ) + for name in ("implicit", "active-old", "active-new", "explicit-one", "explicit-two") + } + catalog = SkillCatalog(skills, ()) + + outcome = catalog.prompt_view( + "work on implicit with $explicit-one then /skill:explicit-two", + max_characters=500, + explicit_names=("explicit-one", "explicit-two"), + active_names=("active-old", "active-new"), + ) + + assert outcome.view is not None + assert [match.skill.name for match in outcome.view.matches] == [ + "explicit-one", + "explicit-two", + "active-new", + "active-old", + "implicit", + ] + rendered = render_skill_prompt_view(outcome.view) + assert len(rendered) == outcome.view.rendered_characters + assert len(rendered) <= 500 + assert "x" * 100 not in rendered + assert str(tmp_path) not in rendered + assert outcome.view.omitted_count == 0 + + +def test_priority_overflow_is_degraded_and_never_truncates_names(tmp_path: Path) -> None: + names = tuple(f"priority-skill-{index:03d}" for index in range(100)) + catalog = SkillCatalog( + { + name: _skill_for_catalog(tmp_path / name / "SKILL.md", name, "description") + for name in names + }, + (), + ) + + outcome = catalog.prompt_view( + "task", + max_characters=500, + explicit_names=names, + ) + + assert outcome.status is SkillProjectionStatus.DEGRADED + assert outcome.reason_code == "priority_candidates_overflowed" + assert outcome.view is not None + assert outcome.view.overflowed_priority_count > 0 + rendered = render_skill_prompt_view(outcome.view) + assert len(rendered) <= 500 + assert all(match.skill.name in rendered for match in outcome.view.matches) + rendered_names = { + line.split("`", 2)[1] for line in rendered.splitlines() if line.startswith("- `") + } + assert rendered_names == {match.skill.name for match in outcome.view.matches} + + +def test_projection_budget_too_small_fails_without_oversized_view(tmp_path: Path) -> None: + catalog = SkillCatalog( + {"alpha": _skill_for_catalog(tmp_path / "alpha" / "SKILL.md", "alpha", "desc")}, + (), + ) + + outcome = catalog.prompt_view("alpha", max_characters=1) + + assert outcome.status is SkillProjectionStatus.FAILED + assert outcome.reason_code == "projection_budget_too_small" + assert outcome.view is None + + +def test_recall_fixture_and_warm_search_performance(tmp_path: Path) -> None: + fixture = json.loads( + (Path(__file__).parents[1] / "fixtures" / "skill_catalog_recall.json").read_text( + encoding="utf-8" + ) + ) + skills = { + f"fixture-{index:04d}": _skill_for_catalog( + tmp_path / f"fixture-{index:04d}" / "SKILL.md", + f"fixture-{index:04d}", + f"generic capability {index}", + ) + for index in range(1_000 - len(fixture["skills"])) + } + for item in fixture["skills"]: + skills[item["name"]] = _skill_for_catalog( + tmp_path / item["name"] / "SKILL.md", + item["name"], + item["description"], + item.get("scope", "user"), + ) + catalog = SkillCatalog(skills, ()) + + assert len(skills) == 1_000 + for item in fixture["cases"]: + matches = catalog.search(item["query"], limit=8) + assert [match.skill.name for match in matches[: len(item["expected"])]] == item["expected"] + assert matches[0].tier.name == item["winner_tier"] + + catalog.search("release deployment", limit=8) + search_result = catalog.search_with_metrics("release deployment", limit=8) + assert search_result.metrics.candidates_evaluated == len(skills) + assert search_result.metrics.match_work_units <= len(skills) * 250 + assert search_result.metrics.sort_items <= len(skills) + assert search_result.metrics.sort_comparison_bound <= len(skills) ** 2 + assert len(search_result.matches) <= 8 + + +@pytest.mark.asyncio +async def test_recall_fixture_shadowed_duplicate_uses_project_winner(tmp_path: Path) -> None: + fixture = json.loads( + (Path(__file__).parents[1] / "fixtures" / "skill_catalog_recall.json").read_text( + encoding="utf-8" + ) + ) + item = next(skill for skill in fixture["skills"] if "shadowed" in skill) + project_root = tmp_path / "project" + user_root = tmp_path / "user" + _write_skill( + project_root, + item["name"], + name=item["name"], + description=item["description"], + ) + _write_skill( + user_root, + item["name"], + name=item["name"], + description=item["shadowed"], + ) + + catalog = await SkillCatalog.discover( + [_root(project_root, "project"), _root(user_root, "user")] + ) + + winner = catalog.resolve(item["name"]) + assert winner is not None + assert winner.scope == "project" + assert winner.description == item["description"] + + +@pytest.mark.asyncio +async def test_concurrent_search_metrics_are_local_and_deterministic(tmp_path: Path) -> None: + catalog = SkillCatalog( + { + f"skill-{index}": _skill_for_catalog( + tmp_path / f"skill-{index}" / "SKILL.md", + f"skill-{index}", + "release workflow", + ) + for index in range(100) + }, + (), + ) + + results = await asyncio.gather( + *( + asyncio.to_thread(catalog.search_with_metrics, query, limit=8) + for query in ("release", "skill 9") * 10 + ) + ) + + assert len({result.metrics.candidates_evaluated for result in results}) == 1 + assert all(result.metrics.sort_items >= len(result.matches) for result in results) + assert not hasattr(catalog, "last_search_operation_count") + + +def test_catalogue_snapshot_stays_coherent_when_legacy_mapping_skill_is_mutated( + tmp_path: Path, +) -> None: + exposed = _skill_for_catalog(tmp_path / "deploy" / "SKILL.md", "deploy", "Deploy applications") + catalog = SkillCatalog({"deploy": exposed}, ()) + + catalog.exhaustive_mapping()["deploy"].name = "mutated-name" + catalog.exhaustive_mapping()["deploy"].description = "Mutated description" + + resolved = catalog.resolve("deploy") + assert resolved is not None + assert resolved.name == "deploy" + assert catalog.resolve("mutated-name") is None + assert [match.skill.name for match in catalog.search("deploy applications", limit=8)] == [ + "deploy" + ] + outcome = catalog.prompt_view("deploy applications", max_characters=8_000) + assert outcome.view is not None + assert "mutated" not in render_skill_prompt_view(outcome.view).casefold() + + +def test_public_resolve_and_search_results_cannot_mutate_catalogue_state(tmp_path: Path) -> None: + catalog = SkillCatalog( + { + "deploy": _skill_for_catalog( + tmp_path / "deploy" / "SKILL.md", "deploy", "Deploy applications" + ) + }, + (), + ) + + resolved = catalog.resolve("deploy") + searched = catalog.search("deploy applications", limit=1)[0].skill + assert resolved is not None + resolved.name = "resolve-mutated" + resolved.description = "resolve-mutated" + searched.name = "search-mutated" + searched.description = "search-mutated" + + repeated_resolve = catalog.resolve("deploy") + repeated_search = catalog.search("deploy applications", limit=1) + outcome = catalog.prompt_view("deploy applications", max_characters=8_000) + assert repeated_resolve is not None + assert repeated_resolve.name == "deploy" + assert repeated_search[0].skill.name == "deploy" + assert outcome.view is not None + rendered = render_skill_prompt_view(outcome.view) + assert "mutated" not in rendered + + +def test_root_and_subagent_runtime_share_catalogue_identity(runtime) -> None: + child = runtime.copy_for_subagent(agent_id="child", subagent_type="coder") + + assert child.skill_catalog is runtime.skill_catalog + assert runtime.skills is runtime.skill_catalog.exhaustive_mapping() + assert child.skills is runtime.skills diff --git a/tests/core/test_skills_prompt.py b/tests/core/test_skills_prompt.py index 136e9dfd..5eeeff73 100644 --- a/tests/core/test_skills_prompt.py +++ b/tests/core/test_skills_prompt.py @@ -129,6 +129,24 @@ def test_format_skills_for_prompt_sorts_within_scope(): assert a_idx < m_idx < z_idx +def test_format_skills_for_prompt_preserves_literal_compatibility_bytes(): + skills = [ + _skill("zeta", "user", description="User description"), + _skill("alpha", "project", description="Project description"), + ] + + assert format_skills_for_prompt(skills) == ( + "### Project\n" + "- alpha\n" + " - Path: /tmp/project/alpha/SKILL.md\n" + " - Description: Project description\n\n" + "### User\n" + "- zeta\n" + " - Path: /tmp/user/zeta/SKILL.md\n" + " - Description: User description" + ) + + @pytest.mark.asyncio async def test_discovered_skills_carry_scope(tmp_path, monkeypatch): """End-to-end: scoped discovery stamps each skill with its origin scope.""" diff --git a/tests/core/test_toolset.py b/tests/core/test_toolset.py index 7e2213e8..0501787a 100644 --- a/tests/core/test_toolset.py +++ b/tests/core/test_toolset.py @@ -10,10 +10,13 @@ from typing import Any, ClassVar, cast import mcp +import pytest from pydantic import BaseModel from pythinker_core.tooling import CallableTool, CallableTool2, ToolOk, ToolReturnValue from pythinker_core.tooling.error import ToolNotFoundError as PythinkerCoreToolNotFoundError +from pythinker_code.hooks import HookDef, HookEngine +from pythinker_code.hooks.runner import HookResult from pythinker_code.soul.toolset import ( MCPServerInfo, MCPTool, @@ -647,6 +650,118 @@ async def fake_trigger(*_args: object, **_kwargs: object) -> list[SimpleNamespac ] +async def test_pre_tool_use_block_survives_telemetry_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Hook telemetry is best-effort; it must never erase a security block.""" + entered = asyncio.Event() + release = asyncio.Event() + engine = HookEngine([HookDef(event="PreToolUse", matcher="ToolA", command="unused")]) + + async def execute_hooks(*_args: object, **_kwargs: object) -> list[HookResult]: + entered.set() + await release.wait() + return [HookResult(action="block", reason="policy denied")] + + def fail_telemetry(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("telemetry unavailable") + + monkeypatch.setattr(engine, "_execute_hooks", execute_hooks) + monkeypatch.setattr("pythinker_code.telemetry.track", fail_telemetry) + toolset = _make_toolset() + toolset.set_hook_engine(engine) + toolset.begin_step([]) + + result = toolset.handle( + ToolCall( + id="blocked-with-telemetry-failure", + function=ToolCall.FunctionBody(name="ToolA", arguments="{}"), + ) + ) + assert isinstance(result, asyncio.Task) + await entered.wait() + release.set() + completed = await result + + assert completed.return_value.is_error is True + assert completed.return_value.brief == "Hook blocked" + assert completed.return_value.message == "policy denied" + + +async def test_pre_tool_use_exception_fails_open_and_later_call_recovers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The documented hook-engine policy is fail-open for execution errors.""" + engine = HookEngine([HookDef(event="PreToolUse", matcher="ToolA", command="unused")]) + attempts = 0 + + async def execute_hooks(*_args: object, **_kwargs: object) -> list[HookResult]: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("hook transport failed") + return [HookResult(action="allow")] + + monkeypatch.setattr(engine, "_execute_hooks", execute_hooks) + toolset = _make_toolset() + toolset.set_hook_engine(engine) + + for index in range(2): + toolset.begin_step([], step_no=index + 1) + result = toolset.handle( + ToolCall( + id=f"hook-recovery-{index}", + function=ToolCall.FunctionBody( + name="ToolA", arguments=json.dumps({"value": str(index)}) + ), + ) + ) + assert isinstance(result, asyncio.Task) + completed = await result + assert completed.return_value.is_error is False + + assert attempts == 2 + + +async def test_post_tool_use_failure_isolated_from_result_and_later_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fire-and-forget post-hook failure cannot rewrite a successful tool result.""" + engine = HookEngine([HookDef(event="PostToolUse", matcher="ToolA", command="unused")]) + failed = asyncio.Event() + + async def execute_hooks(event: str, *_args: object, **_kwargs: object) -> list[HookResult]: + if event == "PostToolUse": + failed.set() + raise RuntimeError("post hook failed") + return [] + + monkeypatch.setattr(engine, "_execute_hooks", execute_hooks) + toolset = _make_toolset() + toolset.set_hook_engine(engine) + + for index in range(2): + toolset.begin_step([], step_no=index + 1) + result = toolset.handle( + ToolCall( + id=f"post-hook-recovery-{index}", + function=ToolCall.FunctionBody( + name="ToolA", arguments=json.dumps({"value": str(index)}) + ), + ) + ) + assert isinstance(result, asyncio.Task) + completed = await result + assert completed.return_value.is_error is False + await failed.wait() + failed.clear() + + pending = tuple(engine._pending_fire_and_forget) # pyright: ignore[reportPrivateUsage] + if pending: + await asyncio.gather(*pending) + assert all(task.done() for task in pending) + + async def test_cross_step_duplicate_uses_sparse_stronger_reminders(): """The stronger reminder appears at the fifth repeat and includes canonical args.""" ts = _make_toolset() diff --git a/tests/core/test_toolset_characterization.py b/tests/core/test_toolset_characterization.py new file mode 100644 index 00000000..6dc67df3 --- /dev/null +++ b/tests/core/test_toolset_characterization.py @@ -0,0 +1,619 @@ +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +import time +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +import pythinker_code.benchmark.toolset_characterization as characterization_mod +from pythinker_code.benchmark.toolset_characterization import ( + CancellationResult, + CharacterizationReport, + EnvironmentSnapshot, + FixtureShape, + LeakSnapshot, + PhaseSamples, + ScenarioKind, + ScenarioResult, + ThresholdState, + _await_handle, + _BarrierTool, + _ExecutionProbeToolset, + _measure_execution, + _new_tool, + _tool_call, + build_threshold_decisions, + deterministic_registry_hash, + evaluate_threshold, + fixture_matrix, + interval_union_duration_ns, + run_characterization, +) + + +def _decision_scenario( + kind: ScenarioKind, + size: int, + phases: dict[str, tuple[int, ...]], +) -> ScenarioResult: + return ScenarioResult( + fixture=FixtureShape(kind=kind, size=size, concurrency=size), + warmups=1, + iterations=5, + phases={name: PhaseSamples.from_samples(values) for name, values in phases.items()}, + registry_hash=deterministic_registry_hash((kind, str(size))), + allocation_peak_bytes=0, + retained_object_delta=0, + cancellation=CancellationResult(completed=True, completion_ns=0), + leaks=LeakSnapshot(tasks=0, processes=0, sessions=0), + task_count_peak=0, + operation_count=5, + category_counts={}, + projection_counts={}, + lifecycle_status="settled" if kind == "mcp" else "completed", + ) + + +@pytest.fixture +def characterization_report() -> CharacterizationReport: + return CharacterizationReport( + environment=EnvironmentSnapshot.current(), + scenarios=( + ScenarioResult( + fixture=FixtureShape(kind="execution_safe", size=1, concurrency=1), + warmups=1, + iterations=2, + phases={ + "framework_overhead": PhaseSamples.from_samples((10, 20)), + "tool_duration": PhaseSamples.from_samples((1, 1)), + }, + registry_hash=deterministic_registry_hash(("Noop",)), + allocation_peak_bytes=100, + retained_object_delta=0, + cancellation=CancellationResult(completed=True, completion_ns=10), + leaks=LeakSnapshot(tasks=0, processes=0, sessions=0), + task_count_peak=1, + operation_count=1, + category_counts={"builtin": 1}, + projection_counts={"visible": 1}, + lifecycle_status="completed", + ), + ), + decisions=(), + ) + + +def test_threshold_crosses_only_with_four_of_five_and_crossing_median() -> None: + decision = evaluate_threshold( + name="registry_500_p95", + threshold=5.0, + values=(5.1, 5.2, 1.0, 5.3, 5.4), + ) + + assert decision.state is ThresholdState.CROSSED + assert decision.crossing_count == 4 + assert decision.median == 5.2 + assert decision.rerun_required is False + + +def test_measured_short_safe_framework_overhead_records_crossed_threshold() -> None: + decision = evaluate_threshold( + name="execution_framework_overhead_percent_short_safe_size_1", + threshold=10.0, + values=( + 99.41801566579635, + 99.44035789159838, + 99.56835157490183, + 99.6366887606035, + 99.62501511204897, + ), + ) + + assert decision.state is ThresholdState.CROSSED + assert decision.median == 99.56835157490183 + + +def test_decision_builder_derives_all_gates_from_raw_scenarios() -> None: + scenarios = ( + _decision_scenario( + "execution_safe", + 1, + { + "framework_overhead": (11, 12, 13, 14, 15), + "end_to_end": (100, 100, 100, 100, 100), + }, + ), + _decision_scenario( + "execution_mixed", + 10, + { + "read_write_gate_wait": (20, 20, 20, 20, 20), + "end_to_end": (100, 100, 100, 100, 100), + }, + ), + _decision_scenario( + "advertisement", + 500, + {"registry_projection_p95": (4_000_000,) * 5}, + ), + *( + _decision_scenario( + "mcp", + size, + { + "startup_to_ready": (100,) * 5, + "mcp_lifecycle": (10,) * 5, + "cleanup": (1_000_000_000,) * 5, + }, + ) + for size in (1, 10, 50) + ), + ) + + decisions = build_threshold_decisions(scenarios) + + assert [decision.name for decision in decisions] == [ + "execution_framework_overhead_percent_short_safe_size_1", + "mcp_lifecycle_startup_percent_10_servers", + "mcp_cleanup_seconds_1_servers", + "mcp_cleanup_seconds_10_servers", + "mcp_cleanup_seconds_50_servers", + "registry_projection_p95_ms_500_tools", + "mixed_gate_wait_end_to_end_percent_10_pairs", + ] + assert decisions[5].values == (4.0, 4.0, 4.0, 4.0, 4.0) + assert decisions[5].state is ThresholdState.UNCROSSED + + +def test_phase_samples_preserve_raw_within_run_projection_samples() -> None: + within_runs = ( + (1_000_000, 2_000_000, 3_000_000), + (2_000_000, 3_000_000, 4_000_000), + ) + + phase = PhaseSamples.from_samples( + (3_000_000, 4_000_000), + within_run_samples_ns=within_runs, + ) + + assert phase.within_run_samples_ns == within_runs + + +async def test_full_five_run_all_builds_tracked_decisions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + scenarios = ( + _decision_scenario( + "execution_safe", + 1, + { + "framework_overhead": (11, 12, 13, 14, 15), + "end_to_end": (100,) * 5, + }, + ), + _decision_scenario( + "execution_mixed", + 10, + { + "read_write_gate_wait": (20,) * 5, + "end_to_end": (100,) * 5, + }, + ), + _decision_scenario( + "advertisement", + 500, + {"registry_projection_p95": (4_000_000,) * 5}, + ), + *( + _decision_scenario( + "mcp", + size, + { + "startup_to_ready": (100,) * 5, + "mcp_lifecycle": (10,) * 5, + "cleanup": (1_000_000_000,) * 5, + }, + ) + for size in (1, 10, 50) + ), + ) + by_fixture = { + (scenario.fixture.kind, scenario.fixture.size): scenario for scenario in scenarios + } + + monkeypatch.setattr( + characterization_mod, + "fixture_matrix", + lambda *, smoke=False: tuple(scenario.fixture for scenario in scenarios), + ) + + async def measure(fixture: FixtureShape, *, runs: int, warmups: int) -> ScenarioResult: + assert runs == 5 + assert warmups == 1 + return by_fixture[(fixture.kind, fixture.size)] + + monkeypatch.setattr(characterization_mod, "_measure_fixture", measure) + + report = await run_characterization(scenario="all", runs=5) + + assert len(report.decisions) == 7 + assert report.decisions[5].name == "registry_projection_p95_ms_500_tools" + + +def test_threshold_rejects_nonfinite_threshold() -> None: + with pytest.raises(ValueError, match="threshold must be finite"): + evaluate_threshold(name="example", threshold=float("nan"), values=(1, 2, 3, 4, 5)) + + +@pytest.mark.parametrize( + "values", + [ + (1.0, 2.0, 3.0, 4.0, 5.0), + (1.0, 2.0, 6.0, 7.0, 8.0), + ], +) +def test_threshold_is_uncrossed_without_repeatable_crossing( + values: tuple[float, float, float, float, float], +) -> None: + decision = evaluate_threshold(name="example", threshold=5.0, values=values) + + assert decision.state is ThresholdState.UNCROSSED + assert decision.rerun_required is False + + +def test_single_crossing_outlier_requests_exactly_one_complete_rerun() -> None: + first = evaluate_threshold( + name="example", + threshold=5.0, + values=(1.0, 1.1, 1.2, 1.3, 50.0), + ) + + assert first.state is ThresholdState.INCONCLUSIVE + assert first.rerun_required is True + + rerun = evaluate_threshold( + name="example", + threshold=5.0, + values=(1.0, 1.1, 1.2, 1.3, 50.0), + rerun_values=(1.0, 1.1, 1.2, 1.3, 75.0), + ) + + assert rerun.state is ThresholdState.INCONCLUSIVE + assert rerun.primary_state is ThresholdState.INCONCLUSIVE + assert rerun.rerun_state is ThresholdState.INCONCLUSIVE + assert rerun.rerun_required is False + assert rerun.rerun_values == (1.0, 1.1, 1.2, 1.3, 75.0) + + +def test_complete_rerun_replaces_an_inconclusive_primary_decision() -> None: + decision = evaluate_threshold( + name="example", + threshold=5.0, + values=(1.0, 1.1, 1.2, 1.3, 50.0), + rerun_values=(6.0, 6.1, 6.2, 6.3, 1.0), + ) + + assert decision.state is ThresholdState.CROSSED + assert decision.primary_state is ThresholdState.INCONCLUSIVE + assert decision.rerun_state is ThresholdState.CROSSED + assert decision.crossing_count == 4 + assert decision.median == 6.1 + assert decision.rerun_required is False + + +def test_evaluator_rejects_partial_primary_run() -> None: + with pytest.raises(ValueError, match="exactly five"): + evaluate_threshold(name="example", threshold=5.0, values=(1.0, 2.0)) + + +def test_evaluator_rejects_rerun_without_inconclusive_primary() -> None: + with pytest.raises(ValueError, match="only valid after an inconclusive"): + evaluate_threshold( + name="example", + threshold=5.0, + values=(6.0, 6.1, 6.2, 6.3, 6.4), + rerun_values=(1.0, 1.1, 1.2, 1.3, 1.4), + ) + + +def test_fixture_matrix_covers_the_approved_directional_shapes() -> None: + fixtures = fixture_matrix() + + assert {(item.kind, item.size) for item in fixtures} == { + *(("execution_safe", size) for size in (1, 10, 100)), + *(("execution_exclusive", size) for size in (1, 10, 100)), + *(("execution_mixed", size) for size in (1, 10, 100)), + *(("dedupe", size) for size in (1024, 100 * 1024, 1024 * 1024)), + *(("advertisement", size) for size in (50, 500, 5000)), + *(("mcp", size) for size in (1, 10, 50)), + } + assert { + (item.kind, item.size): item.concurrency + for item in fixtures + if item.kind.startswith("execution_") + } == { + **{("execution_safe", size): size for size in (1, 10, 100)}, + **{("execution_exclusive", size): size for size in (1, 10, 100)}, + **{("execution_mixed", size): size * 2 for size in (1, 10, 100)}, + } + + +def test_registry_hash_is_order_sensitive_and_reproducible() -> None: + first = deterministic_registry_hash(("Alpha", "Beta", "Gamma")) + + assert first == deterministic_registry_hash(("Alpha", "Beta", "Gamma")) + assert first != deterministic_registry_hash(("Beta", "Alpha", "Gamma")) + + +def test_overlapping_tool_intervals_use_union_not_sum() -> None: + assert interval_union_duration_ns(((10, 30), (20, 40), (50, 55))) == 35 + + +def test_unmeasured_phase_is_explicit_not_zero() -> None: + phase = PhaseSamples.unmeasured("no stable public phase boundary") + + assert phase.measurement_status == "unmeasured" + assert phase.samples_ns == () + assert phase.median_ns is None + assert phase.p95_ns is None + assert phase.reason == "no stable public phase boundary" + + +def test_report_schema_contains_required_measurement_and_safety_fields() -> None: + schema = CharacterizationReport.model_json_schema() + scenario_schema = schema["$defs"]["ScenarioResult"]["properties"] + phase_schema = schema["$defs"]["PhaseSamples"]["properties"] + + assert set(schema["properties"]) >= {"schema_version", "environment", "scenarios", "decisions"} + assert set(scenario_schema) >= { + "fixture", + "warmups", + "iterations", + "phases", + "registry_hash", + "allocation_peak_bytes", + "retained_object_delta", + "cancellation", + "leaks", + "task_count_peak", + "operation_count", + "category_counts", + "projection_counts", + "lifecycle_status", + } + assert set(phase_schema) >= { + "measurement_status", + "reason", + "samples_ns", + "median_ns", + "p95_ns", + "throughput_per_second", + "within_run_samples_ns", + } + + +def test_report_json_is_machine_readable(characterization_report: CharacterizationReport) -> None: + payload = json.loads(characterization_report.model_dump_json()) + + assert payload["schema_version"] == 2 + assert payload["environment"]["python_version"] + assert payload["scenarios"][0]["phases"]["framework_overhead"]["samples_ns"] == [10, 20] + + +async def test_execution_smoke_separates_tool_duration_from_framework_overhead() -> None: + report = await run_characterization(scenario="execution", runs=1, smoke=True) + + assert [scenario.fixture.kind for scenario in report.scenarios] == [ + "execution_safe", + "execution_exclusive", + "execution_mixed", + ] + expected_phases = { + "lookup_suggestion", + "json_parse_canonicalize", + "deduplication", + "permission_approval", + "pre_hook", + "read_write_gate_wait", + "tool_call", + "post_hook_reminder", + "telemetry_wire", + "end_to_end", + "framework_overhead", + } + for scenario in report.scenarios: + assert set(scenario.phases) == expected_phases + assert ( + scenario.phases["end_to_end"].samples_ns[0] + >= scenario.phases["tool_call"].samples_ns[0] + ) + assert scenario.phases["lookup_suggestion"].measurement_status == "unmeasured" + expected_operations = ( + scenario.fixture.size * 2 + if scenario.fixture.kind == "execution_mixed" + else scenario.fixture.size + ) + assert scenario.operation_count == expected_operations + assert scenario.cancellation.completed is True + assert scenario.cancellation.queued_reader_completed is True + assert scenario.cancellation.queued_writer_completed is True + assert scenario.cancellation.recovery_completed is True + assert scenario.leaks.tasks == 0 + + mixed = report.scenarios[-1] + assert mixed.fixture.size == 1 + assert mixed.fixture.concurrency == 2 + assert mixed.operation_count == 2 + assert mixed.phases["read_write_gate_wait"].samples_ns[0] > 0 + + +async def test_gate_wait_measures_request_to_admission_before_tool_call() -> None: + toolset = _ExecutionProbeToolset() + entered = asyncio.Event() + release = asyncio.Event() + holder = _BarrierTool( + name="Holder", + description="Known reader holder.", + parameters={"type": "object", "properties": {}}, + ) + holder.configure(entered=entered, release=release) + queued = _new_tool("Queued", parallel=False) + toolset.add(holder) + toolset.add(queued) + toolset.begin_step([]) + + holder_result = toolset.handle(_tool_call("holder", "Holder", 0)) + await entered.wait() + queued_result = toolset.handle(_tool_call("queued", "Queued", 1)) + await toolset.gate_requested.setdefault("Queued", asyncio.Event()).wait() + + assert queued.intervals_ns == () + released_ns = time.monotonic_ns() + release.set() + await asyncio.gather(_await_handle(holder_result), _await_handle(queued_result)) + + _, requested_ns, admitted_ns = next( + interval for interval in toolset.gate_wait_intervals_ns if interval[0] == "Queued" + ) + tool_started_ns, tool_finished_ns = queued.intervals_ns[0] + assert requested_ns < released_ns <= admitted_ns + assert admitted_ns <= tool_started_ns < tool_finished_ns + assert admitted_ns - requested_ns > 0 + + +async def test_mixed_task_peak_samples_all_dispatched_operations() -> None: + sample = await _measure_execution(FixtureShape(kind="execution_mixed", size=10, concurrency=20)) + + assert sample.operation_count == 20 + assert sample.task_count >= 20 + + +async def test_dedupe_smoke_executes_one_tool_for_two_identical_calls() -> None: + report = await run_characterization(scenario="dedupe", runs=1, smoke=True) + scenario = report.scenarios[0] + + assert scenario.fixture.payload_bytes == 1024 + assert scenario.fixture.concurrency == 2 + assert scenario.operation_count == 1 + assert scenario.phases["deduplication"].measurement_status == "unmeasured" + + +async def test_advertisement_smoke_measures_hidden_and_unhidden_projection() -> None: + report = await run_characterization(scenario="advertisement", runs=2, smoke=True) + scenario = report.scenarios[0] + + assert set(scenario.phases) == { + "visibility_enabled_hidden", + "visibility_enabled_unhidden", + "visibility_disabled_hidden", + "visibility_disabled_unhidden", + "repeated_unchanged_projection", + "registry_projection_p95", + "rebuild_after_mcp_publication", + } + projection = scenario.phases["registry_projection_p95"] + assert len(projection.within_run_samples_ns) == 2 + assert all(len(run) >= 5 for run in projection.within_run_samples_ns) + assert all(scenario.category_counts[origin] > 0 for origin in ("builtin", "plugin", "mcp")) + assert ( + scenario.projection_counts["enabled_hidden"] + < scenario.projection_counts["enabled_unhidden"] + ) + assert scenario.projection_counts["rebuild"] == scenario.fixture.size + assert scenario.operation_count == scenario.fixture.size * 2 + + +async def test_mcp_smoke_measures_current_background_lifecycle_and_cleanup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from pythinker_code.soul.agent import Runtime + + create_spy = AsyncMock(wraps=Runtime.create) + monkeypatch.setattr(Runtime, "create", create_spy) + report = await run_characterization(scenario="mcp", runs=1, warmups=0, smoke=True) + scenario = report.scenarios[0] + + assert create_spy.await_count == 1 + assert set(scenario.phases) == { + "startup_to_ready", + "mcp_lifecycle", + "time_to_first_inventory", + "time_to_settled_inventory", + "cleanup", + } + assert scenario.task_count_peak >= 1 + assert scenario.operation_count == scenario.fixture.size + assert ( + scenario.phases["startup_to_ready"].samples_ns[0] + >= scenario.phases["mcp_lifecycle"].samples_ns[0] + ) + assert ( + scenario.phases["time_to_first_inventory"].samples_ns[0] + <= scenario.phases["time_to_settled_inventory"].samples_ns[0] + ) + assert scenario.projection_counts["visible"] == scenario.fixture.size + assert scenario.projection_counts["visible_at_first_publication"] == scenario.fixture.size + assert scenario.lifecycle_status == "settled" + assert scenario.leaks == LeakSnapshot(tasks=0, processes=0, sessions=0) + + +def test_runner_help_documents_scenario_runs_and_output() -> None: + completed = subprocess.run( + [sys.executable, "scripts/benchmark_toolset.py", "--help"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + + assert completed.returncode == 0 + assert "--scenario" in completed.stdout + assert "--runs" in completed.stdout + assert "--output" in completed.stdout + + +def test_runner_smoke_writes_json_without_network_or_secrets(tmp_path: Path) -> None: + output = tmp_path / "toolset.json" + completed = subprocess.run( + [ + sys.executable, + "scripts/benchmark_toolset.py", + "--scenario", + "advertisement", + "--runs", + "1", + "--smoke", + "--output", + str(output), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + + assert completed.returncode == 0, completed.stderr + payload = json.loads(output.read_text(encoding="utf-8")) + assert [item["fixture"]["size"] for item in payload["scenarios"]] == [50] + assert payload["decisions"] == [] + + +def test_runner_rejects_nonpositive_run_count() -> None: + completed = subprocess.run( + [sys.executable, "scripts/benchmark_toolset.py", "--runs", "0"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + + assert completed.returncode != 0 + assert "must be positive" in completed.stderr diff --git a/tests/core/test_toolset_concurrency.py b/tests/core/test_toolset_concurrency.py index cd0b268c..293aa7cf 100644 --- a/tests/core/test_toolset_concurrency.py +++ b/tests/core/test_toolset_concurrency.py @@ -13,6 +13,7 @@ import json from pathlib import Path +import pytest from pythinker_core.tooling import ToolReturnValue from pythinker_code.hooks.engine import HookEngine @@ -39,6 +40,24 @@ async def call(self, arguments: object) -> ToolReturnValue: return ToolReturnValue(is_error=False, output="ok", message="ok", display=[]) +class _AdmissionBlockingTool(_RecordingTool): + def __init__(self, name: str, events: list[tuple[str, str]]) -> None: + super().__init__(name, events, parallel=True, delay=0) + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.closed = asyncio.Event() + + async def call(self, arguments: object) -> ToolReturnValue: + self._events.append(("enter", self.name)) + self.entered.set() + try: + await self.release.wait() + return ToolReturnValue(is_error=False, output="ok", message="ok", display=[]) + finally: + self._events.append(("close", self.name)) + self.closed.set() + + def _toolset(*tools: _RecordingTool, cwd: Path) -> PythinkerToolset: toolset = PythinkerToolset() toolset._hook_engine = HookEngine([], cwd=str(cwd)) @@ -225,6 +244,126 @@ async def writer() -> None: await asyncio.gather(held, writer_task, queued, return_exceptions=True) +async def test_cancelled_queued_reader_releases_permit_and_later_reader_recovers() -> None: + from pythinker_code.soul.toolset import _ReadWriteGate + + gate = _ReadWriteGate(max_concurrent_readers=1) + writer_entered = asyncio.Event() + release_writer = asyncio.Event() + reader_requested = asyncio.Event() + recovered = asyncio.Event() + + async def writer() -> None: + async with gate.exclusive(): + writer_entered.set() + await release_writer.wait() + + async def queued_reader() -> None: + reader_requested.set() + async with gate.shared(): + raise AssertionError("cancelled reader entered the protected region") + + writer_task = asyncio.create_task(writer()) + await writer_entered.wait() + reader_task = asyncio.create_task(queued_reader()) + await reader_requested.wait() + reader_task.cancel() + with pytest.raises(asyncio.CancelledError): + await reader_task + release_writer.set() + await writer_task + + async with gate.shared(): + recovered.set() + assert recovered.is_set() + + +async def test_cancelled_queued_writer_does_not_block_later_writer() -> None: + from pythinker_code.soul.toolset import _ReadWriteGate + + gate = _ReadWriteGate(max_concurrent_readers=1) + reader_entered = asyncio.Event() + release_reader = asyncio.Event() + writer_requested = asyncio.Event() + + async def reader() -> None: + async with gate.shared(): + reader_entered.set() + await release_reader.wait() + + async def queued_writer() -> None: + writer_requested.set() + async with gate.exclusive(): + raise AssertionError("cancelled writer entered the protected region") + + reader_task = asyncio.create_task(reader()) + await reader_entered.wait() + writer_task = asyncio.create_task(queued_writer()) + await writer_requested.wait() + writer_task.cancel() + with pytest.raises(asyncio.CancelledError): + await writer_task + release_reader.set() + await reader_task + + async with gate.exclusive(): + pass + + +async def test_cancellation_after_reader_admission_restores_gate_state() -> None: + from pythinker_code.soul.toolset import _ReadWriteGate + + gate = _ReadWriteGate(max_concurrent_readers=1) + admitted = asyncio.Event() + + async def admitted_reader() -> None: + async with gate.shared(): + admitted.set() + await asyncio.Event().wait() + + task = asyncio.create_task(admitted_reader()) + await admitted.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + async with gate.exclusive(): + pass + + +async def test_handle_cancellation_after_admission_closes_and_recovers(tmp_path: Path) -> None: + events: list[tuple[str, str]] = [] + blocking = _AdmissionBlockingTool("Read", events) + recovery = _RecordingTool("Write", events, parallel=False, delay=0) + toolset = _toolset(blocking, recovery, cwd=tmp_path) + toolset.begin_step([]) + running = toolset.handle( + ToolCall( + id="admitted-read", + function=ToolCall.FunctionBody(name="Read", arguments='{"attempt":1}'), + ) + ) + assert isinstance(running, asyncio.Task) + await blocking.entered.wait() + + running.cancel() + with pytest.raises(asyncio.CancelledError): + await running + await blocking.closed.wait() + + recovered = toolset.handle( + ToolCall( + id="recovery-write", + function=ToolCall.FunctionBody(name="Write", arguments='{"attempt":2}'), + ) + ) + assert isinstance(recovered, asyncio.Task) + result = await recovered + + assert result.return_value.is_error is False + assert events == [("enter", "Read"), ("close", "Read"), ("enter", "Write"), ("exit", "Write")] + + class TestPluginToolDefault: async def test_plugin_tool_without_supports_parallel_runs_exclusively( self, tmp_path: Path diff --git a/tests/e2e/test_shell_pty_e2e.py b/tests/e2e/test_shell_pty_e2e.py index 1a37bf03..48b0e058 100644 --- a/tests/e2e/test_shell_pty_e2e.py +++ b/tests/e2e/test_shell_pty_e2e.py @@ -946,7 +946,14 @@ def test_shell_cancel_running_command_kills_process_and_recovers(tmp_path: Path) shell.send_line("start cancellable command") shell.read_until_contains("Bash(sleep 5", after=cancel_mark) shell.send_key("escape") - shell.read_until_contains("Interrupted by user", after=cancel_mark) + # The "Interrupted by user" acknowledgement only prints after the soul + # re-raises the cancellation, which first awaits a shielded, disk-first + # context append (the interrupted-tool marker write) so history never + # keeps an unanswered tool_call. That append hops through the shared + # thread pool, so under heavy CPU contention the acknowledgement can + # legitimately trail the default 15s budget; give this one wait generous + # headroom to keep the e2e stable on loaded CI without masking a hang. + shell.read_until_contains("Interrupted by user", after=cancel_mark, timeout=45.0) cancel_prompt_mark = shell.mark() _read_until_prompt(shell, after=cancel_prompt_mark) diff --git a/tests/fixtures/skill_catalog_recall.json b/tests/fixtures/skill_catalog_recall.json new file mode 100644 index 00000000..36d69dbf --- /dev/null +++ b/tests/fixtures/skill_catalog_recall.json @@ -0,0 +1,25 @@ +{ + "skills": [ + {"name": "release-deploy", "description": "Deploy release artifacts safely", "scope": "builtin"}, + {"name": "review-pr", "description": "Review pull requests for correctness"}, + {"name": "spreadsheets", "description": "Create polished spreadsheet workbooks"}, + {"name": "diagnose-ci-failures", "description": "Diagnose failing continuous integration jobs"}, + {"name": "gh-fix-ci", "description": "Fix GitHub continuous integration failures"}, + {"name": "art", "description": "Create illustrations and drawings"}, + {"name": "cartography", "description": "Create geographic maps"}, + {"name": "release-notes", "description": "Write release documentation", "scope": "builtin"}, + {"name": "generic-release", "description": "General release workflow", "scope": "project"}, + {"name": "shadowed", "description": "Project winning specialization", "scope": "project", "shadowed": "User losing specialization"} + ], + "cases": [ + {"query": "deploy the release safely", "expected": ["release-deploy"], "winner_tier": "NAME_TOKEN"}, + {"query": "review this pull request", "expected": ["review-pr"], "winner_tier": "NAME_TOKEN"}, + {"query": "create a polished spreadsheet workbook", "expected": ["spreadsheets"], "winner_tier": "DESCRIPTION_TOKEN"}, + {"query": "debug a failing continuous integration job", "expected": ["diagnose-ci-failures"], "winner_tier": "DESCRIPTION_TOKEN"}, + {"query": "/skill:github:gh-fix-ci", "expected": ["gh-fix-ci"], "winner_tier": "EXACT_NAME"}, + {"query": "cartography", "expected": ["cartography"], "winner_tier": "EXACT_NAME"}, + {"query": "release notes", "expected": ["release-notes"], "winner_tier": "EXACT_NAME"}, + {"query": "release workflow", "expected": ["generic-release", "release-deploy", "release-notes"], "winner_tier": "NAME_TOKEN"}, + {"query": "project winning specialization", "expected": ["shadowed"], "winner_tier": "DESCRIPTION_TOKEN"} + ] +} diff --git a/tests/telemetry/test_instrumentation.py b/tests/telemetry/test_instrumentation.py index b20c3da7..e089aed1 100644 --- a/tests/telemetry/test_instrumentation.py +++ b/tests/telemetry/test_instrumentation.py @@ -853,6 +853,7 @@ class TestCompactionTracking: def _make_soul(self, *, before_tokens: int, estimated_after: int) -> Any: """Construct a minimal PythinkerSoul stub bypassing __init__.""" from pythinker_code.soul.pythinkersoul import PythinkerSoul + from pythinker_code.soul.request_lifecycle import RequestLifecycle soul = object.__new__(PythinkerSoul) @@ -866,10 +867,7 @@ def _make_soul(self, *, before_tokens: int, estimated_after: int) -> Any: ctx = MagicMock() ctx.token_count = before_tokens ctx.history = [] - ctx.clear = AsyncMock() - ctx.write_system_prompt = AsyncMock() - ctx.append_message = AsyncMock() - ctx.update_token_count = AsyncMock() + ctx.replace_history = AsyncMock() soul._context = ctx soul._hook_engine = MagicMock() @@ -885,6 +883,7 @@ def _make_soul(self, *, before_tokens: int, estimated_after: int) -> Any: soul._loop_control = loop_control soul._checkpoint = AsyncMock() + soul._checkpoint_with_user_message = False # _run_with_connection_recovery returns a value with .messages and # .estimated_token_count — shape it with MagicMock to avoid depending @@ -900,6 +899,8 @@ def _make_soul(self, *, before_tokens: int, estimated_after: int) -> Any: soul._run_with_connection_recovery = AsyncMock(return_value=fake_result) soul._injection_providers = [] + soul._request_lifecycle = RequestLifecycle([]) + soul._notified_context_generations = set() return soul @pytest.mark.asyncio diff --git a/tests/test_installation_docs.py b/tests/test_installation_docs.py index 1ba2b566..6a75fe70 100644 --- a/tests/test_installation_docs.py +++ b/tests/test_installation_docs.py @@ -205,7 +205,6 @@ def test_no_old_repo_owner_references() -> None: old_owner + ".github.io/Pythinker-Code", ] skip_dirs = { - "graphify-out", ".git", "dist", ".vitepress", diff --git a/tests/tools/test_skill_tool.py b/tests/tools/test_skill_tool.py index b9392052..be0b6956 100644 --- a/tests/tools/test_skill_tool.py +++ b/tests/tools/test_skill_tool.py @@ -6,7 +6,12 @@ from pythinker_host.path import HostPath import pythinker_code.skill as skill_module -from pythinker_code.skill import Skill, read_skill_text_with_local_specialization +from pythinker_code.skill import ( + ScopedSkillsRoot, + Skill, + SkillCatalog, + read_skill_text_with_local_specialization, +) from pythinker_code.tools.skill import ReadSkill @@ -52,6 +57,74 @@ async def test_read_skill_reports_missing_skill(runtime) -> None: assert result.is_error assert result.brief == "Skill not found" + assert "status: not_found" in result.message + + +async def test_read_skill_missing_name_returns_only_bounded_suggestions( + runtime, tmp_path: Path +) -> None: + runtime.skills = { + f"deploy-{index}": _skill(f"deploy-{index}", tmp_path / f"deploy-{index}.md") + for index in range(20) + } + runtime.skill_catalog = SkillCatalog(runtime.skills, ()) + + result = await ReadSkill(runtime)(ReadSkill.params(skill_name="deploy")) + + assert result.is_error + assert result.message.count("deploy-") <= 5 + assert "deploy-19" not in result.message + + +async def test_read_skill_distinguishes_unavailable_discovered_source( + runtime, tmp_path: Path +) -> None: + root = tmp_path / "skills" + broken = root / "broken" + broken.mkdir(parents=True) + (broken / "SKILL.md").write_text( + "---\nname: broken\ntype: unsupported\n---\n", + encoding="utf-8", + ) + runtime.skill_catalog = await SkillCatalog.discover( + [ + ScopedSkillsRoot( + root=HostPath.unsafe_from_local_path(root), + scope="project", + ) + ] + ) + runtime.skills = dict(runtime.skill_catalog.exhaustive_mapping()) + + result = await ReadSkill(runtime)(ReadSkill.params(skill_name="broken")) + + assert result.is_error + assert result.brief == "Skill unavailable" + assert "status: unavailable" in result.message + assert str(tmp_path) not in result.message + + +async def test_read_skill_deleted_after_discovery_reports_structured_unavailable( + runtime, tmp_path: Path +) -> None: + root = tmp_path / "skills" + skill_dir = root / "ephemeral" + skill_dir.mkdir(parents=True) + skill_path = skill_dir / "SKILL.md" + skill_path.write_text("---\nname: ephemeral\ndescription: Temporary\n---\n", encoding="utf-8") + runtime.skill_catalog = await SkillCatalog.discover( + [ScopedSkillsRoot(root=HostPath.unsafe_from_local_path(root), scope="project")] + ) + runtime.skills = runtime.skill_catalog.exhaustive_mapping() + skill_path.unlink() + + result = await ReadSkill(runtime)(ReadSkill.params(skill_name="ephemeral")) + + assert result.is_error + assert result.brief == "Skill unavailable" + assert result.message.startswith("status: unavailable\n") + assert "could not be read" in result.message + assert str(tmp_path) not in result.message async def test_read_skill_resolves_plugin_style_alias(runtime, tmp_path: Path) -> None: @@ -84,6 +157,7 @@ async def test_read_skill_mcp_bridge_when_filesystem_skill_missing(runtime) -> N assert not result.is_error assert isinstance(result.output, str) assert "MCP bridge" in result.output + assert "status: mcp_fallback" in result.output assert "mcp__designer-skill__get_design_system" in result.output assert "anti_slop_checklist" in result.output @@ -104,6 +178,23 @@ async def test_read_skill_mcp_bridge_works_for_user_added_server(runtime) -> Non assert "get_design_system" not in result.output +async def test_missing_skill_caps_connected_mcp_server_hint(runtime) -> None: + runtime.skills = {} + runtime.skill_catalog = SkillCatalog({}, ()) + runtime.mcp_tools = {f"mcp__server-{index:03d}__search": object() for index in range(50)} + + result = await ReadSkill(runtime)(ReadSkill.params(skill_name="missing")) + + assert result.is_error + assert len(result.message) < 500 + assert ( + "Connected MCP servers: server-000, server-001, server-002, server-003, server-004" + in result.message + ) + assert "45 omitted" in result.message + assert "server-049" not in result.message + + async def test_read_skill_appends_resource_manifest(runtime, tmp_path: Path) -> None: # skills-1: a subdirectory skill referencing scripts/ and references/ must # surface those bundled files at runtime so the model knows they exist and diff --git a/tests/ui_and_conv/test_btw.py b/tests/ui_and_conv/test_btw.py index 4d682ceb..7cc8b084 100644 --- a/tests/ui_and_conv/test_btw.py +++ b/tests/ui_and_conv/test_btw.py @@ -10,12 +10,19 @@ from pythinker_core.message import Message, ToolCall from pythinker_core.tooling import Tool, ToolError, ToolResult +from pythinker_code.soul.agent import Agent, Runtime from pythinker_code.soul.btw import ( _build_btw_context, _DenyAllToolset, _tool_result_to_message, execute_side_question, ) +from pythinker_code.soul.context import Context +from pythinker_code.soul.request_assembly import ( + AssembledRequest, + RequestManifest, + RequestStatus, +) from pythinker_code.ui.shell.prompt import PromptMode, UserInput from pythinker_code.ui.shell.visualize import ( InputAction, @@ -90,6 +97,55 @@ def _mixed_text_and_tool_result(text: str, tool_name: str = "Read") -> _FakeStep ) +class _ExecuteSideQuestionSoul: + """Typed fake for the default assembler-backed side-question contract.""" + + def __init__(self, *, llm_set: bool = True) -> None: + llm = MagicMock() if llm_set else None + if llm is not None: + llm.chat_provider = MagicMock() + self._runtime: Runtime = MagicMock(spec=Runtime) + self._runtime.llm = llm + self._agent: Agent = MagicMock(spec=Agent) + self._agent.system_prompt = "sys" + self._agent.toolset = MagicMock() + self._agent.toolset.tools = [] + self._context: Context = MagicMock(spec=Context) + self._context.history = [] + + @property + def runtime(self) -> Runtime: + return self._runtime + + @property + def agent(self) -> Agent: + return self._agent + + @property + def context(self) -> Context: + return self._context + + async def assemble_side_request(self, question: str, reminder_text: str) -> AssembledRequest: + history = (Message(role="user", content=f"{reminder_text}\n\n{question}"),) + return AssembledRequest( + system_prompt="sys", + provider_history=history, + history_appends=(), + manifest=RequestManifest( + status=RequestStatus.SUCCEEDED, + reason_code=None, + outcomes=(), + budget_tokens=0, + budgeted_admitted_tokens=0, + non_budgeted_estimated_tokens=0, + ), + ) + + +def _execute_soul() -> _ExecuteSideQuestionSoul: + return _ExecuteSideQuestionSoul() + + # --------------------------------------------------------------------------- # classify_input # --------------------------------------------------------------------------- @@ -175,6 +231,7 @@ def _make_soul(): soul = MagicMock() soul._agent.system_prompt = "You are a helpful assistant." soul._agent.toolset.tools = [MagicMock(spec=Tool)] + soul.agent = soul._agent soul.context.history = [ Message(role="user", content="hello"), Message(role="assistant", content="hi there"), @@ -241,19 +298,14 @@ def test_converts_error_to_tool_message(self): class TestExecuteSideQuestion: def test_llm_not_set_returns_error(self): - soul = MagicMock() - soul._runtime.llm = None + soul = _ExecuteSideQuestionSoul(llm_set=False) response, error = asyncio.run(execute_side_question(soul, "hi")) assert response is None assert error is not None and "LLM is not set" in error def test_text_on_first_turn(self): """LLM returns text immediately → return it.""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() async def fake_step(provider, sys_prompt, toolset, history, **kw): # Simulate streaming callback @@ -269,11 +321,7 @@ async def fake_step(provider, sys_prompt, toolset, history, **kw): def test_tool_call_then_text_on_second_turn(self): """LLM calls tool on turn 1 (denied), returns text on turn 2.""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() call_count = 0 @@ -297,11 +345,7 @@ async def fake_step(provider, sys_prompt, toolset, history, **kw): def test_tool_calls_on_both_turns(self): """LLM calls tools on both turns → error with tool names.""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() async def fake_step(provider, sys_prompt, toolset, history, **kw): return _tool_call_result("Bash") @@ -316,11 +360,7 @@ async def fake_step(provider, sys_prompt, toolset, history, **kw): def test_mixed_text_and_tool_retries_and_returns_second_turn(self): """LLM outputs text + tool_call on turn 1 → retry → turn 2 text is the answer.""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() call_count = 0 @@ -348,11 +388,7 @@ async def fake_step(provider, sys_prompt, toolset, history, **kw): def test_mixed_text_and_tool_on_both_turns_reports_error(self): """LLM outputs text + tool_call on both turns → error with tool names.""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() async def fake_step(provider, sys_prompt, toolset, history, **kw): if kw.get("on_message_part"): @@ -368,11 +404,7 @@ async def fake_step(provider, sys_prompt, toolset, history, **kw): def test_mixed_output_streaming_callback_receives_both_turns(self): """on_text_chunk receives chunks from both turns (preamble + real answer).""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() call_count = 0 chunks: list[str] = [] @@ -403,11 +435,7 @@ async def fake_step(provider, sys_prompt, toolset, history, **kw): def test_exception_returns_error(self): """LLM call raises exception → return error string.""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() async def fake_step(*args, **kw): raise RuntimeError("API timeout") @@ -420,11 +448,7 @@ async def fake_step(*args, **kw): def test_on_text_chunk_callback(self): """Streaming chunks are forwarded to on_text_chunk.""" - soul = MagicMock() - soul._runtime.llm.chat_provider = MagicMock() - soul._agent.system_prompt = "sys" - soul._agent.toolset.tools = [] - soul.context.history = [] + soul = _execute_soul() chunks: list[str] = [] diff --git a/tests_e2e/test_wire_protocol.py b/tests_e2e/test_wire_protocol.py index c2b6ea11..be2b32dc 100644 --- a/tests_e2e/test_wire_protocol.py +++ b/tests_e2e/test_wire_protocol.py @@ -54,6 +54,11 @@ def test_initialize_handshake(tmp_path) -> None: "description": "Recap Pythinker sessions. Usage: /recap [on|off|today|yesterday|week|YYYY-MM-DD]", "aliases": [], }, + { + "name": "prompt-manifest", + "description": "Show the latest sanitized request assembly manifest", + "aliases": [], + }, { "name": "compact", "description": "Compact the context (optionally with a custom focus, e.g. /compact keep db discussions)", @@ -331,6 +336,11 @@ def test_initialize_external_tool_conflict(tmp_path) -> None: "description": "Recap Pythinker sessions. Usage: /recap [on|off|today|yesterday|week|YYYY-MM-DD]", "aliases": [], }, + { + "name": "prompt-manifest", + "description": "Show the latest sanitized request assembly manifest", + "aliases": [], + }, { "name": "compact", "description": "Compact the context (optionally with a custom focus, e.g. /compact keep db discussions)",