From ceeeeb78b6975660d050c0a318badc08800a410a Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Thu, 11 Jun 2026 23:47:58 -0400 Subject: [PATCH 01/49] fix(tools): wrap FetchURL output as untrusted after truncation FetchURL wrote pre-rendered envelopes into the ToolResultBuilder, so when a page exceeded the builder's character limit, truncation cut off the closing tag. The model then received an unterminated untrusted-data envelope (the exact failure mode the wrapper exists to prevent), and strip_untrusted_envelope could not strip the torn envelope, so the raw envelope leaked into display/UI paths. Write the raw text and call builder.mark_untrusted() instead (the same idiom SearchWeb uses), so wrapping happens after truncation in ok() and the closing tag can never be cut. Applied to all three write sites: the verbatim text/plain+markdown path, the trafilatura extraction path, and the fetch-service path. The spill file now holds raw unwrapped output, matching the documented spill contract. Regression tests cover all three paths with >50k-char pages that trigger builder truncation, asserting the envelope stays well-formed and strip_untrusted_envelope round-trips. --- src/pythinker_code/tools/web/fetch.py | 13 ++++-- tests/tools/test_fetch_url.py | 60 ++++++++++++++++++++++++++ tests/tools/test_untrusted_wrapping.py | 60 +++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 5 deletions(-) diff --git a/src/pythinker_code/tools/web/fetch.py b/src/pythinker_code/tools/web/fetch.py index 8d653c1d..e24f06b4 100644 --- a/src/pythinker_code/tools/web/fetch.py +++ b/src/pythinker_code/tools/web/fetch.py @@ -18,7 +18,6 @@ from pythinker_code.tools.web._allowlist import host_in_allowlist from pythinker_code.utils.aiohttp import new_client_session from pythinker_code.utils.logging import logger -from pythinker_code.utils.trust import UntrustedData MAX_FETCH_BYTES = 5 * 1024 * 1024 MAX_FETCH_REDIRECTS = 10 # matches aiohttp's default redirect cap @@ -229,7 +228,11 @@ async def fetch_with_http_get( content_type = response.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower() if content_type.startswith(("text/plain", "text/markdown")): - builder.write(UntrustedData(resp_text).render_for_prompt()) + # Write the raw page text; mark_untrusted wraps the + # already-truncated block in ok(), so truncation can + # never cut off the closing tag. + builder.mark_untrusted() + builder.write(resp_text) # Spill the full page off the event loop before building the result. await builder.spill_to_disk() return builder.ok("The returned content is the full content of the page.") @@ -274,7 +277,8 @@ async def fetch_with_http_get( brief="No content extracted", ) - builder.write(UntrustedData(extracted_text).render_for_prompt()) + builder.mark_untrusted() + builder.write(extracted_text) return builder.ok("The returned content is the main text content extracted from the page.") async def _fetch_with_service(self, params: Params) -> ToolReturnValue: @@ -335,7 +339,8 @@ async def _fetch_with_service(self, params: Params) -> ToolReturnValue: f"Failed to fetch URL via service: response exceeds {max_mb}MB.", brief="Response too large", ) - builder.write(UntrustedData(content).render_for_prompt()) + builder.mark_untrusted() + builder.write(content) return builder.ok( "The returned content is the main content extracted from the page." ) diff --git a/tests/tools/test_fetch_url.py b/tests/tools/test_fetch_url.py index 22565a45..b5084985 100644 --- a/tests/tools/test_fetch_url.py +++ b/tests/tools/test_fetch_url.py @@ -396,6 +396,66 @@ async def service_handler(request: web.Request) -> web.Response: await runner.cleanup() +async def test_fetch_url_with_service_truncated_envelope_stays_closed(runtime) -> None: + """Service-fetched content larger than the builder limit must keep a + well-formed envelope: wrapping happens after truncation, + so the closing tag can never be cut off.""" + from pydantic import SecretStr + + from pythinker_code.config import Config, PythinkerAIFetchConfig, Services + from pythinker_code.tools.utils import DEFAULT_MAX_CHARS + from pythinker_code.utils.trust import strip_untrusted_envelope + + big_content = "".join(f"line {i}: service filler text\n" for i in range(4000)) + assert len(big_content) > DEFAULT_MAX_CHARS + + async def service_handler(request: web.Request) -> web.Response: # noqa: ARG001 + return web.Response(text=big_content) + + app = web.Application() + app.router.add_post("/fetch", service_handler) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, host="127.0.0.1", port=0) + await site.start() + port = site._server.sockets[0].getsockname()[1] # pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess] + + try: + config = Config( + services=Services( + pythinker_ai_fetch=PythinkerAIFetchConfig( + base_url=f"http://127.0.0.1:{port}/fetch", + api_key=SecretStr("test-key"), + ) + ) + ) + fetch_tool = FetchURL(config=config, runtime=runtime) + + from pythinker_code.soul.toolset import current_tool_call + from pythinker_code.wire.types import ToolCall + + token = current_tool_call.set( + ToolCall( + id="test-call-id", function=ToolCall.FunctionBody(name="FetchURL", arguments=None) + ) + ) + try: + result = await fetch_tool(Params(url="https://example.com")) + finally: + current_tool_call.reset(token) + + assert not result.is_error + assert isinstance(result.output, str) + assert "truncated" in result.message + # unwrap_untrusted asserts the envelope is well-formed (closing tag intact). + inner = unwrap_untrusted(result.output) + assert len(inner) < len(big_content) + assert strip_untrusted_envelope(result.output) == inner + + finally: + await runner.cleanup() + + @pytest_asyncio.fixture async def serve_app() -> AsyncIterator[Callable[[web.Application], Awaitable[str]]]: """Serve an arbitrary aiohttp app on a random loopback port and clean up.""" diff --git a/tests/tools/test_untrusted_wrapping.py b/tests/tools/test_untrusted_wrapping.py index 1ee814ac..26a809d4 100644 --- a/tests/tools/test_untrusted_wrapping.py +++ b/tests/tools/test_untrusted_wrapping.py @@ -16,10 +16,11 @@ from pythinker_host.path import HostPath from pythinker_code.tools.file.read import Params, ReadFile +from pythinker_code.tools.utils import DEFAULT_MAX_CHARS from pythinker_code.tools.web import fetch as fetch_module from pythinker_code.tools.web.fetch import FetchURL from pythinker_code.tools.web.fetch import Params as FetchParams -from pythinker_code.utils.trust import UntrustedData +from pythinker_code.utils.trust import UntrustedData, strip_untrusted_envelope from tests.tools._untrusted import assert_wrapped, unwrap_untrusted WRAPPER_RE = re.compile(r'^\n.*\n$', re.DOTALL) @@ -270,6 +271,63 @@ async def test_fetchurl_error_results_are_not_wrapped( assert " None: + """Truncation must never cut the closing ```` tag. + + A page larger than the builder's character limit used to be wrapped + *before* truncation, so the closing tag was truncated away — leaving an + unterminated envelope for the model and a raw envelope leaking into + display paths (``strip_untrusted_envelope`` only strips well-formed + envelopes). Wrapping must happen after truncation. + """ + body = "".join(f"line {i}: markdown filler text\n" for i in range(4000)) + assert len(body) > DEFAULT_MAX_CHARS + base, runner = await _start_server(body, "text/markdown; charset=utf-8") + try: + result = await fetch_url_tool(FetchParams(url=base)) + finally: + await runner.cleanup() + + assert not result.is_error + assert isinstance(result.output, str) + # Truncation actually happened (not a no-op page). + assert "truncated" in result.message + assert WRAPPER_RE.match(result.output), ( + f"truncated output tore the envelope: {result.output[-200:]!r}" + ) + inner = unwrap_untrusted(result.output) + assert len(inner) < len(body) + # The display-surface stripper round-trips: the envelope is removed cleanly. + assert strip_untrusted_envelope(result.output) == inner + + +async def test_fetchurl_truncated_extracted_html_envelope_stays_closed( + fetch_url_tool: FetchURL, + _bypass_ssrf_validation: None, +) -> None: + """The trafilatura-extraction path must also wrap after truncation.""" + paragraphs = "".join( + f"

paragraph {i} " + "lorem ipsum filler words " * 20 + "

" for i in range(200) + ) + body = f"

Big

{paragraphs}
" + base, runner = await _start_server(body, "text/html") + try: + result = await fetch_url_tool(FetchParams(url=base)) + finally: + await runner.cleanup() + + assert not result.is_error + assert isinstance(result.output, str) + assert "truncated" in result.message + assert WRAPPER_RE.match(result.output), ( + f"truncated output tore the envelope: {result.output[-200:]!r}" + ) + assert strip_untrusted_envelope(result.output) == unwrap_untrusted(result.output) + + # ── UntrustedData primitive round-trip via the tools ───────────────── From 7e96dfb838bff56b1279d411da2e21e25810c0f4 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Fri, 12 Jun 2026 00:02:35 -0400 Subject: [PATCH 02/49] refactor(soul): expose public turn() contract for external drivers External drivers (FlowRunner._flow_turn, the /goal and /learn slash handlers) were calling the private PythinkerSoul._turn directly, each carrying a pyright reportPrivateUsage suppression. Add a public turn() method that documents the single-turn contract: one user message in, one full agent turn out (model steps plus tool calls until the model stops), with TurnOutcome conveying stop_reason (no_tool_calls / tool_rejected / stuck), the final assistant message, and step count. turn() does not emit TurnBegin/TurnEnd wire framing; callers frame the turn themselves, as run() does. turn() is a thin delegate to _turn on purpose: many tests monkeypatch soul._turn to stub turn execution, so _turn stays the single implementation/patch point and those patches keep intercepting turns started through turn(). The three external call sites now use turn() with the suppressions removed; internal self._turn calls are unchanged, and the runtime_checkable Soul protocol is deliberately untouched. --- src/pythinker_code/soul/flow_runner.py | 2 +- src/pythinker_code/soul/pythinkersoul.py | 33 +++++++++++++++++++ src/pythinker_code/soul/slash.py | 8 ++--- tests/core/test_public_turn.py | 40 ++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 tests/core/test_public_turn.py diff --git a/src/pythinker_code/soul/flow_runner.py b/src/pythinker_code/soul/flow_runner.py index aea780b2..0622c381 100644 --- a/src/pythinker_code/soul/flow_runner.py +++ b/src/pythinker_code/soul/flow_runner.py @@ -209,7 +209,7 @@ async def _flow_turn( ) -> TurnOutcome: wire_send(TurnBegin(user_input=prompt)) try: - res = await soul._turn(Message(role="user", content=prompt)) # type: ignore[reportPrivateUsage] + res = await soul.turn(Message(role="user", content=prompt)) finally: wire_send(TurnEnd()) return res diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index 578b56dd..d33fc4ed 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -1141,6 +1141,39 @@ async def _run_goal_continuations(self, primary_outcome: TurnOutcome) -> None: if outcome.stop_reason != "no_tool_calls": return + async def turn(self, user_message: Message) -> TurnOutcome: + """ + Run one full agent turn for ``user_message`` and return its outcome. + + The message is appended to the context, then the agent loop steps the + model — executing any tool calls it makes — until the model stops. + The returned ``TurnOutcome`` carries the ``stop_reason`` + (``"no_tool_calls"``: the model finished with a plain response; + ``"tool_rejected"``: the user rejected a tool call; ``"stuck"``: the + turn was cut short as a degenerate tool-call loop), the final + assistant message (``None`` when a tool call was rejected), and the + number of steps taken. + + This is the public entry point for external drivers (flows, slash + command handlers). It emits per-step wire framing but NOT + ``TurnBegin``/``TurnEnd``. Callers starting a new wire-level turn + (e.g. ``FlowRunner``) must wrap the call in ``TurnBegin``/``TurnEnd``; + callers already executing inside a framed turn (e.g. slash-command + handlers running under ``run()``) must not add extra framing. + + Raises: + LLMNotSet: When the LLM is not set. + LLMNotSupported: When the LLM does not have required capabilities. + ChatProviderError: When the LLM provider returns an error. + MaxStepsReached: When the per-turn step limit is reached. + asyncio.CancelledError: When the turn is cancelled by user. + """ + # Thin delegate by design: many tests monkeypatch ``soul._turn`` to + # stub turn execution, so ``_turn`` must remain the single + # implementation/patch point that both ``turn()`` and internal + # callers go through. + return await self._turn(user_message) + async def _turn(self, user_message: Message) -> TurnOutcome: from pythinker_code.extensions import shared_event_bus from pythinker_code.telemetry import metrics as _m diff --git a/src/pythinker_code/soul/slash.py b/src/pythinker_code/soul/slash.py index f5b20aa1..41002dd0 100644 --- a/src/pythinker_code/soul/slash.py +++ b/src/pythinker_code/soul/slash.py @@ -282,9 +282,7 @@ async def goal(soul: PythinkerSoul, args: str): "with /goal clear." ) ) - await soul._turn( # pyright: ignore[reportPrivateUsage] - Message(role="user", content=prompts.GOAL_SET.format(objective=text)) - ) + await soul.turn(Message(role="user", content=prompts.GOAL_SET.format(objective=text))) @registry.command @@ -297,9 +295,7 @@ async def learn(soul: PythinkerSoul, args: str): else "No specific focus was given; review the whole session." ) wire_send(TextPart(text="Reviewing the session for lessons worth keeping...")) - await soul._turn( # pyright: ignore[reportPrivateUsage] - Message(role="user", content=prompts.LEARN.format(focus=focus_line)) - ) + await soul.turn(Message(role="user", content=prompts.LEARN.format(focus=focus_line))) @registry.command(name="best-practices", aliases=["bp"]) diff --git a/tests/core/test_public_turn.py b/tests/core/test_public_turn.py new file mode 100644 index 00000000..50699508 --- /dev/null +++ b/tests/core/test_public_turn.py @@ -0,0 +1,40 @@ +"""Tests for the public ``PythinkerSoul.turn()`` contract.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock + +from pythinker_core.message import Message +from pythinker_core.tooling.empty import EmptyToolset + +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul, TurnOutcome + + +def _make_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + return PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + + +async def test_turn_delegates_to_patched_private_turn(runtime: Runtime, tmp_path: Path) -> None: + """``turn()`` must delegate to ``self._turn`` so tests patching ``_turn`` + intercept turns started through the public entry point too.""" + soul = _make_soul(runtime, tmp_path) + sentinel = TurnOutcome(stop_reason="no_tool_calls", final_message=None, step_count=1) + turn_mock = AsyncMock(return_value=sentinel) + soul._turn = turn_mock # type: ignore[method-assign] + + message = Message(role="user", content="hello") + outcome = await soul.turn(message) + + assert turn_mock.await_count == 1 + assert turn_mock.await_args is not None + assert turn_mock.await_args.args[0] is message + assert outcome is sentinel From eaa96d6e7165c2bc56f88e56b1c6e987262afd3b Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Fri, 12 Jun 2026 00:22:23 -0400 Subject: [PATCH 03/49] refactor(soul): replace string-matched tool gates with declarative flags Two gates identified special tool classes by mechanisms that break invisibly on rename/move: - check_tool_call_allowed (permission.py) and _is_external_side_effect_tool (toolset.py) matched external adapters (MCPTool, WireExternalTool, PluginTool) by module/qualname strings. This is a permission gate that FAILS OPEN: moving or renaming any of these classes silently drops them out of external-tool permission gating with no test failure. A planned refactor moves MCPTool out of toolset.py, so the trap is defused first. - _tool_defers_execution_started duck-typed on the private _approval attribute to decide whether ToolExecutionStarted is deferred until after approval. Both gates now read explicit class-level flags instead: - external_side_effect_tool (ClassVar) on MCPTool, WireExternalTool, and PluginTool, documented as a security contract and pinned by tests so a future move/rename that loses the flag fails CI instead of failing open. - emits_tool_execution_started_after_approval on every approval-gated tool class (Shell, WriteFile, StrReplaceFile, TaskInput, TaskStop, Terminal, PluginTool), matching the existing RunAgentsTool precedent; the hasattr(_approval) fallback is removed. Routing is behavior-preserving: the same tool classes pass through the same gates before and after, and the Shell/network branches keep priority in check_tool_call_allowed. --- src/pythinker_code/acp/tools.py | 2 + src/pythinker_code/plugin/tool.py | 14 +++- src/pythinker_code/soul/permission.py | 10 +-- src/pythinker_code/soul/toolset.py | 43 +++++++---- .../tools/background/__init__.py | 2 + src/pythinker_code/tools/file/replace.py | 1 + src/pythinker_code/tools/file/write.py | 1 + src/pythinker_code/tools/shell/__init__.py | 1 + tests/core/test_permission_profiles.py | 77 +++++++++++++++++++ tests/core/test_toolset.py | 49 ++++++++++++ 10 files changed, 180 insertions(+), 20 deletions(-) diff --git a/src/pythinker_code/acp/tools.py b/src/pythinker_code/acp/tools.py index 37ac9716..1a21d377 100644 --- a/src/pythinker_code/acp/tools.py +++ b/src/pythinker_code/acp/tools.py @@ -48,6 +48,8 @@ class HideOutputDisplayBlock(DisplayBlock): class Terminal(CallableTool2[ShellParams]): + emits_tool_execution_started_after_approval = True + def __init__( self, shell_tool: Shell, diff --git a/src/pythinker_code/plugin/tool.py b/src/pythinker_code/plugin/tool.py index 61a184ea..e5ce94b0 100644 --- a/src/pythinker_code/plugin/tool.py +++ b/src/pythinker_code/plugin/tool.py @@ -5,7 +5,7 @@ import asyncio import json from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from loguru import logger from pythinker_core.tooling import CallableTool, ToolError, ToolOk @@ -43,6 +43,18 @@ class PluginTool(CallableTool): (not baked into config files) to handle OAuth token refresh. """ + external_side_effect_tool: ClassVar[bool] = True + """Marks tool adapters whose side effects cannot be statically classified. + + Consumed by the permission guard in ``permission.check_tool_call_allowed`` + — which routes flagged tools through ``check_external_tool_allowed`` — and + by profile-gated tool visibility filtering + (``PythinkerToolset._is_tool_visible``). Removing or failing to set this + flag on an external adapter disables its permission gating. + """ + + emits_tool_execution_started_after_approval: ClassVar[bool] = True + def __init__( self, tool_spec: PluginToolSpec, diff --git a/src/pythinker_code/soul/permission.py b/src/pythinker_code/soul/permission.py index 93550770..73660b09 100644 --- a/src/pythinker_code/soul/permission.py +++ b/src/pythinker_code/soul/permission.py @@ -418,12 +418,10 @@ def check_tool_call_allowed( if tool_name in _NETWORK_TOOLS: return check_network_tool_allowed(runtime, tool_name) - tool_type = type(tool) - module = getattr(tool_type, "__module__", "") - qualname = getattr(tool_type, "__qualname__", "") - if module == "pythinker_code.plugin.tool" and qualname.endswith("PluginTool"): - return check_external_tool_allowed(runtime, tool_name) - if module == "pythinker_code.soul.toolset" and qualname in {"MCPTool", "WireExternalTool"}: + # Declarative flag set by external adapters (MCPTool, WireExternalTool, + # PluginTool) whose side effects cannot be statically classified; see the + # `external_side_effect_tool` declarations for the fail-closed contract. + if getattr(tool, "external_side_effect_tool", False): return check_external_tool_allowed(runtime, tool_name) return None diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index 95a66ac6..2aae5fc0 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from datetime import timedelta from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, cast, overload +from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast, overload from pythinker_core.tooling import ( CallableTool, @@ -123,21 +123,18 @@ def emit_current_tool_execution_started() -> None: def _tool_defers_execution_started(tool: ToolType) -> bool: - return bool( - getattr(tool, "emits_tool_execution_started_after_approval", False) - or hasattr(tool, "_approval") - ) + return bool(getattr(tool, "emits_tool_execution_started_after_approval", False)) def _is_external_side_effect_tool(tool: ToolType) -> bool: - """Return True for tool adapters whose side effects are not statically classified.""" - tool_type = type(tool) - module = getattr(tool_type, "__module__", "") - qualname = getattr(tool_type, "__qualname__", "") - return bool( - (module == "pythinker_code.plugin.tool" and qualname.endswith("PluginTool")) - or (module == "pythinker_code.soul.toolset" and qualname in {"MCPTool", "WireExternalTool"}) - ) + """Return True for tool adapters whose side effects are not statically classified. + + Reads the declarative ``external_side_effect_tool`` class flag (declared on + ``MCPTool``, ``WireExternalTool``, and ``PluginTool``) instead of matching + module/qualname strings, so a moved or renamed adapter cannot silently fall + out of the side-effect classification. + """ + return bool(getattr(tool, "external_side_effect_tool", False)) def _mcp_stderr_log_path(runtime: Runtime, server_name: str) -> Path: @@ -1098,6 +1095,16 @@ class MCPServerInfo: class MCPTool[T: ClientTransport](CallableTool): + external_side_effect_tool: ClassVar[bool] = True + """Marks tool adapters whose side effects cannot be statically classified. + + Consumed by the permission guard in ``permission.check_tool_call_allowed`` + — which routes flagged tools through ``check_external_tool_allowed`` — and + by profile-gated tool visibility filtering + (``PythinkerToolset._is_tool_visible``). Removing or failing to set this + flag on an external adapter disables its permission gating. + """ + def __init__( self, server_name: str, @@ -1198,6 +1205,16 @@ async def __call__(self, *args: Any, **kwargs: Any) -> ToolReturnValue: class WireExternalTool(CallableTool): + external_side_effect_tool: ClassVar[bool] = True + """Marks tool adapters whose side effects cannot be statically classified. + + Consumed by the permission guard in ``permission.check_tool_call_allowed`` + — which routes flagged tools through ``check_external_tool_allowed`` — and + by profile-gated tool visibility filtering + (``PythinkerToolset._is_tool_visible``). Removing or failing to set this + flag on an external adapter disables its permission gating. + """ + def __init__(self, *, name: str, description: str, parameters: dict[str, Any]) -> None: super().__init__( name=name, diff --git a/src/pythinker_code/tools/background/__init__.py b/src/pythinker_code/tools/background/__init__.py index 73aa71fd..ea76928c 100644 --- a/src/pythinker_code/tools/background/__init__.py +++ b/src/pythinker_code/tools/background/__init__.py @@ -469,6 +469,7 @@ class TaskInput(CallableTool2[TaskInputParams]): name: str = "TaskInput" description: str = load_desc(Path(__file__).parent / "input.md") params: type[TaskInputParams] = TaskInputParams + emits_tool_execution_started_after_approval = True def __init__(self, runtime: Runtime, approval: Approval): super().__init__() @@ -549,6 +550,7 @@ class TaskStop(CallableTool2[TaskStopParams]): name: str = "TaskStop" description: str = load_desc(Path(__file__).parent / "stop.md") params: type[TaskStopParams] = TaskStopParams + emits_tool_execution_started_after_approval = True def __init__(self, runtime: Runtime, approval: Approval): super().__init__() diff --git a/src/pythinker_code/tools/file/replace.py b/src/pythinker_code/tools/file/replace.py index 086d3566..01d2bfe6 100644 --- a/src/pythinker_code/tools/file/replace.py +++ b/src/pythinker_code/tools/file/replace.py @@ -128,6 +128,7 @@ class StrReplaceFile(CallableTool2[Params]): name: str = "StrReplaceFile" description: str = _BASE_DESCRIPTION params: type[Params] = Params + emits_tool_execution_started_after_approval = True def __init__(self, runtime: Runtime, approval: Approval): super().__init__() diff --git a/src/pythinker_code/tools/file/write.py b/src/pythinker_code/tools/file/write.py index f1fd89ca..f291e9ac 100644 --- a/src/pythinker_code/tools/file/write.py +++ b/src/pythinker_code/tools/file/write.py @@ -43,6 +43,7 @@ class WriteFile(CallableTool2[Params]): name: str = "WriteFile" description: str = _BASE_DESCRIPTION params: type[Params] = Params + emits_tool_execution_started_after_approval = True def __init__(self, runtime: Runtime, approval: Approval): super().__init__() diff --git a/src/pythinker_code/tools/shell/__init__.py b/src/pythinker_code/tools/shell/__init__.py index a8655a83..adae1a44 100644 --- a/src/pythinker_code/tools/shell/__init__.py +++ b/src/pythinker_code/tools/shell/__init__.py @@ -84,6 +84,7 @@ def _validate_background_fields(self) -> Self: class Shell(CallableTool2[Params]): name: str = "Shell" params: type[Params] = Params + emits_tool_execution_started_after_approval = True def __init__(self, approval: Approval, environment: Environment, runtime: Runtime): is_powershell = environment.shell_name == "Windows PowerShell" diff --git a/tests/core/test_permission_profiles.py b/tests/core/test_permission_profiles.py index f3847329..beebd05a 100644 --- a/tests/core/test_permission_profiles.py +++ b/tests/core/test_permission_profiles.py @@ -652,6 +652,83 @@ async def test_toolset_denies_plugin_tool_in_read_only_profile( assert "permission profile blocks external tool" in result.return_value.message +def test_external_adapters_declare_side_effect_flag() -> None: + """Security contract: ``check_tool_call_allowed`` identifies external tool + adapters via the ``external_side_effect_tool`` class flag. If a future + move/rename drops the flag, the permission gate FAILS OPEN — these pins turn + that into a test failure instead of a silent bypass.""" + from pythinker_code.plugin.tool import PluginTool + from pythinker_code.soul.toolset import MCPTool, WireExternalTool + + assert MCPTool.external_side_effect_tool is True + assert WireExternalTool.external_side_effect_tool is True + assert PluginTool.external_side_effect_tool is True + + +async def test_check_tool_call_allowed_routes_external_adapters( + runtime: Runtime, + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Each external adapter (plugin/MCP/wire) must be routed through + ``check_external_tool_allowed``; unflagged builtins must not be.""" + from types import SimpleNamespace + from typing import Any, cast + + import mcp + + from pythinker_code.plugin import PluginToolSpec + from pythinker_code.plugin.tool import PluginTool + from pythinker_code.soul import permission + from pythinker_code.soul.toolset import MCPTool, WireExternalTool + + sentinel = permission.ToolError(message="external gate consulted", brief="sentinel") + consulted: list[str] = [] + + def fake_external_gate(rt: Runtime, tool_name: str) -> permission.ToolError: + consulted.append(tool_name) + return sentinel + + monkeypatch.setattr(permission, "check_external_tool_allowed", fake_external_gate) + + plugin_dir = tmp_path / "plugin" + plugin_dir.mkdir() + plugin_tool = PluginTool( + PluginToolSpec(name="plugin_tool", description="test", command=["true"]), + plugin_dir=plugin_dir, + inject={}, + config=runtime.config, + ) + wire_tool = WireExternalTool(name="wire_tool", description="test", parameters={}) + mcp_tool = MCPTool( + "srv", + mcp.Tool( + name="mcp_tool", description="x", inputSchema={"type": "object", "properties": {}} + ), + cast(Any, SimpleNamespace()), + runtime=runtime, + ) + + for tool in (plugin_tool, wire_tool, mcp_tool): + assert permission.check_tool_call_allowed(runtime, tool.name, {}, tool=tool) is sentinel + assert consulted == ["plugin_tool", "wire_tool", "mcp_tool"] + + # Negative control: an unflagged builtin must NOT consult the external gate. + consulted.clear() + write_file = WriteFile(runtime, Approval(yolo=True)) + assert permission.check_tool_call_allowed(runtime, write_file.name, {}, tool=write_file) is None + assert consulted == [] + + # Ordering pin: the Shell branch stays first — a Shell call is classified by + # the shell gate even if the tool object carries the external flag. + flagged_shell_stub = SimpleNamespace(external_side_effect_tool=True) + result = permission.check_tool_call_allowed( + runtime, "Shell", {"command": "ls"}, tool=flagged_shell_stub + ) + assert result is not sentinel + assert consulted == [] + + @pytest.mark.skipif( platform.system() == "Windows", reason="Shell mutation guard examples use POSIX" ) diff --git a/tests/core/test_toolset.py b/tests/core/test_toolset.py index 882bf15d..0b31be27 100644 --- a/tests/core/test_toolset.py +++ b/tests/core/test_toolset.py @@ -542,3 +542,52 @@ async def test_cross_step_dedup_not_triggered_after_back_to_the_future(): # Should NOT have the cross-step reminder appended assert tr.return_value.output == "a" assert ts.dedup_triggered is False + + +# --- execution-started deferral --- + + +def test_approval_gated_tools_declare_deferred_execution_started() -> None: + """Every tool that requests approval mid-call must carry the explicit flag. + + ``_tool_defers_execution_started`` used to duck-type on the private + ``_approval`` attribute; the explicit + ``emits_tool_execution_started_after_approval`` flag is now the only + signal, so each approval-gated tool class must declare it or + ``ToolExecutionStarted`` fires before approval resolves. + """ + from pythinker_code.acp.tools import Terminal + from pythinker_code.plugin.tool import PluginTool + from pythinker_code.tools.agent import RunAgents + from pythinker_code.tools.background import TaskInput, TaskStop + from pythinker_code.tools.file.replace import StrReplaceFile + from pythinker_code.tools.file.write import WriteFile + from pythinker_code.tools.shell import Shell + + approval_gated = ( + Shell, + WriteFile, + StrReplaceFile, + TaskInput, + TaskStop, + Terminal, + PluginTool, + RunAgents, + ) + for tool_class in approval_gated: + assert tool_class.emits_tool_execution_started_after_approval is True, tool_class + + +def test_tool_defers_execution_started_reads_flag_only() -> None: + from pythinker_code.soul.toolset import _tool_defers_execution_started + + flagged = SimpleNamespace(emits_tool_execution_started_after_approval=True) + assert _tool_defers_execution_started(cast(Any, flagged)) is True + + unflagged = SimpleNamespace() + assert _tool_defers_execution_started(cast(Any, unflagged)) is False + + # A private `_approval` attribute alone must no longer defer the event; + # the explicit flag is the single contract. + approval_only = SimpleNamespace(_approval=object()) + assert _tool_defers_execution_started(cast(Any, approval_only)) is False From aba2c1388cd49187af8cf8d778161c1930158aeb Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Fri, 12 Jun 2026 00:48:52 -0400 Subject: [PATCH 04/49] docs(tasks): record design adoption blueprint and branch task log Add the verified design-adoption blueprint (ranked, review-caveated refactor plan for cleaner agent/runtime layering) and update the task log with this branch's three landed tasks, their review outcomes, and the deferred follow-ups, including the known machine-local PTY shell-cancel test failure verified pre-existing on main. --- tasks/design-adoption-blueprint.md | 179 +++++++++++++++++++++++++++++ tasks/todo.md | 53 ++++++++- 2 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 tasks/design-adoption-blueprint.md diff --git a/tasks/design-adoption-blueprint.md b/tasks/design-adoption-blueprint.md new file mode 100644 index 00000000..aeb68bd2 --- /dev/null +++ b/tasks/design-adoption-blueprint.md @@ -0,0 +1,179 @@ +# Design Adoption Blueprint — cleaner layering for pythinker + +Source: multi-agent architecture study (12 subsystem maps, 2 architect lenses, adversarial +verification per recommendation) comparing pythinker against a cleanly layered reference +agent harness (local clone under `blackbox/`, gitignored). All recommendations below +survived adversarial verification against both codebases. Each is independently landable +and behavior-preserving unless flagged. + +## Diagnosis + +Pythinker's runtime behavior is fundamentally sound — typed wire events, fail-closed +permission gates, persisted-everything subagents, exemplary ContextVar discipline. The +structural debt is **concentrated, not diffuse**: + +- Three god modules fuse loop, policy, and integration glue: + `soul/pythinkersoul.py` (2390 lines), `soul/permission.py` (1377), `soul/toolset.py` (1330). +- `soul` ↔ `tools` import each other in both directions, patched with dozens of + function-local imports. +- Core reaches into UI in two places: `soul/toolset.py:944` (toast import) and + `tools/usage.py` pricing display. +- `cli/__init__.py` (~700-line typer callback) + `PythinkerCLI.create` (~290 lines) + fuse a dozen lifecycle concerns. +- Naming requires tribal knowledge: `skill/` (code) vs `skills/` (data), + `agents/` (data) vs `subagents/` (code). +- The "turn" boundary is private: `flow_runner.py` and `slash.py` call `soul._turn` + with pyright suppressions. + +Reference design moves worth adopting (the *style*, not the language idioms): + +1. **Functional core, stateful shell** — pure loop functions over (context, config, + emit, signal); a stateful class owns transcript, queues, run lifecycle. +2. **Everything observable is an event** — internal state is updated by reducing the + same event stream UIs subscribe to, so views can't diverge. +3. **One contracts surface** — closed event union + hook contracts with documented + must-not-throw invariants in one types module that reads as a spec. +4. **Declarative metadata on definitions** — per-tool / per-agent-type capability + metadata on the definition object, never string-matching on names/modules. +5. **Layered construction** — services (cwd-bound deps) → session → runtime factory, + with **diagnostics returned as data** (app layer decides what's fatal). +6. **Tools own their full behavior contract** — schema, limits interpolated into the + LLM-facing description from the same constants the code enforces, truncation that + always names the next action, self-described UI metadata. +7. **Tiny system prompt** — capability comes from tools/repo files/lazy skill indexes; + code enforces mechanics, the prompt encodes only judgment. + +## Ranked recommendations (verified) + +### P1a (L) Split PythinkerSoul into stateful shell + extracted loop/recovery/compaction modules +Extract from `soul/pythinkersoul.py`: turn-loop sequencing, connection/OAuth recovery, +and compaction orchestration into sibling modules; the soul keeps state + lifecycle. +- Cheap first slice: recovery extraction (two methods already static; rest need only + `_runtime`/`_current_step_no`). +- Couplings: 11 test files (39 hits) monkeypatch `soul._step`/`_agent_loop`/`_turn` — + keep thin delegating methods or budget test migration. `_agent_loop`/`_step` touch + ~15 `self` members; expect file-stratum readability, not full purity. +- No circular imports (slash.py imports the soul TYPE_CHECKING-only); no wire-snapshot churn. + +### P1b (L) Public turn contract + functional-core/stateful-shell loop split +Step 1 (low-risk): make `turn()` public so `flow_runner.py:212` and `slash.py:285,300` +stop calling a private with pyright suppressions. Define nesting semantics for the +ralph path (FlowRunner currently nests TurnBegin inside `run()`'s framing). +Step 2 (genuinely L): hoist `_step`'s pure parts; the deps surface includes sleep +inhibitor, telemetry, event bus, hook engine, `_settle_shielded` cancellation — design +the seam around those. ~20 test files patch loop internals; reroute deliberately. + +### P2a (M) Split toolset.py: dispatch pipeline / dedup state machine / MCP module; remove core→UI toast +- **Security-critical coupling**: `permission.py:426` and `toolset.py:139` string-match + `module == 'pythinker_code.soul.toolset'` + qualname `MCPTool`/`WireExternalTool`. + Moving MCPTool without updating both **fails OPEN on a permission gate**. Fix the + detection to declarative flags as part of the move (see P3a). +- MCP lifecycle is entangled with toolset instance state (`_mcp_servers`, + `_register_mcp_tools`, `runtime.mcp_tools`) → extract as a delegate class, not a file + move; ~10 call sites in pythinkersoul/agent need a facade. +- Toast removal: notification hub + shell toast subscription (`ui/shell/__init__.py:856`) + and `StatusUpdate.mcp_status` already exist as the right channel. + +### P2b (L, two PRs) Per-tool-call pipeline: prepare / execute / finalize with discriminated outcomes +Restructure `toolset.handle` (toolset.py:514-773, ~260 lines, three nested closures). +- `handle()` must stay sync per the core Toolset protocol → prepare's permission/PreToolUse + stages run inside the created task. +- Cancellation contract (toolset.py:759-767): finalize must run in the same task, never a + wrapper task; the same-step dedup join is an awaiting Task, not an immediate outcome. + +### P3a (M) Tools as a real layer below soul: neutral contracts module + declarative per-tool metadata +Kill string matching: `extract_key_argument`'s 25-name match (tools/__init__.py:29-120), +`hasattr(tool, '_approval')` duck-typing (toolset.py:125-129), module/qualname matching +(toolset.py:132-140, permission.py:421-427) → flags/metadata on tool definitions. +- `key_argument` consumers (`ui/shell/visualize/_blocks.py:860`, `acp/session.py:109`) + only have wire tool names → needs a name→spec registry, not just a class attribute; + behaviors are pinned by `tests/tools/test_extract_key_argument.py`. +- Riskiest sub-part: Protocol views of Runtime/Approval collide with DI — + `toolset._load_tool` keys deps by exact annotation identity (agent.py:504-515). +- Keep `SkipThisTool` tools-owned (soul→tools is already the correct direction). +- Easy wins first: flags + pricing-display move. +- Precedent: `emits_tool_execution_started_after_approval` (tools/agent/__init__.py:564). + +### P3b (M) Consolidate per-subagent-type behavior onto AgentTypeDefinition +Today scattered: tool policy in `agents/default/*.yaml`, permission floor in +`soul/permission.py:86-101`, summary min-lengths in `subagents/runner.py:45-56`, +explore-only git-context branch in `subagents/core.py:90`. Silent-fallback bug is real +(planner/scout missing from the tables). `tool_policy` and `supports_background` already +live on AgentTypeDefinition — extend that pattern. +- **Security**: do NOT let markdown frontmatter declare `permission_profile` + (project-local agents could self-escalate); keep the read_only fallback and plan-mode + downgrade (permission.py:283-286) applied to the *resolved* profile. +- Couplings: `tests/core/test_permission_profiles.py` (43 tests) uses bare + subagent_type strings with empty labor markets → register type defs in fixtures or + keep a name-table fallback; `test_summary_continuation.py` imports the private tables. + +### P3c (L, hard) Decompose CLI entry: entry → mode resolution → services → runtime composition +Split the ~700-line typer callback (`cli/__init__.py`) and `PythinkerCLI.create` +(app.py:163-453) into layered factories with diagnostics-as-data. +- Corrections from verification: create() does not print inline (stderr pre-redirected + to loguru; warnings surface via run_shell banner) — diagnostics motivation is weaker + here; pythinker never consults TTY for mode selection — do NOT add TTY sniffing. +- Hidden couplings: Reload/ExitCode/Input-OutputFormat re-exports (~15 importers, + circular-import risk with app.py), test_startup_imports lazy-import pins, flock + release-before-reacquire ordering, per-session attach_sink for multi-session ACP. + +### P4a (M) Compaction/prune orchestration out of the soul: prepare/execute split with one rollback +Duplicated rollback confirmed (pythinkersoul.py:1988-2001 vs 2095-2173) → one shared +`with_context_rewrite()` guard. `SimpleCompaction.prepare()` (compaction.py:194-238) +already supplies the pure plan half. PostCompact/SessionStart hooks fire inside the +guarded rewrite; usage accounting, root-role task snapshot, injection re-arm stay +soul-coupled → runner takes callbacks. Realistic reduction ~150-200 lines. +`test_context_pruning.py` pins rollback behavior (aids verification). + +### P4b (M) Background runner gets a public seam; finish AgentTypeDefinition consolidation +`background/agent_runner.py` is a privacy-violation zone (file-wide +reportPrivateUsage=false; imports `_SUMMARY_MIN_LENGTH_BY_TYPE`; pokes +`manager._live_agent_tasks`, `_mark_task_running`, `_mark_task_awaiting_approval`). +Promote a narrow interface on the manager. ~15 test sites poke the same privates across +4 test files (incl. tests/background/test_manager.py:545-617) — migrate them with it. + +### P5a (M) Uniform tool result surface: every tool owns status, brief/tail, untrusted wrapping +Older file tools (ReadFile/WriteFile/Glob) return bare ToolOk/ToolError; newer tools use +ToolResultBuilder with `extras.status`. Migrate stragglers. +- **Real bug found**: FetchURL writes pre-wrapped untrusted text into the builder + (web/fetch.py:232,277,338) so truncation cuts the closing `` tag AND + breaks `strip_untrusted_envelope` (endswith check) → envelope leaks to display. + Fix by switching to `mark_untrusted()` (wrap after truncation). +- Byte-identical ReadFile migration needs `max_line_length=None` + raised max_chars + (100KB cap vs builder's 50K default; marker text differs). +- Pins: `test_untrusted_wrapping.py` WRAPPER_RE; `test_extract_key_argument.py` + (Bash|Shell alias, path normalization, raw-JSON default must survive). + +### P5b (M) Split skill/__init__.py (872 lines) into named submodules; disambiguate code-vs-data dirs +Submodules: roots/discovery/frontmatter/prompt-rendering/resources. Decide owner of the +Skill model. `skills/` and `agents/` data dirs have no `__init__.py` — prefer a README +in data dirs over making them importable. ~12 tests monkeypatch barrel attributes +(`get_builtin_skills_dir`, `_supports_builtin_skills`, `_SKILL_RESOURCE_SCAN_CEILING` — +tests/core/test_skill.py, tests/tools/test_skill_tool.py:142) → retarget or re-export. + +## Rejected (do not pursue as designed) + +- **Telemetry seam derived from the wire stream**: the wire does not carry the needed + facts (auto/yolo approvals resolve pre-wire; ToolResult lacks duration/error_type; + CompactionEnd fires even on failure; dedup/slash/skill/agent_stuck never hit the wire). + Salvageable kernel: a scoped wire subscriber for turn lifecycle, manual approvals, + StepRetry only. Full inline-`track()` cleanup would require wire-protocol enrichment + (serde back-compat + tests_e2e snapshots + ACP/web clients). + +## Anti-patterns in the reference (do not import) + +- Its own god files (3135-line session class, 5741-line interactive mode) — adopt the + boundaries, not the file sizes. +- Throw-by-default hook errors that fail a user turn after state was committed. +- Stringly-typed event bus channels; last-result-wins multi-handler hook chaining. +- Whole-file session load with no torn-line repair and no multi-writer locking + (pythinker is already ahead here — keep our invariants). +- Magic-string provider detection inside providers; in-place mutation with scratch fields. + +## Suggested landing order + +1. P1b step 1 (public `turn()`) + P3a easy wins (flags, pricing move) + P5a FetchURL bug fix — small, immediate. +2. P1a recovery slice → compaction (P4a) → loop split (P1b step 2). +3. P2a toolset split (with the permission-gate string-match fix) → P2b pipeline. +4. P3b type-def consolidation → P4b background seam. +5. P5b skill split; P3c CLI decomposition last (hardest, most coupled). diff --git a/tasks/todo.md b/tasks/todo.md index df972b25..6768c468 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -2,9 +2,56 @@ ## Active -- [ ] `mythos-enhancements` PR #118: opened; CI test failures fixed (statusline - reap deadlock, diff-marker style assertion). CodeRabbit gate before merge - (first review attempt was rate-limited; re-review triggers on push). +Branch `refactor/agent-contract-and-tool-metadata` — landing step 1 of +`tasks/design-adoption-blueprint.md` (agent-logic/coding-flow cleanup): + +- [x] Task 1: FetchURL untrusted-envelope fix — DONE (ceeeeb78; 3 TDD tests, + spec + quality review approved). Deferred: add `await + builder.spill_to_disk()` at the trafilatura + fetch-service sites for + event-loop hygiene (pre-existing asymmetry; no output difference). + Original: `tools/web/fetch.py:232,277,338` + write pre-rendered `UntrustedData(...).render_for_prompt()` into the + builder, so truncation can cut the closing envelope tag and break + `strip_untrusted_envelope` (endswith). Switch to raw write + + `builder.mark_untrusted()` (idiom: `tools/web/search.py:174-177`). + → verify: new TDD test reproducing tag truncation fails before / passes + after; `tests/utils` untrusted-wrapping suite green. +- [x] Task 2: Public `turn()` contract on PythinkerSoul — DONE (7e96dfb8; + thin delegate keeps `_turn` as the test patch point; 3 call sites + migrated, suppressions removed; contract docstring distinguishes + framed vs unframed callers; spec + quality review approved). +- [x] Task 3: Declarative tool metadata for the dispatch/permission gates — + DONE (eaa96d6e; `external_side_effect_tool` ClassVar on the 3 adapters, + `emits_tool_execution_started_after_approval` pinned on 8 classes, + both string-match consumers rewritten; spec review, quality review, + and security review (SAFE TO MERGE) all passed). + +Review: branch `refactor/agent-contract-and-tool-metadata` (3 commits on top +of bff54f94) final-reviewed READY TO MERGE. Full `make test-pythinker-code`: +5279 passed, 1 failed — the failure is +`tests/e2e/test_shell_pty_e2e.py::test_shell_cancel_running_command_kills_process_and_recovers`, +verified PRE-EXISTING and machine-local: fails identically in isolation on +main (bff54f94) and on 7caeca33 / d51ef649 / 2904de00, all of which merged +with green CI. It is the ONLY test that sends ESC, so the local ESC-interrupt +PTY path has no corroborating coverage. Needs its own debugging session +(suspect ESC flush timing in the local macOS/Python 3.14 PTY environment). + +Out of scope this PR (logged): pricing display move out of core, the +`extract_key_argument` name→spec registry, larger soul/toolset splits +(blueprint P1a/P2a/P2b). + +### Deferred from this branch's reviews +- `await builder.spill_to_disk()` missing at FetchURL trafilatura + + fetch-service sites (pre-existing; sync-spill fallback is correct, just + blocks the event loop briefly). +- No structural enforcement that FUTURE external adapters declare + `external_side_effect_tool` (pin tests cover the current three only); + consider an `__init_subclass__` check or lint rule. +- MCPTool fires ToolExecutionStarted before its approval dialog resolves + (no `emits_tool_execution_started_after_approval`); pre-existing event + ordering inconsistency vs Shell/WriteFile — decide deliberately. + +Done: `mythos-enhancements` PR #118 merged (d51ef649). ### Deferred (documented, not silently dropped) From 047a0b29350a221c6e344fd3383d3006031364c5 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Fri, 12 Jun 2026 04:51:47 -0400 Subject: [PATCH 05/49] feat: sharpen orchestration guidance and genericize design comments Add a root-only OrchestrationInjectionProvider that nudges substantial normal-mode tasks toward the lightest effective work shape (direct tools, SetTodoList, foreground RunAgents, verification), throttled via a history-scanned reminder marker and suppressed under plan/auto/goal/subagent modes. Sharpen the matching system-prompt guidance, refresh a feature tip to promote /goal, and rephrase design-source comment attributions as generic agent-enhancement notes. --- CHANGELOG.md | 1 + README.md | 2 +- src/pythinker_code/agents/default/system.md | 2 +- src/pythinker_code/llm.py | 2 +- .../soul/dynamic_injections/model_defense.py | 4 +- .../soul/dynamic_injections/orchestration.py | 129 ++++++++++++++ src/pythinker_code/soul/pythinkersoul.py | 13 +- src/pythinker_code/thinking.py | 4 +- src/pythinker_code/tools/todo/__init__.py | 2 +- .../ui/shell/components/bash_execution.py | 2 +- .../ui/shell/components/markdown.py | 4 +- .../ui/shell/components/render_utils.py | 2 +- src/pythinker_code/ui/shell/prompt.py | 2 +- src/pythinker_code/ui/shell/stats_pricing.py | 2 +- src/pythinker_code/ui/shell/tips.py | 2 +- tasks/agent-enhancement-remaining-plan.md | 4 +- tests/core/test_auto_injection.py | 2 +- tests/core/test_goal_auto_continuation.py | 2 +- .../test_orchestration_injection_provider.py | 166 ++++++++++++++++++ tests/core/test_skip_auto_prompt_injection.py | 4 +- tests/tools/test_todo.py | 7 +- tests/ui_and_conv/test_prompt_tips.py | 33 +++- tests/ui_and_conv/test_tui_components.py | 2 +- 23 files changed, 358 insertions(+), 35 deletions(-) create mode 100644 src/pythinker_code/soul/dynamic_injections/orchestration.py create mode 100644 tests/core/test_orchestration_injection_provider.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cdd4856e..462cd47d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Agent orchestration guidance is sharper for substantial tasks.** The default prompt now sharpens work-shaping guidance, and a new root-only runtime reminder nudges substantial normal-mode tasks toward the lightest effective path — direct tools, `SetTodoList`, foreground `RunAgents`, or verification — while backing off for plan mode, `/goal`, auto mode, and subagents. - **The in-app updater recovers from Homebrew's untrusted-tap refusal.** Homebrew 5.0 (`HOMEBREW_REQUIRE_TAP_TRUST`) refuses to load formulas from third-party taps until `brew trust ` is run once, which made the in-app `brew upgrade` fail with only a generic "run manually" hint. The updater now detects both the hard `Refusing to load … from untrusted tap` refusal and the soft `Skipping … not trusted` warning, offers to run `brew trust pythoughts-labs/pythinker` and retry the upgrade once on an interactive terminal, and otherwise prints the exact remediation. It also catches the silent no-op where an untrusted tap is skipped during `brew update` and `brew upgrade` exits 0 without advancing the version. - **Session exports redact secrets surfaced by tool output.** A tool result (e.g. `grep`/`cat` over a `.env`) could write a secret value into an exported transcript in plaintext. `/export` now redacts the value of secret-named keys (`password`, `token`, `api_key`, `secret`, …) to `[REDACTED]` in both the markdown and YAML formats, while leaving non-secret keys such as `token_count`, usernames, and ports intact. - **The welcome logo's antenna blinks a fixed number of times on launch, then settles.** Replaces the terminal's indefinite slow-blink with a bounded boot animation — the antenna ball blinks seven times after the banner prints and then holds steady. It is skipped under reduced motion, on non-interactive output, and when the terminal is too short to keep the antenna row on screen. diff --git a/README.md b/README.md index 85318722..a0480d70 100644 --- a/README.md +++ b/README.md @@ -562,7 +562,7 @@ Pythinker loads [Model Context Protocol](https://modelcontextprotocol.io/) tools ### 🛠️ Manage persistent MCP servers ```sh -# 📚 Context7 stdio server (Codex-style: NAME -- COMMAND) +# 📚 Context7 stdio server (positional form: NAME -- COMMAND) pythinker mcp add context7 -- npx -y @upstash/context7-mcp --api-key YOUR-API-KEY # Added MCP server 'context7' to ~/.pythinker/mcp.json diff --git a/src/pythinker_code/agents/default/system.md b/src/pythinker_code/agents/default/system.md index 5b2359a6..477db54e 100644 --- a/src/pythinker_code/agents/default/system.md +++ b/src/pythinker_code/agents/default/system.md @@ -105,7 +105,7 @@ For research or multimedia tasks (images, video, PDFs, docs, spreadsheets, prese **Act with tools; prose is not action.** Code that appears only in your reply is not saved — use `WriteFile` to create or overwrite, `StrReplaceFile` to edit, `Shell` to run and verify; iterate on failures. Follow each tool's parameter spec exactly. Don't narrate routine tool calls. Do not re-read a file after a successful edit tool call. -**Parallelize.** Before every tool response, ask whether another independent read/search/check can run in the same turn — you may emit any number of tool calls in one response; batch non-interfering calls. Serializing independent operations wastes time and grows context. This is very important to your performance. +**Parallelize.** Before every tool response, ask whether another independent read/search/check can run in the same turn — you may emit any number of tool calls in one response; batch non-interfering calls. Choose the lightest effective work shape: direct tools for known-path checks, `SetTodoList` once a substantial approach is clear, foreground `RunAgents` when independent children feed immediate synthesis, and background agents only when you can make other progress while they run. Serializing independent operations wastes time and grows context. This is very important to your performance. **Spend context deliberately.** The context window is a finite budget: read targeted ranges instead of whole files when the region is known, distill long command output to what the task needs, and push bulky exploration into subagents that return summaries rather than raw dumps. diff --git a/src/pythinker_code/llm.py b/src/pythinker_code/llm.py index 7616f089..9d1c6b3f 100644 --- a/src/pythinker_code/llm.py +++ b/src/pythinker_code/llm.py @@ -350,7 +350,7 @@ def create_llm( elif supports_thinking: effective_effort = requested_effort else: - # Match pi-main's clamp-to-model behavior: non-reasoning models have + # Clamp to the model's supported levels: non-reasoning models have # only the off level, so explicit non-off requests become off instead of # being recorded as active but ignored by the provider. effective_effort = "off" if requested_effort is not None else None diff --git a/src/pythinker_code/soul/dynamic_injections/model_defense.py b/src/pythinker_code/soul/dynamic_injections/model_defense.py index 54428d4c..1b774dfd 100644 --- a/src/pythinker_code/soul/dynamic_injections/model_defense.py +++ b/src/pythinker_code/soul/dynamic_injections/model_defense.py @@ -34,8 +34,8 @@ class ModelDefenseFragment: """A model-family-keyed defense fragment. ``patterns`` and ``excludes`` are case-insensitive substrings matched against - the model name (``excludes`` veto a match — mirrors Kilo's isLing matcher with - excludes). Keep ``content`` short; it is wrapped in a ````. + the model name (``excludes`` veto a match). Keep ``content`` short; it is + wrapped in a ````. """ name: str diff --git a/src/pythinker_code/soul/dynamic_injections/orchestration.py b/src/pythinker_code/soul/dynamic_injections/orchestration.py new file mode 100644 index 00000000..d7515df0 --- /dev/null +++ b/src/pythinker_code/soul/dynamic_injections/orchestration.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import re +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from pythinker_core.message import Message, TextPart + +from pythinker_code.notifications import is_notification_message +from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider +from pythinker_code.soul.message import is_system_reminder_message + +if TYPE_CHECKING: + from pythinker_code.soul.pythinkersoul import PythinkerSoul + +_TURN_INTERVAL = 5 +_INJECTION_TYPE = "orchestration" +_REMINDER_MARKER = "Orchestration reminder:" +_SUBSTANTIAL_TERMS = ( + "implement", + "enhance", + "refactor", + "fix", + "debug", + "review", + "audit", + "orchestration", + "system prompt", + "runtime", + "multiple", + "test", + "tests", + "plan", +) +# Word-boundary matching: bare substring checks misfire on words like +# "prefix" (fix), "explanation" (plan), or "protest" (test). +_SUBSTANTIAL_RE = re.compile( + r"\b(?:" + "|".join(re.escape(term) for term in _SUBSTANTIAL_TERMS) + r")\b", + re.IGNORECASE, +) +_PATHISH_RE = re.compile(r"(?:^|\s)(?:[\w.-]+/)+[\w.-]+") + + +class OrchestrationInjectionProvider(DynamicInjectionProvider): + """Inject sparse orchestration guidance for substantial root tasks. + + Stateless by design: throttling is derived from history (the literal + reminder marker), so it survives restarts and re-arms naturally when + compaction collapses a prior reminder into the summary. + """ + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + if _stronger_mode_active(soul): + return [] + + task_text = _latest_real_user_text(history) + if not task_text or not _looks_substantial(task_text): + return [] + + if not _should_inject(history): + return [] + + return [DynamicInjection(type=_INJECTION_TYPE, content=_reminder())] + + +def _stronger_mode_active(soul: PythinkerSoul) -> bool: + """Defer to modes that carry their own work-shaping guidance. + + Plan mode, auto mode, /goal continuations, and subagent overlays each + inject stronger task framing already; stacking this reminder on top + would dilute them. + """ + if soul.is_subagent or soul.plan_mode or soul.is_auto: + return True + goal = soul.runtime.session.state.goal + return goal is not None and goal.status == "active" + + +def _latest_real_user_text(history: Sequence[Message]) -> str | None: + for message in reversed(history): + if message.role != "user": + continue + if is_notification_message(message) or is_system_reminder_message(message): + continue + text = message.extract_text(" ").strip() + if text: + return text + return None + + +def _looks_substantial(text: str) -> bool: + if _SUBSTANTIAL_RE.search(text): + return True + return len(_PATHISH_RE.findall(text)) >= 2 + + +def _should_inject(history: Sequence[Message]) -> bool: + turns_since_last = 0 + for message in reversed(history): + if message.role == "user" and _is_orchestration_reminder(message): + return turns_since_last >= _TURN_INTERVAL + if message.role == "assistant": + turns_since_last += 1 + return True + + +def _is_orchestration_reminder(message: Message) -> bool: + if message.role != "user": + return False + for part in message.content: + if isinstance(part, TextPart) and _REMINDER_MARKER in part.text: + return True + return False + + +def _reminder() -> str: + return ( + "Orchestration reminder: choose the lightest effective work shape. " + "Use direct tools for known-path or one-file work. Use SetTodoList after " + "the approach is clear for substantial multi-step work. Use foreground " + "RunAgents when independent investigation, review, or verification can run " + "in parallel and your next step is synthesis; use background agents only " + "when you can make other progress while they run. Keep progress updates " + "short and verify with concrete commands before claiming completion." + ) diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index 578b56dd..537d9677 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -79,6 +79,7 @@ from pythinker_code.soul.dynamic_injections.goal_mode import GoalModeInjectionProvider from pythinker_code.soul.dynamic_injections.inline_commands import InlineCommandReminderProvider from pythinker_code.soul.dynamic_injections.model_defense import ModelDefenseInjectionProvider +from pythinker_code.soul.dynamic_injections.orchestration import OrchestrationInjectionProvider from pythinker_code.soul.dynamic_injections.plan_mode import PlanModeInjectionProvider from pythinker_code.soul.flow_runner import FLOW_COMMAND_PREFIX, FlowRunner from pythinker_code.soul.message import ( @@ -206,7 +207,7 @@ def _is_hard_usage_limit(exception: BaseException) -> bool: """Whether a 429 is a subscription usage cap (resets in hours) rather than a transient RPM/TPM burst (clears in seconds). - Hard caps — e.g. ChatGPT Codex ``usage_limit_reached`` — should NOT be retried: + Hard caps — e.g. ChatGPT ``usage_limit_reached`` — should NOT be retried: the backoff just delays the inevitable failure. Detected from the parsed body when present, else from the stringified message (the streaming 429 often carries only the bare text).""" @@ -447,6 +448,9 @@ def __init__( # Self-filtering: root-only; flags inline /command references in the # latest user message that the shell could not have executed. InlineCommandReminderProvider(), + # Self-filtering: root-only; nudges substantial normal-mode tasks toward + # direct tools, todos, RunAgents, and verification. + OrchestrationInjectionProvider(), *( [] if self._runtime.config.skip_auto_prompt_injection @@ -804,9 +808,8 @@ def set_thinking_effort_from_manual(self, effort: ThinkingEffort) -> ThinkingEff """Apply a user-selected thinking level to the live runtime. Returns the effective/clamped level, or ``None`` when no LLM/model is - active. Best-effort persistence mirrors pi-main's settings update, but - a config write failure must not prevent the current session from using - the new level. + active. Persistence is best-effort: a config write failure must not + prevent the current session from using the new level. """ if self._runtime.llm is None or self._runtime.llm.model_config is None: return None @@ -1116,7 +1119,7 @@ async def run( async def _run_goal_continuations(self, primary_outcome: TurnOutcome) -> None: """Auto-continue toward the active /goal after the primary turn. - Ported from Codex CLI's automatic goal continuations, bounded per user + Automatic goal continuations, bounded per user submission by ``goal.max_continuations``. Hard stops (cancellation, MaxStepsReached, provider errors) propagate out of ``_turn`` and end the loop together with the run; a rejected tool call, a stuck turn, or diff --git a/src/pythinker_code/thinking.py b/src/pythinker_code/thinking.py index 0d30f963..b5aa4170 100644 --- a/src/pythinker_code/thinking.py +++ b/src/pythinker_code/thinking.py @@ -1,6 +1,6 @@ """Shared reasoning/thinking effort helpers. -The UI exposes the same provider-neutral effort dial as pi-main. Provider +The UI exposes a provider-neutral effort dial. Provider adapters may map or clamp unsupported levels internally, but callers should preserve the user's requested level in config/session state and pass it through to ``ChatProvider.with_thinking`` when the selected model advertises reasoning @@ -125,7 +125,7 @@ def clamp_thinking_effort( ) -> ThinkingEffort: """Clamp *effort* to the nearest selectable entry in *levels*. - Match pi-main's behavior: if the exact level is unsupported, first search + If the exact level is unsupported, first search upward for a stronger available level, then downward. This keeps requests like ``xhigh`` on high-only models useful without silently disabling thinking. diff --git a/src/pythinker_code/tools/todo/__init__.py b/src/pythinker_code/tools/todo/__init__.py index 33a036ab..7b46cef9 100644 --- a/src/pythinker_code/tools/todo/__init__.py +++ b/src/pythinker_code/tools/todo/__init__.py @@ -116,7 +116,7 @@ async def __call__(self, params: Params) -> ToolReturnValue: ) in_progress = sum(1 for todo in todos if todo.status == "in_progress") if in_progress > 1: - # Codex plan-tool contract, softened: parallel-subagent fan-out + # Single-in-progress discipline, softened: parallel-subagent fan-out # legitimately tracks one in_progress sub-todo per running child. result = _with_appended_note( result, diff --git a/src/pythinker_code/ui/shell/components/bash_execution.py b/src/pythinker_code/ui/shell/components/bash_execution.py index 2ac365d3..d287db77 100644 --- a/src/pythinker_code/ui/shell/components/bash_execution.py +++ b/src/pythinker_code/ui/shell/components/bash_execution.py @@ -2,7 +2,7 @@ -Render bash as a compact Codex-style execution cell: a lifecycle bullet, +Render bash as a compact execution cell: a lifecycle bullet, a ``$ `` header, indented output, and small status/footer hints. We model the same shape as a stateless Rich renderable factory so callers (the bash tool renderer or future ``Shell`` history) can drive it. diff --git a/src/pythinker_code/ui/shell/components/markdown.py b/src/pythinker_code/ui/shell/components/markdown.py index 54f0f060..4094fd2b 100644 --- a/src/pythinker_code/ui/shell/components/markdown.py +++ b/src/pythinker_code/ui/shell/components/markdown.py @@ -237,8 +237,8 @@ def _unwrap_fenced_markdown_tables(markup: str) -> str: """Unwrap ```` ```md ```` fences whose body contains a markdown table. Models sometimes wrap a whole markdown answer — tables included — in a - ``md``/``markdown`` fence, which renders the table as opaque code. Mirror - the Codex heuristic (markdown.rs): only fences explicitly tagged ``md`` or + ``md``/``markdown`` fence, which renders the table as opaque code. Apply + a conservative heuristic: only fences explicitly tagged ``md`` or ``markdown`` *and* containing a header+delimiter pair are unwrapped. Other languages, untagged fences, md fences without tables, and unclosed fences pass through unchanged. diff --git a/src/pythinker_code/ui/shell/components/render_utils.py b/src/pythinker_code/ui/shell/components/render_utils.py index c27ed009..1b8c2935 100644 --- a/src/pythinker_code/ui/shell/components/render_utils.py +++ b/src/pythinker_code/ui/shell/components/render_utils.py @@ -84,7 +84,7 @@ def truncate_middle_to_visual_lines( *, hint: str = "ctrl+o to expand", ) -> VisualTruncateResult: - """Truncate visual lines with a Codex-style head/tail ellipsis in the middle. + """Truncate visual lines with a head/tail ellipsis in the middle. Unlike :func:`truncate_to_visual_lines`, this preserves both early context and the most recent tail. This is better for terminal/tool output where the diff --git a/src/pythinker_code/ui/shell/prompt.py b/src/pythinker_code/ui/shell/prompt.py index 66145bca..de593c0a 100644 --- a/src/pythinker_code/ui/shell/prompt.py +++ b/src/pythinker_code/ui/shell/prompt.py @@ -576,7 +576,7 @@ def _append_footer_hint_fragments( tip_style: str, key_style: str, ) -> None: - """Append toolbar tips with Codex-like key emphasis while preserving plain text.""" + """Append toolbar tips with bold key emphasis while preserving plain text.""" parts = tip_text.split(_TIP_SEPARATOR) for index, part in enumerate(parts): if index: diff --git a/src/pythinker_code/ui/shell/stats_pricing.py b/src/pythinker_code/ui/shell/stats_pricing.py index 3f47b353..278bb776 100644 --- a/src/pythinker_code/ui/shell/stats_pricing.py +++ b/src/pythinker_code/ui/shell/stats_pricing.py @@ -3,7 +3,7 @@ from pythinker_core.chat_provider import TokenUsage # Prices in USD per million tokens. -# Source: Pi's models.generated.ts (blackbox/pi-main/packages/ai/src/models.generated.ts) +# Pricing snapshot from a public multi-provider model catalog; refresh manually when prices change. # Format: {model_id: (input, output, cache_read, cache_write)} _PRICE_TABLE: dict[str, tuple[float, float, float, float]] = { # Anthropic — direct API diff --git a/src/pythinker_code/ui/shell/tips.py b/src/pythinker_code/ui/shell/tips.py index 9a9e345e..ecbea27a 100644 --- a/src/pythinker_code/ui/shell/tips.py +++ b/src/pythinker_code/ui/shell/tips.py @@ -10,7 +10,7 @@ FEATURE_TIPS: Final = ( "Shift+Tab changes thinking effort levels", "Subagents keep your main context clean", - "/verify before declaring work done", + "/goal keeps the agent looping until it's verifiably done", "/learn captures a lesson after a correction", "@-mention files to attach them to the next message", "/feedback sends a note to the Pythinker team", diff --git a/tasks/agent-enhancement-remaining-plan.md b/tasks/agent-enhancement-remaining-plan.md index 00f14264..0c618d5a 100644 --- a/tasks/agent-enhancement-remaining-plan.md +++ b/tasks/agent-enhancement-remaining-plan.md @@ -183,7 +183,7 @@ WS-SOUL queue. A7-first removes the compaction-module half of the collision, not ## 4. config.py cross-branch collision (the live hazard) -The current working tree (`feat/tui-codex-theme`) has **uncommitted `config.py` edits** (theme tokens). +The current working tree (the TUI theme branch) has **uncommitted `config.py` edits** (theme tokens). Three remaining items (ctxmgmt-1, ctxmgmt-2, memory-2) also add `config.py` fields. If both streams edit `config.py` independently they will conflict. @@ -255,7 +255,7 @@ Every item already carries a `Verify.` line in the source plan; the workstream-l 3. **memory-2 posture:** ✅ **Opt-in "durable memory" profile** — do *not* flip defaults; ship a documented profile that enables harvest+journal. Reversible, no default privacy change. 4. **Execution:** ✅ **Start now**, on `feat/agent-phase0-enhancements`, in an isolated git worktree - (the current tree has uncommitted `feat/tui-codex-theme` work that must not be disturbed). + (the current tree has uncommitted TUI theme branch work that must not be disturbed). **Execution order:** WS-FINISH (`injdef-2-grep`) → WS-SOUL non-collision items (`obs-eval-5`, `sysprompt-2`) → then per-collision extract-first (A7→ctxmgmt-2, A3→sysprompt-1) → parallel diff --git a/tests/core/test_auto_injection.py b/tests/core/test_auto_injection.py index 2732a915..f83827e3 100644 --- a/tests/core/test_auto_injection.py +++ b/tests/core/test_auto_injection.py @@ -162,7 +162,7 @@ async def test_rearms_after_context_compaction() -> None: class TestApprovalModeValidationGuidance: - """Codex gpt_5_2_prompt.md:146-150 — validation effort keyed to approval mode.""" + """Validation effort keyed to approval mode.""" def test_auto_prompts_encourage_proactive_validation(self): from pythinker_code.soul.dynamic_injections.auto_mode import ( diff --git a/tests/core/test_goal_auto_continuation.py b/tests/core/test_goal_auto_continuation.py index 36b3c9b3..931be22d 100644 --- a/tests/core/test_goal_auto_continuation.py +++ b/tests/core/test_goal_auto_continuation.py @@ -1,4 +1,4 @@ -"""Tests for the goal auto-continuation loop (Codex goals port).""" +"""Tests for the goal auto-continuation loop.""" from __future__ import annotations diff --git a/tests/core/test_orchestration_injection_provider.py b/tests/core/test_orchestration_injection_provider.py new file mode 100644 index 00000000..d7a5d160 --- /dev/null +++ b/tests/core/test_orchestration_injection_provider.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +from pythinker_core.message import Message, TextPart + +from pythinker_code.session_state import GoalState +from pythinker_code.soul.dynamic_injections.orchestration import OrchestrationInjectionProvider + + +def _user(text: str) -> Message: + return Message(role="user", content=[TextPart(text=text)]) + + +def _assistant(text: str = "step") -> Message: + return Message(role="assistant", content=[TextPart(text=text)]) + + +def _notification() -> Message: + return _user( + 'done' + ) + + +def _system_reminder(text: str = "internal reminder") -> Message: + return _user(f"\n{text}\n") + + +def _make_soul( + *, + is_subagent: bool = False, + plan_mode: bool = False, + is_auto: bool = False, + goal: GoalState | None = None, +) -> MagicMock: + soul = MagicMock() + soul.is_subagent = is_subagent + soul.plan_mode = plan_mode + soul.is_auto = is_auto + soul.runtime.session.state.goal = goal + return soul + + +class TestOrchestrationInjectionProvider: + async def test_injects_for_substantial_root_task(self) -> None: + provider = OrchestrationInjectionProvider() + history = [_user("Enhance system prompts and runtime orchestration with tests")] + + result = await provider.get_injections(history, _make_soul()) + + assert len(result) == 1 + assert result[0].type == "orchestration" + assert "RunAgents" in result[0].content + assert "SetTodoList" in result[0].content + assert "direct tools" in result[0].content + assert "verification" in result[0].content + + async def test_does_not_inject_for_simple_conversation(self) -> None: + provider = OrchestrationInjectionProvider() + + result = await provider.get_injections([_user("hi")], _make_soul()) + + assert result == [] + + async def test_does_not_inject_for_subagent(self) -> None: + provider = OrchestrationInjectionProvider() + history = [_user("Refactor the runtime orchestration")] + + result = await provider.get_injections(history, _make_soul(is_subagent=True)) + + assert result == [] + + async def test_does_not_inject_in_plan_mode(self) -> None: + provider = OrchestrationInjectionProvider() + history = [_user("Enhance runtime orchestration")] + + result = await provider.get_injections(history, _make_soul(plan_mode=True)) + + assert result == [] + + async def test_does_not_inject_when_goal_active(self) -> None: + provider = OrchestrationInjectionProvider() + history = [_user("Enhance runtime orchestration")] + goal = GoalState(objective="ship the feature", status="active") + + result = await provider.get_injections(history, _make_soul(goal=goal)) + + assert result == [] + + async def test_does_not_inject_in_auto_mode(self) -> None: + provider = OrchestrationInjectionProvider() + history = [_user("Enhance runtime orchestration")] + + result = await provider.get_injections(history, _make_soul(is_auto=True)) + + assert result == [] + + async def test_ignores_notification_and_system_reminder_when_finding_task(self) -> None: + provider = OrchestrationInjectionProvider() + history = [ + _user("Enhance runtime orchestration with tests"), + _notification(), + _system_reminder("Plan mode mentions implementation but is not the task"), + ] + + result = await provider.get_injections(history, _make_soul()) + + assert len(result) == 1 + + async def test_throttles_after_recent_reminder(self) -> None: + provider = OrchestrationInjectionProvider() + history = [ + _user("Enhance runtime orchestration with tests"), + _system_reminder( + "Orchestration reminder: choose direct tools, SetTodoList, " + "RunAgents, and verification." + ), + _assistant(), + ] + + result = await provider.get_injections(history, _make_soul()) + + assert result == [] + + async def test_reinjects_after_interval(self) -> None: + provider = OrchestrationInjectionProvider() + history = [ + _user("Enhance runtime orchestration with tests"), + _system_reminder( + "Orchestration reminder: choose direct tools, SetTodoList, " + "RunAgents, and verification." + ), + *[_assistant() for _ in range(5)], + ] + + result = await provider.get_injections(history, _make_soul()) + + assert len(result) == 1 + + async def test_substantial_terms_match_whole_words_only(self) -> None: + provider = OrchestrationInjectionProvider() + # "prefix" contains "fix" and "explanation" contains "plan" — neither + # is a substantial-task signal on its own. + history = [_user("Could you explain the prefix explanation?")] + + result = await provider.get_injections(history, _make_soul()) + + assert result == [] + + async def test_reinjects_when_compaction_drops_prior_reminder(self) -> None: + # The provider is stateless: once compaction collapses the reminder + # into the summary, the marker scan finds nothing and re-arms. + provider = OrchestrationInjectionProvider() + throttled_history = [ + _user("Enhance runtime orchestration with tests"), + _system_reminder("Orchestration reminder: choose the lightest effective work shape."), + _assistant(), + ] + assert await provider.get_injections(throttled_history, _make_soul()) == [] + + compacted_history = [_user("Enhance runtime orchestration with tests")] + + result = await provider.get_injections(compacted_history, _make_soul()) + + assert len(result) == 1 diff --git a/tests/core/test_skip_auto_prompt_injection.py b/tests/core/test_skip_auto_prompt_injection.py index 7bffbd35..d7852818 100644 --- a/tests/core/test_skip_auto_prompt_injection.py +++ b/tests/core/test_skip_auto_prompt_injection.py @@ -14,6 +14,7 @@ from pythinker_code.soul.agent import Agent, Runtime from pythinker_code.soul.context import Context from pythinker_code.soul.dynamic_injections.auto_mode import AutoModeInjectionProvider +from pythinker_code.soul.dynamic_injections.orchestration import OrchestrationInjectionProvider from pythinker_code.soul.dynamic_injections.plan_mode import PlanModeInjectionProvider from pythinker_code.soul.pythinkersoul import PythinkerSoul @@ -41,8 +42,9 @@ def test_skip_auto_prompt_injection_gates_auto_provider( soul = _make_soul(runtime, tmp_path) types_ = _provider_types(soul) - # Plan is always present and never gated by this flag. + # Plan and orchestration are always present and never gated by this flag. assert PlanModeInjectionProvider in types_ + assert OrchestrationInjectionProvider in types_ assert not any(provider.__name__.lower().startswith("yolo") for provider in types_) if skip: diff --git a/tests/tools/test_todo.py b/tests/tools/test_todo.py index e325d6fc..1c0e3c66 100644 --- a/tests/tools/test_todo.py +++ b/tests/tools/test_todo.py @@ -344,10 +344,9 @@ async def test_subagent_malformed_individual_item(self, runtime: Runtime): class TestSingleInProgressInvariant: - """Ported from Codex CLI's plan tool contract (plan_spec.rs): at most one - step in_progress at a time — softened to a notice because pythinker's - parallel-subagent fan-out legitimately tracks one in_progress sub-todo - per running child (system.md orchestration rules).""" + """At most one step in_progress at a time — softened to a notice because + pythinker's parallel-subagent fan-out legitimately tracks one in_progress + sub-todo per running child (system.md orchestration rules).""" async def test_multiple_in_progress_accepted_with_notice( self, set_todo_list_tool: SetTodoList, runtime: Runtime diff --git a/tests/ui_and_conv/test_prompt_tips.py b/tests/ui_and_conv/test_prompt_tips.py index 063f91ba..6351f287 100644 --- a/tests/ui_and_conv/test_prompt_tips.py +++ b/tests/ui_and_conv/test_prompt_tips.py @@ -304,6 +304,29 @@ def test_working_spinner_tip_mentions_thinking_effort_shortcut() -> None: assert current_tip(0) == "Shift+Tab changes thinking effort levels" +def test_feature_tips_reference_only_real_slash_commands() -> None: + """Every /command a tip advertises must exist in a slash registry. + + Tips are marketing for real features; a tip naming a nonexistent + command (e.g. a past "/verify" tip) erodes trust in all of them. + """ + import re as _re + + from pythinker_code.soul.slash import registry as soul_registry + from pythinker_code.ui.shell.slash import registry as shell_registry + from pythinker_code.ui.shell.slash import shell_mode_registry + from pythinker_code.ui.shell.tips import FEATURE_TIPS + + known = ( + set(soul_registry._command_aliases) + | set(shell_registry._command_aliases) + | set(shell_mode_registry._command_aliases) + ) + for tip in FEATURE_TIPS: + for command in _re.findall(r"/([a-z][a-z-]*)", tip): + assert command in known, f"tip advertises unknown command /{command}: {tip!r}" + + # ── _display_width ───────────────────────────────────────────────────────────── @@ -552,7 +575,7 @@ def test_background_working_status_uses_pulsing_circle(monkeypatch: Any) -> None assert "background agent" not in second # footer owns the count -def test_card_toolbar_shows_codex_style_background_task_summary(monkeypatch: Any) -> None: +def test_card_toolbar_shows_compact_background_task_summary(monkeypatch: Any) -> None: prompt_session = _make_toolbar_session(model_name="fast-model", tips=[]) prompt_session._background_task_count_provider = lambda: BgTaskCounts(bash=2, agent=1) @@ -971,10 +994,10 @@ def get_size() -> Any: assert status_call_count == 0, "_get_git_status must not be called when branch is None" -# ── Prompt layout (Codex-style lower text area, running/idle message) ───────── +# ── Prompt layout (lower text area, running/idle message) ───────── -def test_running_prompt_uses_shared_toolbar_and_codex_input_layout(monkeypatch: Any) -> None: +def test_running_prompt_uses_shared_toolbar_and_bottom_input_layout(monkeypatch: Any) -> None: width = 72 prompt_session = object.__new__(CustomPromptSession) prompt_session._mode = PromptMode.AGENT @@ -1293,7 +1316,7 @@ def _dummy_slash_func(*_args: Any, **_kwargs: Any) -> None: return None -def test_slash_completer_uses_codex_prefix_order_and_canonical_insertions() -> None: +def test_slash_completer_uses_prefix_order_and_canonical_insertions() -> None: completer = SlashCommandCompleter( [ SlashCommand( @@ -1443,7 +1466,7 @@ def test_prompt_rule_keeps_rightmost_column_clear() -> None: assert shell_prompt._prompt_rule(8) == "─" * 7 -def test_idle_agent_prompt_uses_same_codex_input_layout(monkeypatch: Any) -> None: +def test_idle_agent_prompt_uses_same_bottom_input_layout(monkeypatch: Any) -> None: width = 64 prompt_session = object.__new__(CustomPromptSession) prompt_session._running_prompt_delegate = None diff --git a/tests/ui_and_conv/test_tui_components.py b/tests/ui_and_conv/test_tui_components.py index 60beb9f9..245d4c31 100644 --- a/tests/ui_and_conv/test_tui_components.py +++ b/tests/ui_and_conv/test_tui_components.py @@ -247,7 +247,7 @@ def render(self, width: int): # --------------------------------------------------------------------------- -def test_bash_execution_uses_codex_style_compact_layout(): +def test_bash_execution_uses_compact_execution_cell_layout(): out = render_plain( render_bash_execution( BashExecutionState( From b40cdb71e37d5bd37c92df8192378996571bb681 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Fri, 12 Jun 2026 04:59:33 -0400 Subject: [PATCH 06/49] feat(acp): stop advertising the question tool to ACP clients ACP sessions cannot present interactive questions (the session loop signals QuestionNotSupported), so advertising AskUserQuestion invites a wasted model step per question. replace_tools now hides the tool from the model-facing list while keeping it registered, so a stray call still resolves through the graceful textual fallback. Harmonize the task log after merging refactor/agent-contract-and-tool-metadata. --- src/pythinker_code/acp/tools.py | 6 ++++ tasks/todo.md | 18 +++++++++- tests/acp/test_acp_tool_visibility.py | 48 +++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/acp/test_acp_tool_visibility.py diff --git a/src/pythinker_code/acp/tools.py b/src/pythinker_code/acp/tools.py index 1a21d377..021d5c93 100644 --- a/src/pythinker_code/acp/tools.py +++ b/src/pythinker_code/acp/tools.py @@ -10,6 +10,7 @@ from pythinker_code.soul.approval import Approval from pythinker_code.soul.permission import check_shell_command_allowed from pythinker_code.soul.toolset import PythinkerToolset +from pythinker_code.tools.ask_user import AskUserQuestion from pythinker_code.tools.shell import Params as ShellParams from pythinker_code.tools.shell import Shell from pythinker_code.tools.utils import ToolResultBuilder @@ -28,6 +29,11 @@ def replace_tools( # Only replace tools when running locally or under ACPHost. return + # ACP clients get no interactive question UI (the session loop signals + # QuestionNotSupported), so don't advertise the tool — a hallucinated call + # still resolves through the registered tool's graceful fallback. + toolset.hide(AskUserQuestion.name) + if client_capabilities.terminal and (shell_tool := toolset.find(Shell)): # Replace the Shell tool with the ACP Terminal tool if supported. toolset.add( diff --git a/tasks/todo.md b/tasks/todo.md index 6768c468..82fd03f0 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -2,7 +2,23 @@ ## Active -Branch `refactor/agent-contract-and-tool-metadata` — landing step 1 of +- [ ] Agent-harness adoption arc (`feat/agent-harness-enhancements`): port the + reference harness's remaining coding-agent design (blackbox/agent_x) into + pythinker, generically framed. Checkpoint 0 committed (`047a0b29`): + orchestration injection provider + name-scrub. Gap-map workflow running + (14 clusters, map+verify); next: synthesize adoption plan → + checkpointed TDD implementation, clean-code-guard per checkpoint. +- [ ] Windows shell hardening (researched, not yet implemented): bash-first + shell policy (Git Bash probe → pwsh → powershell, never cmd), Windows + tool-description guidance (`;` not `&&` on PS 5.1, `$env:`, quoting), + docker-daemon-down interceptor (`error during connect` + + `pipe/docker` → actionable remediation incl. `docker desktop start`), + POSIX-ism lint under PowerShell, `CTRL_BREAK_EVENT` + `taskkill /T` + tree-kill, `-EncodedCommand` UTF-16LE for PowerShell args. Full brief in + session notes 2026-06-12; permission tokenization is POSIX-blind for + PowerShell syntax (gate review needed before shipping). + +Merged from `refactor/agent-contract-and-tool-metadata` — step 1 of `tasks/design-adoption-blueprint.md` (agent-logic/coding-flow cleanup): - [x] Task 1: FetchURL untrusted-envelope fix — DONE (ceeeeb78; 3 TDD tests, diff --git a/tests/acp/test_acp_tool_visibility.py b/tests/acp/test_acp_tool_visibility.py new file mode 100644 index 00000000..9c2c0890 --- /dev/null +++ b/tests/acp/test_acp_tool_visibility.py @@ -0,0 +1,48 @@ +"""ACP toolset adaptation: tools a client cannot service are hidden up front. + +The session loop already degrades a stray ``AskUserQuestion`` call gracefully +(``QuestionNotSupported`` → textual fallback), but advertising the tool to the +model invites a wasted step per question. ``replace_tools`` hides it instead; +the tool stays registered so a hallucinated call still hits the graceful path. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import acp +from pythinker_host.local import local_host + +import pythinker_code.acp.tools as acp_tools +from pythinker_code.acp.tools import replace_tools +from pythinker_code.soul.toolset import PythinkerToolset +from pythinker_code.tools.ask_user import AskUserQuestion + + +def _capabilities(*, terminal: bool = False) -> acp.schema.ClientCapabilities: + return acp.schema.ClientCapabilities(terminal=terminal) + + +def _make_toolset() -> PythinkerToolset: + toolset = PythinkerToolset() + toolset.add(AskUserQuestion()) + return toolset + + +class TestReplaceToolsHidesQuestionTool: + def test_ask_user_question_is_hidden_from_model(self, monkeypatch) -> None: + monkeypatch.setattr(acp_tools, "get_current_host", lambda: local_host) + toolset = _make_toolset() + + replace_tools(_capabilities(), MagicMock(), "sid", toolset, MagicMock()) + + visible = [tool.name for tool in toolset.tools] + assert "AskUserQuestion" not in visible + + def test_ask_user_question_remains_registered_for_graceful_fallback(self, monkeypatch) -> None: + monkeypatch.setattr(acp_tools, "get_current_host", lambda: local_host) + toolset = _make_toolset() + + replace_tools(_capabilities(), MagicMock(), "sid", toolset, MagicMock()) + + assert toolset.find(AskUserQuestion) is not None From 54e8070ea0db46f2ffda281db632beaeb23d1050 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Fri, 12 Jun 2026 05:05:45 -0400 Subject: [PATCH 07/49] fix: apply review-deferred fixes from the contract/metadata work - FetchURL: await spill_to_disk() at the trafilatura and fetch-service sites so large pages spill off the event loop instead of falling back to the synchronous spill inside ok(). - MCPTool: declare emits_tool_execution_started_after_approval so the ToolExecutionStarted event defers until approval resolves, matching every other approval-gated tool; the old _approval duck-typing missed this class because it requests via runtime.approval. Pinned in test_toolset.py. - spinner_words: genericize remaining external credit wording. - tasks/todo.md: record the fixes; document why structural flag enforcement for future adapters is deferred (no shared adapter base until the toolset split lands). --- src/pythinker_code/soul/toolset.py | 10 ++++++++++ src/pythinker_code/tools/web/fetch.py | 4 ++++ src/pythinker_code/ui/shell/spinner_words.py | 10 +++++----- tasks/todo.md | 17 +++++++++-------- tests/core/test_toolset.py | 5 +++++ 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index 2aae5fc0..1f99d13f 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -1105,6 +1105,16 @@ class MCPTool[T: ClientTransport](CallableTool): flag on an external adapter disables its permission gating. """ + emits_tool_execution_started_after_approval: ClassVar[bool] = True + """Defer ToolExecutionStarted until ``approval.request`` resolves. + + ``__call__`` requests approval as its first step, and ``Approval.request`` + emits the started event after resolution (idempotent per call id), so the + UI shows the approval prompt before the tool reads as "running" — the same + ordering as Shell/WriteFile. The old ``_approval`` duck-typing missed this + class because it requests via ``runtime.approval`` instead. + """ + def __init__( self, server_name: str, diff --git a/src/pythinker_code/tools/web/fetch.py b/src/pythinker_code/tools/web/fetch.py index e24f06b4..9292e6fc 100644 --- a/src/pythinker_code/tools/web/fetch.py +++ b/src/pythinker_code/tools/web/fetch.py @@ -279,6 +279,8 @@ async def fetch_with_http_get( builder.mark_untrusted() builder.write(extracted_text) + # Spill the full text off the event loop before building the result. + await builder.spill_to_disk() return builder.ok("The returned content is the main text content extracted from the page.") async def _fetch_with_service(self, params: Params) -> ToolReturnValue: @@ -341,6 +343,8 @@ async def _fetch_with_service(self, params: Params) -> ToolReturnValue: ) builder.mark_untrusted() builder.write(content) + # Spill the full text off the event loop before building the result. + await builder.spill_to_disk() return builder.ok( "The returned content is the main content extracted from the page." ) diff --git a/src/pythinker_code/ui/shell/spinner_words.py b/src/pythinker_code/ui/shell/spinner_words.py index 94872b3f..7e1a6e11 100644 --- a/src/pythinker_code/ui/shell/spinner_words.py +++ b/src/pythinker_code/ui/shell/spinner_words.py @@ -1,4 +1,4 @@ -"""Blackbox-inspired loading spinner words for the shell TUI.""" +"""Loading spinner words for the shell TUI.""" from __future__ import annotations @@ -8,12 +8,12 @@ # import them from the spinner module. from pythinker_code.ui.shell.glyphs import SPINNER_FRAME_INTERVAL_S, SPINNER_FRAMES -# Keep each verb on-screen long enough to be readable, matching Blackbox's -# stable loading-word feel rather than changing every frame. +# Keep each verb on-screen long enough to be readable — a stable loading word +# rather than one that changes every frame. SPINNER_VERB_INTERVAL_S = 600.0 -# Ported from blackbox/src/constants/spinnerVerbs.ts. Keep the list broad so -# long-running turns do not look frozen even when no new tool output arrives. +# Keep the list broad so long-running turns do not look frozen even when no +# new tool output arrives. SPINNER_VERBS: tuple[str, ...] = ( "Accomplishing", "Actioning", diff --git a/tasks/todo.md b/tasks/todo.md index 82fd03f0..7a9e06e4 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -57,15 +57,16 @@ Out of scope this PR (logged): pricing display move out of core, the (blueprint P1a/P2a/P2b). ### Deferred from this branch's reviews -- `await builder.spill_to_disk()` missing at FetchURL trafilatura + - fetch-service sites (pre-existing; sync-spill fallback is correct, just - blocks the event loop briefly). +- [x] FetchURL spill awaits — DONE: `await builder.spill_to_disk()` added at + the trafilatura + fetch-service sites. +- [x] MCPTool event ordering — DONE: `emits_tool_execution_started_after_approval` + set on MCPTool (Approval.request emits after resolution, idempotent per + call id); pinned in test_toolset.py alongside the other 8 classes. - No structural enforcement that FUTURE external adapters declare - `external_side_effect_tool` (pin tests cover the current three only); - consider an `__init_subclass__` check or lint rule. -- MCPTool fires ToolExecutionStarted before its approval dialog resolves - (no `emits_tool_execution_started_after_approval`); pre-existing event - ordering inconsistency vs Shell/WriteFile — decide deliberately. + `external_side_effect_tool` (pin tests cover the current three only). + Deliberately NOT bolted on now: the three adapters share no in-package + base class to hang an `__init_subclass__` hook on; revisit when the + toolset split (blueprint P2a) introduces an adapter base. Done: `mythos-enhancements` PR #118 merged (d51ef649). diff --git a/tests/core/test_toolset.py b/tests/core/test_toolset.py index 0b31be27..cfc7a625 100644 --- a/tests/core/test_toolset.py +++ b/tests/core/test_toolset.py @@ -558,6 +558,7 @@ def test_approval_gated_tools_declare_deferred_execution_started() -> None: """ from pythinker_code.acp.tools import Terminal from pythinker_code.plugin.tool import PluginTool + from pythinker_code.soul.toolset import MCPTool from pythinker_code.tools.agent import RunAgents from pythinker_code.tools.background import TaskInput, TaskStop from pythinker_code.tools.file.replace import StrReplaceFile @@ -573,6 +574,10 @@ def test_approval_gated_tools_declare_deferred_execution_started() -> None: Terminal, PluginTool, RunAgents, + # MCPTool requests approval via runtime.approval (not _approval) as the + # first step of __call__; the flag defers ToolExecutionStarted until + # that approval resolves, matching every other approval-gated tool. + MCPTool, ) for tool_class in approval_gated: assert tool_class.emits_tool_execution_started_after_approval is True, tool_class From 36cedafd445716f239db47911e0f5fd920d833f1 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Fri, 12 Jun 2026 05:20:32 -0400 Subject: [PATCH 08/49] docs(tasks): add verified agent-harness adoption plan (124 ranked items) Synthesized from a 14-cluster map+adversarial-verify workflow comparing the local reference agent harness against src/pythinker_code. Each item records current state, verifier evidence, an adoption sketch fitted to pythinker's design, effort/value, and target files; three refuted claims are pinned so they are not re-implemented. Includes execution discipline for the multi-writer branch (checkpoint = TDD + clean-code-guard + green gates, hot-file serialization). --- tasks/agent-harness-adoption-plan.md | 1331 ++++++++++++++++++++++++++ tasks/todo.md | 12 +- 2 files changed, 1338 insertions(+), 5 deletions(-) create mode 100644 tasks/agent-harness-adoption-plan.md diff --git a/tasks/agent-harness-adoption-plan.md b/tasks/agent-harness-adoption-plan.md new file mode 100644 index 00000000..3313f6a8 --- /dev/null +++ b/tasks/agent-harness-adoption-plan.md @@ -0,0 +1,1331 @@ +# Agent Harness Adoption Plan — verified gap map + +**Generated:** 2026-06-12 from a 14-cluster / 28-agent map+adversarial-verify workflow comparing the +local reference agent harness against `src/pythinker_code`. Every item survived a refutation pass +against live source (124 kept, 3 refuted). `/` = the reference workspace root under +`blackbox/` (Rust crates); pythinker paths are repo-relative. Naming rule: all adopted work is framed as +generic pythinker agent enhancements — no external product names in code, comments, commits, or docs. + +## Execution discipline + +- One checkpoint = one item (or one tight cluster of S items), TDD (red→green), `clean-code-guard` + pass on the diff, `make check` + targeted pytest green, then commit. Branch: `feat/agent-harness-enhancements`. +- Multi-writer tree: re-check `git status`/`git log` before each checkpoint; accept and harmonize + concurrent work, never revert what you did not author. +- Hot files (serialize across checkpoints): `soul/pythinkersoul.py`, `soul/toolset.py`, `config.py`, + `soul/permission.py`, `agents/default/system.md` (test-pinned: use `--inline-snapshot=fix` deliberately). +- L-effort items get their own design note in `tasks/` before code. + +## Tier 1 — high value, S/M effort (27 items) + +### `context-mgmt/history-invariant-repair-at-restore-prompt-build-synthesize-` — partial, S, high + +**Today.** Partial. Write-time coverage exists: interrupted tool calls get synthetic 'interrupted by user' results (src/pythinker_code/soul/pythinkersoul.py:1763-1790) and torn-line repair fixes partial JSONL writes (src/pythinker_code/soul/context.py:35-44). But normalize_history only merges adjacent user messages (src/pythinker_code/soul/dynamic_injection.py:185-211); a crash between appending the assistant message (with tool_calls) and the tool results leaves a dangling call that, after restore, makes every subsequent API call fail with a pairing error. Image stripping is gated only at ingestion (src/pythinker_code/tools/file/read_media.py:56,194). + +**Verifier note.** Claim confirmed as partial. Minor line drift: the synthetic 'Tool call interrupted by user.' results are at pythinkersoul.py:1801-1817 (claim cited 1763-1790). + +**Adopt.** Extend normalize_history (or a restore-time pass in Context.restore) to scan for assistant messages whose tool_calls lack a matching tool-role message and insert a synthetic 'aborted' tool result right after, and to drop tool messages whose tool_call_id has no preceding call. Pure-function change, easily unit-tested against crafted crash transcripts. + +**Files.** `src/pythinker_code/soul/dynamic_injection.py`, `src/pythinker_code/soul/context.py`, `src/pythinker_code/soul/pythinkersoul.py` + +### `prompts-instructions/decision-complete-plan-mode-interviewing-protocol-and-plan-d` — partial, S, high + +**Today.** Plan mode is mechanically stronger (tool-enforced read-only, plan file, mandatory Verification section, ExitPlanMode options for multiple approaches, AskUserQuestion guidance, 2-3 approaches max — soul/dynamic_injections/plan_mode.py _full_reminder/_sparse_reminder), but the prompting lacks the discoverable-vs-preference unknowns taxonomy, the recommended-default-and-proceed-as-assumption rule, the decision-completeness bar, and any plan-document structure/brevity rubric. + +**Verifier note.** Claim survives for interactive plan mode, but two adjacent implementations were missed and should temper the gap: (1) the recommended-default-and-proceed-as-assumption rule exists nearly verbatim in prompts/best_practices.md line 23 ('enumerate the plausible interpretations and say which one you are taking... otherwise proceed and record the assumption') — though only opt-in via /best-practices (soul/slash.py lines 301-324), not wired into plan mode; (2) the plan subagent spec (agents/default/plan.yaml) enforces a decision-completeness-like bar ('Every load-bearing task must be executable as written. "Figure out X during implementation" is not a task — it is either an explicit explore task or a BLOCKER'; Escalation: 'list the exact questions under BLOCKERS instead of planning on assumptions') plus a full plan-document structure contract (SUMMARY/CONTEXT/TASK DEPENDENCY GRAPH/PLAN/EVIDENCE/RISKS/BLOCKERS). Interactive plan mode itself (plan_mode.py _full_reminder/_sparse_reminder, tools/plan/enter_description.md) has only a rudimentary preference trigger ('When the best approach depends on user preferences... use AskUserQuestion'; 'User Preferences Matter' condition 7) — no discoverable-vs-preference taxonomy, no decision-completeness bar, and no plan-file brevity rubric anywhere. + +**Adopt.** Extend _full_reminder (and a line in _sparse_reminder) in plan_mode.py: (a) two-unknowns rule — explore repo-discoverable facts before asking, ask preferences early via AskUserQuestion with 2-4 options + a recommended default, record un-answered defaults as Assumptions; (b) finalization bar — exit only when the plan is decision-complete; (c) plan-file shaping — 3-5 short sections incl. Assumptions, subsystem-grouped bullets, minimal path-naming. Pure prompt edits; update phrase-pinned tests with --inline-snapshot=fix. + +**Files.** `src/pythinker_code/soul/dynamic_injections/plan_mode.py` + +### `protocol-headless/strict-stdout-stderr-channel-discipline-and-structured-error` — partial, S, high + +**Today.** Partial. src/pythinker_code/ui/print/__init__.py uses `from rich import print` to stdout for all failure paths (LLMNotSet, ChatProviderError, MaxStepsReached + handoff, 'Interrupted by user', 'Unknown error') and echoes the command to stdout in text mode — in stream-json mode these plain-text lines corrupt the JSONL stream for parsers. Background-task timeout notices correctly use open_original_stderr(), showing the right pattern exists but is not applied uniformly. + +**Verifier note.** Claim confirmed as stated; could not refute. src/pythinker_code/ui/print/__init__.py:18 does `from rich import print`, and every failure path prints plain text to stdout: LLMNotSet (line 412), LLMNotSupported (416), ChatProviderError (420), MaxStepsReached + handoff block (424, 436), 'Interrupted by user' (440), 'Unknown error' (444) — none are gated on output_format, so they corrupt the stream-json stdout channel. The command echo (lines 90-91) IS gated to text mode (`output_format == "text" and not final_only`), matching the claim's wording. No structured error WireMessage type exists for JsonPrinter to emit. The background-timeout notice correctly uses open_original_stderr() (lines 296-308), confirming the right pattern exists but is not applied to the exception handlers. + +**Adopt.** In Print.run, route all human-facing diagnostics through the original-stderr writer; when output_format == stream-json, additionally emit a final structured error event (reuse Notification or a new ErrorEvent wire type) on stdout before exiting. Add a unit test asserting every stdout line in stream-json mode parses as JSON across each failure path. + +**Files.** `src/pythinker_code/ui/print/__init__.py`, `/exec/src/lib.rs` + +### `review-mode/deterministic-review-target-resolution-and-prompt-synthesis` — partial, S, high + +**Today.** The standalone review engine resolves diffs deterministically with base/staged/working-tree/range modes, fallback refs, and a fallback audit trail (packages/pythinker-review/src/pythinker_review/engine/diff_source.py). But agent-mediated review dispatch leaves git scoping entirely to the model — system.md only instructs it prose-style to compute the merge base (src/pythinker_code/agents/default/system.md:120), and git-context injection is gated to explore subagents only (src/pythinker_code/subagents/core.py:90). + +**Verifier note.** Claim CONFIRMED as stated; all three cited anchors verified. The standalone engine is deterministic; the agent-mediated path has no deterministic diff scoping anywhere — the review/code-reviewer subagent specs receive scope purely via the parent's prompt text. + +**Adopt.** Add a small resolver (target -> prompt + hint) that precomputes the merge-base SHA via the async git helper and renders one of three template prompts (uncommitted / base-branch-with-sha + backup variant / commit-with-title), then prepend it to the review subagent's prompt at dispatch. Also extend collect_git_context injection in subagents/core.py to reviewer-class agents so every review run starts with branch, dirty files, and merge-base already in context instead of burning turns rediscovering them. + +**Files.** `src/pythinker_code/subagents/core.py`, `src/pythinker_code/subagents/git_context.py`, `packages/pythinker-review/src/pythinker_review/engine/diff_source.py`, `src/pythinker_code/agents/default/system.md` + +### `tools-registry-codemode/concurrency-policy-for-parallel-tool-calls-parallel-safe-too` — missing, S, high + +**Today.** PythinkerToolset.handle spawns an asyncio task per call immediately (src/pythinker_code/soul/toolset.py:769) and pythinker_core.step awaits them with no ordering or locking (packages/pythinker-core/src/pythinker_core/__init__.py:86-122). When a provider emits parallel tool calls, WriteFile + Shell + StrReplaceFile from the same assistant message all execute concurrently with no race protection; no lock exists in tools (rg asyncio.Lock). + +**Verifier note.** Claim survives. The only sharing mechanism is same-step dedup of byte-identical calls (same tool name + canonical args await the original task); there is no parallel-safe vs mutating classification and no serialization of mutating tools. The external_side_effect_tool flag exists but is consumed only by permission gating/visibility, not concurrency. + +**Adopt.** Add a class-level supports_parallel flag on tools (default False for Shell, WriteFile, StrReplaceFile, MultiEdit, plugin/MCP side-effect tools; True for ReadFile/Grep/Glob/recall etc.) and an async read/write gate inside the per-call task in PythinkerToolset.handle: parallel-safe tools acquire shared, others exclusive. Keeps streaming dispatch but makes mutation ordering deterministic. + +**Files.** `src/pythinker_code/soul/toolset.py`, `packages/pythinker-core/src/pythinker_core/__init__.py`, `/core/src/tools/parallel.rs` + +### `config-features/per-project-trust-gating-of-project-scope-config-and-hooks` — missing, M, high + +**Today.** Missing. _load_scoped in src/pythinker_code/config.py merges .pythinker/config.toml unconditionally; the `hooks` field (shell commands, src/pythinker_code/hooks/config.py HookDef) is NOT in SCOPE_LOCKED_PATHS, and app.py:390 runs HookEngine(config.hooks) — so a cloned repo's project config can auto-execute shell hooks on SessionStart. Trust exists only as a session-scoped flag (src/pythinker_code/session_state.py TrustStateData, /trust at ui/shell/slash.py:1730) that never gates config loading. + +**Verifier note.** Claim confirmed. Project-scope config merges unconditionally and project-defined hooks execute with no trust gate. Trust/safe-mode is wired only into the approval layer (auto-approve gating), never into config loading or hook execution. Minor citation fix: /trust is defined at ui/shell/slash.py:1718 (state.trusted set at 1730). + +**Adopt.** Persist a project-root -> trust map (in user config or a metadata file, keyed by normalized resolved path; let /trust record it). In _load_scoped, when the project root is untrusted, still read project/local dicts but hold them as disabled scopes with a reason surfaced once at startup ('run /trust to enable project config and hooks'); at minimum gate the hooks list and statusline-adjacent keys behind trust immediately. Treat invalid TOML in untrusted scopes as an empty disabled scope. + +**Files.** `src/pythinker_code/config.py`, `src/pythinker_code/hooks/config.py`, `src/pythinker_code/app.py`, `src/pythinker_code/session_state.py`, `/config/src/loader/mod.rs` + +### `config-features/unknown-config-key-detection-with-source-located-diagnostics` — missing, M, high + +**Today.** Missing. Config models in src/pythinker_code/config.py use Pydantic's default extra='ignore', so a typo'd key (e.g. 'defaut_yolo') silently vanishes and changes agent behavior with no signal; validation errors are scope-attributed via _lookup_provenance but carry no positions, and there is no strict mode. + +**Verifier note.** Claim confirmed. No extra='forbid'/strict mode anywhere in the config models; typo'd keys are silently dropped. Provenance enrichment attaches only scope file-path strings to validation errors, with no line/column positions. + +**Adopt.** After merge, diff the raw per-scope dicts against Config's field tree (recursive walk of model_fields, or model_json_schema) and emit startup warnings naming the file and dotted path of each unrecognized key; add an opt-in strict flag (env or config) that escalates to ConfigError. tomlkit retains item positions if line numbers are wanted later. + +**Files.** `src/pythinker_code/config.py`, `/config/src/strict_config.rs`, `/config/src/diagnostics.rs` + +### `context-mgmt/context-overflow-recovery-shrink-and-retry-instead-of-failin` — missing, M, high + +**Today.** Missing. classify_api_error tags 'context_overflow' for telemetry only (src/pythinker_code/soul/pythinkersoul.py:166-203); _is_retryable_error (pythinkersoul.py:2279-2297) treats context-length 400s as non-retryable, so the step raises and the turn dies. SimpleCompaction concatenates the entire history into one request with no shrink-on-overflow fallback (src/pythinker_code/soul/compaction.py:152-238). + +**Verifier note.** Claim confirmed. One nuance: pythinker does have two PROACTIVE pre-step shrink tiers (prune + auto-compact run before every step), but nothing REACTIVE — a context-length 400 from the provider still kills the turn, which is exactly what the claim says. + +**Adopt.** In the step error path, detect the context_overflow classification and respond by forcing prune_context then compact_context and retrying the step once instead of raising. Inside SimpleCompaction, on a context-length rejection drop the oldest messages from to_compact (preserving tool_call_id pairs) and retry, falling back to summarizing only the newest fitting slice. Bound both loops to avoid infinite retry. + +**Files.** `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/soul/compaction.py` + +### `core-loop/model-switch-context-continuity-compact-with-the-previous-mo` — missing, M, high + +**Today.** Missing. /model creates a brand-new session and reloads ('Starting fresh session for the new model...', ui/shell/slash.py:360-368) — all conversation context is discarded on every model change. There is no path that carries a compacted summary across the switch (rg comp_hash/model_switch shows only the telemetry event). + +**Verifier note.** Claim confirmed. /model with an actual model change unconditionally creates a brand-new Session (copying only additional_dirs) and reloads; no summary, compaction, or history is carried across. Minor precision notes: a thinking-effort-only change keeps the same session (slash.py:368), and session_fork (/fork,/undo) / session_recap exist but are unrelated to model switching. The only model_switch artifact is the telemetry event. + +**Adopt.** On model switch, before Reload, optionally run compact_context with the OUTGOING provider (compaction output is plain text, so it sidesteps provider-specific message-format incompatibilities like thinking blocks), then seed the new session's context with the summary sized to the new model's window. Offer continue-vs-fresh as a prompt or config flag; fall back to fresh on compaction failure. + +**Files.** `src/pythinker_code/ui/shell/slash.py`, `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/soul/compaction.py` + +### `core-loop/reactive-recovery-from-hard-context-overflow-provider-errors` — missing, M, high + +**Today.** Missing. classify_api_error labels 4xx context overflow as 'context_overflow' for telemetry only (pythinkersoul.py:186-196); _is_retryable_error excludes 4xx so the step raises and the turn ends with an error card. The shell only special-cases an LM Studio n_ctx hint (ui/shell/__init__.py:258-300, 1551-1564). Proactive thresholds can miss when the heuristic undercounts (e.g. large pending tool output) and there is no recovery path. + +**Verifier note.** Claim confirmed. There is no reactive compact-and-retry path anywhere: classification is telemetry-only, retry predicates exclude 4xx, and the only post-error special-casing is user-facing LM Studio messaging in the shell. Recovery wrappers handle 401 OAuth refresh and connection/timeout only. Compaction/pruning are exclusively proactive (threshold-driven before each step). + +**Adopt.** In _step's error path (or a wrapper in _agent_loop), detect the context_overflow classification, set the context token count to max_context_size so should_auto_compact is guaranteed to fire, run prune_context then compact_context, and retry the step exactly once (a one-shot flag prevents loops). Track a telemetry event for overflow-recovered vs overflow-failed. + +**Files.** `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/ui/shell/__init__.py` + +### `exec-safety/known-safe-read-only-command-auto-approval-prompt-elision` — missing, M, high + +**Today.** Absent. Every Shell call prompts unless yolo/auto mode or a prior signature-keyed session approval covers it (src/pythinker_code/tools/shell/__init__.py:154, src/pythinker_code/soul/approval.py request flow); the first `ls` or `git status` of a session always interrupts the user. permission.py has the inverse (block-listing) classifiers but no positive safe-list, and no host-path pinning (rsplit('/') basename normalization would let /tmp/fake/git match a future safelist). + +**Verifier note.** Claim confirmed. No positive safe-list exists anywhere; every foreground/background Shell call unconditionally calls Approval.request, which only auto-resolves via yolo/auto mode or a prior session-approval key. PreToolUse hooks cannot elide prompts either — runner.py honors only permissionDecision=deny/exit-2 block; 'allow' just means not-blocked and the approval prompt still fires later inside the tool. The claim's host-path note is also accurate: base-command normalization is bare rsplit('/') basename, no path pinning. + +**Adopt.** Add `is_known_safe_command(command) -> bool` to soul/permission.py reusing the existing tokenizer, _unwrap_command, _shell_hidden_command_reason rejection, and _has_unsafe_git_global_option: a positive safelist of read-only binaries with unsafe-flag exclusions, requiring every ;/&&/||/| segment to be safe. Consult it in Shell.__call__ before Approval.request (only when the execution policy is 'ask', never to override 'deny' profiles), and restrict basename matching for absolute first tokens to known system bin dirs so a workspace-local fake binary cannot ride the safelist. Track elisions in telemetry like auto_session approvals. + +**Files.** `src/pythinker_code/soul/permission.py`, `src/pythinker_code/tools/shell/__init__.py`, `src/pythinker_code/soul/approval.py`, `/shell-command/src/command_safety/is_safe_command.rs` + +### `mcp/per-server-startup-timeout-plus-actionable-startup-failure-d` — partial, M, high + +**Today.** Partial. Only a global mcp.client.tool_call_timeout_ms (60s) exists (config.py MCPClientConfig); there is no startup timeout, so a hung connect leaves a server in 'connecting' forever. The OAuth pre-check logs an actionable 'run pythinker mcp auth X' hint and sets 'unauthorized' status, but generic failures log raw exceptions and MCPServerSnapshot (wire/types.py) carries no error string, so /mcp shows 'failed' with no reason. + +**Verifier note.** Verdict stands, details confirmed: MCPClientConfig has only tool_call_timeout_ms (60000); the only other MCP timeout is _MCP_CLOSE_TIMEOUT_S for teardown, not connect. MCPServerSnapshot (wire/types.py) carries name/status/tools and no error string; /mcp rendering (ui/shell/mcp_status.py) shows an actionable hint only for 'unauthorized' (run: pythinker mcp auth X) and bare 'failed' otherwise. Two additions the claim missed: (a) a hung connect doesn't just leave 'connecting' forever — it blocks every agent turn, since _agent_loop awaits the loading task with no timeout; (b) an out-of-session diagnostic exists: `pythinker mcp test ` (cli/mcp.py) prints the actual connect exception, and the system prompt points users to /mcp and mcp auth (agents/default/system.md:130). + +**Adopt.** Wrap _connect_server in asyncio.wait_for with a per-server startup_timeout_s (config key, default ~30s); add an `error` field to MCPServerSnapshot and render it in /mcp; classify the common failure shapes (timeout -> suggest raising startup timeout; OAuth/401 -> suggest the auth command; command-not-found -> show resolved command) into one short actionable string. + +**Files.** `src/pythinker_code/soul/toolset.py`, `src/pythinker_code/config.py`, `src/pythinker_code/ui/shell/mcp_status.py`, `/mcp/src/connection_manager.rs` + +### `mcp/per-server-tool-allow-deny-filtering-enforced-at-list-time-a` — partial, M, high + +**Today.** Missing. _register_mcp_tools (soul/toolset.py) registers every tool the server lists; mcp.json schema and `pythinker mcp add` (cli/mcp.py) have no enabled/disabled tool fields; no grep hits for enabled_tools/disabled_tools/include_tools outside agentspec.py (which filters built-in tools for subagents, not MCP). + +**Verifier note.** Verdict 'missing' is too strong. Confirmed missing for the main session and config layer: mcp.json has no enabled/disabled tool fields and _register_mcp_tools registers every listed tool unconditionally. BUT the claim's parenthetical that agentspec filtering covers 'built-in tools for subagents, not MCP' is factually wrong: agent specs' allowed_tools/exclude_tools accept named MCP entries keyed mcp____; load_tools deliberately skips them as 'named dynamic tools' and toolset.add_shared_tools attaches them from runtime.mcp_tools (populated at connect: runtime.mcp_tools[f"mcp__{server_name}__{tool.name}"] = tool). So per-tool MCP allowlisting enforced at list time AND call time (tool simply absent from the toolset) exists for subagents/custom agent specs — just not via mcp.json and not for the default main agent, whose load_mcp_tools path bypasses the spec allowlist. + +**Adopt.** Add optional `enabledTools`/`disabledTools` arrays per server in mcp.json; filter in _connect_server before _register_mcp_tools, and re-check membership inside MCPTool.__call__ as the call-time gate. Keeps noisy servers (e.g. browser MCP with 30 tools) from flooding the model tool list and doubles as a safety scoping knob. + +**Files.** `src/pythinker_code/soul/toolset.py`, `src/pythinker_code/cli/mcp.py`, `/mcp/src/tools.rs` + +### `multi-agent/enforced-workspace-isolation-for-parallel-write-capable-chil` — partial, M, high + +**Today.** Partial — intent only. Agent/RunAgents accept isolation="worktree" but the docstring says it merely 'records a git-worktree isolation intent' (src/pythinker_code/tools/agent/__init__.py:88-94); BackgroundTaskManager stores it as task-spec metadata (src/pythinker_code/background/manager.py:334,367) and src/pythinker_code/background/agent_runner.py never reads it, so parallel coder/implementer children share one working tree and can clobber each other. + +**Verifier note.** Claim upheld exactly: intent-only metadata. No code in the repo executes `git worktree` for agents; the word 'worktree' appears in src only in the isolation field docs and prompt prose (system.md/best_practices.md warnings about dirty worktrees). + +**Adopt.** Honor isolation="worktree" for background write-profile children: create a git worktree per agent under the session dir before launch, point the child runtime's work_dir at it, and on completion report the worktree path plus a diff summary in the final report so the orchestrator (or user) merges deliberately. Clean up or retain worktrees per existing recovery rules; reject isolation for non-git roots with an actionable error. + +**Files.** `src/pythinker_code/tools/agent/__init__.py`, `src/pythinker_code/background/agent_runner.py`, `src/pythinker_code/background/manager.py` + +### `multi-agent/spawn-time-context-fork-child-inherits-filtered-parent-histo` — missing, M, high + +**Today.** Missing. prepare_soul in src/pythinker_code/subagents/core.py restores only the child's own persisted context (resume case); new children start blank and rely on the orchestrator hand-writing RunAgents base_prompt / Agent prompt context packets (src/pythinker_code/tools/agent/__init__.py). Session-level fork/recap exists for sessions but there is no parent-to-child history fork at Agent spawn. + +**Verifier note.** Claim upheld. No parent-to-child history fork exists at Agent spawn; checked for fork/inherit/handoff mechanisms — TaskHandoff (tools/background/__init__.py:423) is an info dump for the user, not a context transfer, and session_fork.py operates on the root session wire (enumerate_turns/truncate_wire_at_turn/fork_session) only. + +**Adopt.** Add fork_context: bool to Agent Params (new instances only, reject with model override). In prepare_soul, when forking, seed the child Context from the parent soul's history filtered to user messages and assistant final texts (skip tool call/result records and thinking blocks), then append the task prompt. Persist the seeded history through the existing context file so resume keeps working. + +**Files.** `src/pythinker_code/subagents/core.py`, `src/pythinker_code/tools/agent/__init__.py`, `src/pythinker_code/subagents/store.py` + +### `observability-feedback/consolidated-per-turn-rollup-analytics-event-turn-fact-reduc` — partial, M, high + +**Today.** Partial. The pythinker.turn span records stop_reason/step_count/model/plan_mode (src/pythinker_code/soul/pythinkersoul.py:1171-1202) and record_turn emits counters (src/pythinker_code/telemetry/metrics.py:178); token usage lands on per-call llm spans (pythinkersoul.py:1665-1700) and tool_call/tool_error track events fire per call without turn_id (src/pythinker_code/soul/toolset.py:693-706). No single per-turn record ties tool-type counts, usage, error kind, and resolved config together, so fleet dashboards must join disparate events. + +**Verifier note.** Claim stands as 'partial'; details verified accurate. Full inventory of track() event names (incl. multiline calls) confirms there is no consolidated per-turn analytics event tying tool counts, usage, error kind, and config together. Minor refinements: a turn_id DOES exist internally (soul._current_turn_id, passed to the toolset per step), and the tool_call_dedup_detected track events DO carry turn_id+step_no — but the general tool_call/tool_error track events do not, exactly as claimed. + +**Adopt.** Accumulate a TurnSummary in the soul during _agent_loop (tool counts bucketed by category, cumulative usage delta, steer count, api error kind/status, approval mode, plan/goal mode flags, is_first_turn) and emit one `turn_completed` track event plus span attributes at turn end. Reuse existing classify_api_error and tool categories; add turn_id to the existing tool_call track events for joinability. + +**Files.** `/analytics/src/reducer.rs`, `/analytics/src/facts.rs`, `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/soul/toolset.py`, `src/pythinker_code/telemetry/metrics.py` + +### `observability-feedback/feedback-log-ring-buffer-structured-session-tags-and-connect` — partial, M, high + +**Today.** Partial. /feedback builds a redacted structured payload (git snapshot, 10 recent-error metadata entries from telemetry/errors.py ring, recent messages, tool-call summaries, subagents) with an explicit consent summary (src/pythinker_code/feedback.py, telemetry/errors.py:111-140), and /report-error submits the error ring (src/pythinker_code/ui/shell/slash.py:725). But no in-memory full-log buffer or log-tail attachment exists (logs go to a rotating disk file, src/pythinker_code/app.py:102, never attached), no proxy/connectivity diagnostics, and no session-long structured tag accumulation (e.g. last endpoint/auth state). + +**Verifier note.** Claim stands as 'partial' with one factual correction: the statement 'logs go to a rotating disk file, never attached' is wrong for the export path — `pythinker export` collects recent pythinker.*.log files (session-window + export-window, 100MB cap) and bundles them under logs/ in the export ZIP, and the CLI crash path advertises this ('run pythinker export to share diagnostics'). It is true that /feedback and /report-error do not attach logs, the only in-memory ring is the 10-entry error-metadata ring, there are no proxy/connectivity diagnostics, and no session-long tag accumulation (feedback payload carries only point-in-time client/session/model metadata; Sentry extra_tags are per-event). + +**Adopt.** Add a bounded in-memory loguru sink (byte-capped ring, DEBUG/TRACE level independent of console filter) and attach its redacted snapshot to /feedback and /report-error when the user opts in to logs; add a connectivity-diagnostics section that reports which proxy env vars are set (names + redacted values); accumulate a small capped dict of session diagnostic tags (provider endpoint key, auth mode, last api status) that report_handled_error and the llm layer update, merged into the feedback payload with reserved keys protected. + +**Files.** `/feedback/src/lib.rs`, `/feedback/src/feedback_diagnostics.rs`, `src/pythinker_code/feedback.py`, `src/pythinker_code/telemetry/errors.py`, `src/pythinker_code/app.py`, `src/pythinker_code/ui/shell/slash.py` + +### `patch-file-tools/graduated-fuzzy-matching-ladder-for-edit-location-recovery` — partial, M, high + +**Today.** src/pythinker_code/tools/file/replace.py matches edit.old with exact str.count plus a single CRLF-translation fallback (_crlf_translated_edit); any whitespace drift or smart-quote mismatch hard-fails with 'old string not found', forcing a re-read + retry turn. No trailing-newline normalization in replace.py or tools/file/write.py. + +**Verifier note.** Claim confirmed as stated. The only recovery beyond exact matching is the CRLF translation fallback; no whitespace/smart-quote/indentation fuzzy ladder and no trailing-newline normalization exist anywhere in the edit path. + +**Adopt.** When exact match count is 0 after the CRLF fallback, split file and needle into lines and run a line-wise seek with the same ladder (exact → rstrip-equal → strip-equal → Unicode-punctuation-normalized); on a unique hit, replace the actual matched file slice (never the needle text) and report which relaxation fired in the tool message. Keep ambiguity semantics: >1 fuzzy hit without replace_all still errors. Add an opt-in final-newline normalization for whole-file writes. + +**Files.** `/apply-patch/src/seek_sequence.rs`, `/apply-patch/src/lib.rs`, `src/pythinker_code/tools/file/replace.py`, `src/pythinker_code/tools/file/write.py` + +### `prompts-instructions/dynamic-permissions-state-instructions-rendered-from-live-en` — missing, M, high + +**Today.** Enforcement is rich (PermissionProfile, fail-closed shell mutation/workspace-escape classifiers in src/pythinker_code/soul/permission.py; ApprovalState with session-approved actions in soul/approval.py) but prompt-side the model only sees static 'not sandboxed, be cautious' text (system.md §10) plus auto-mode guidance (auto_mode.py). The injection-provider registry (pythinkersoul.py lines 441-447: plan/auto/goal/inline/model-defense/orchestration) has no permissions provider — the model is never told whether yolo/safe-mode is active, what auto-approves, which actions are session-approved, or how the shlex-based gate segments commands; it discovers policy via failed tool calls. + +**Verifier note.** Verdict correct; one overstatement in the supporting text: the model is not entirely uninformed about what auto-approves — static descriptions of yolo/auto approval semantics exist in tools/plan/enter_description.md ('Auto-approve mode notes: Yolo mode only bypasses permission approval... In auto mode, EnterPlanMode/ExitPlanMode are approved automatically') and in the auto-mode injections ('Tool calls are auto-approved only when the current trust/safe-mode policy allows', 'Outside-workspace file writes are not auto-approved'). But none of this is rendered from live enforcement config: the injection-provider registry (soul/pythinkersoul.py, _injection_providers list at ~lines 441-460: PlanMode, GoalMode, ModelDefense, InlineCommandReminder, Orchestration, AutoMode) has no permissions provider; /yolo toggling only wire_sends UI text with no context injection (soul/slash.py lines 114-140); permission_profile_for_runtime is used solely to set the step enforcement profile (pythinkersoul.py lines 1652-1666), never serialized into the prompt; session-approved actions (approval.py) and the shlex segmentation rules are never surfaced. system.md §10 line 226 is the only always-on text ('not sandboxed... be extremely cautious'). + +**Adopt.** Add a PermissionsInjectionProvider that renders a short block from active_permission_profile + ApprovalState: current trust posture (safe-mode/yolo/auto), what is auto-approved vs always-prompted (git mutations, destructive ops), session-approved action names, and the gate's command-segmentation caveats (subshells/glued operators are not classified — write plain commands). Re-emit on /yolo //auto toggles like AUTO_DISABLED_REMINDER. Saves wasted gated calls and teaches the model to shape commands the classifier can see. + +**Files.** `src/pythinker_code/soul/permission.py`, `src/pythinker_code/soul/approval.py`, `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/soul/dynamic_injections/auto_mode.py` + +### `prompts-instructions/model-initiated-escalation-with-justification-and-suggested-` — missing, M, high + +**Today.** Approval prompts are entirely harness-initiated: the Shell tool has no justification parameter (src/pythinker_code/tools/shell/__init__.py, bash.md), ApprovalResult.feedback flows only on rejection (soul/approval.py; confirmed by memory note), and any 'always allow' scoping is chosen by the user/UI — the model can neither pre-justify a gated command nor propose a scoped reusable rule. + +**Verifier note.** Claim survives as stated. Note the Shell tool does have a `description` parameter, but it is exclusively a background-task label (defaulted via _default_background_description when run_in_background), not an approval justification, and it is not surfaced in the approval prompt. + +**Adopt.** Add an optional justification field to Shell (and other gated tools) surfaced in the approval prompt UI, plus an optional model-suggested allow-prefix that the approval runtime validates against a banned-prefix list (bare interpreters, rm/git-reset class, heredoc-containing commands) before offering a persist-this-rule option. Prompt-side: one paragraph in the Shell description telling the model when to supply each. Reduces blind approval prompts and makes persisted rules better-scoped. + +**Files.** `src/pythinker_code/tools/shell/__init__.py`, `src/pythinker_code/tools/shell/bash.md`, `src/pythinker_code/soul/approval.py`, `src/pythinker_code/approval_runtime/runtime.py` + +### `protocol-headless/stable-machine-readable-jsonl-event-stream-with-thread-turn-` — partial, M, high + +**Today.** Partial. --print --output-format stream-json (src/pythinker_code/ui/print/visualize.py JsonPrinter) emits merged assistant Message JSON, tool-result messages, PlanDisplay, ProgressNote/Suggestion and Notifications — but no session-id event at stream start, no turn started/completed/failed lifecycle events, no terminal status, no token usage (StatusUpdate with TokenUsage exists in src/pythinker_code/wire/types.py but falls into JsonPrinter's ignore branch), and no item ids linking tool start/result. A resume hint with the session id goes to stderr only at exit (src/pythinker_code/cli/__init__.py _print_resume_hint). + +**Verifier note.** Claimed verdict 'partial' is correct, but one evidence point is wrong: tool start/result lines ARE linked by ids. The merged assistant message includes tool_calls each carrying ToolCall.id (packages/pythinker-core/src/pythinker_core/message.py:174-182), and each tool-result line is a Message with tool_call_id set from the originating call (src/pythinker_code/soul/message.py:58-63, tool_result_to_message). The rest holds: TurnBegin/TurnEnd ARE wire events and ARE sent (src/pythinker_code/wire/types.py:37-60; src/pythinker_code/soul/flow_runner.py:210,214) but JsonPrinter drops them in its `case _` ignore branch, so the emitted JSONL has no turn lifecycle; StatusUpdate (wire/types.py:217-229, token_usage) is likewise dropped by JsonPrinter while the ACP frontend does consume it (src/pythinker_code/acp/session.py:186); no session-id event is emitted in-stream — the resume hint goes to the original stderr fd at exit via _emit_fatal_error (src/pythinker_code/cli/__init__.py:683-691, 1089-1092, 1117, 1172); no terminal status event exists (exit code only). + +**Adopt.** Add a v2 stream format that serializes WireMessageEnvelope events (TurnBegin/StepBegin/ToolCall/ToolResult/StatusUpdate/PlanDisplay/TurnEnd) as JSONL, prefixed by a session-started event carrying session_id/model/work_dir and terminated by a turn-completed event carrying final token usage and a terminal status (completed|failed|interrupted). Reuse the existing envelope from wire/types.py so the headless stream, wire server, and session wire-file share one schema; keep the current stream-json as legacy. + +**Files.** `src/pythinker_code/ui/print/visualize.py`, `src/pythinker_code/wire/types.py`, `/exec/src/exec_events.rs`, `/exec/src/event_processor_with_jsonl_output.rs` + +### `protocol-headless/structured-final-output-constrained-by-a-caller-supplied-jso` — missing, M, high + +**Today.** Missing. rg over src/pythinker_code finds no output-schema/response_format/structured-output plumbing in the CLI (src/pythinker_code/cli/__init__.py), print UI, or soul run path; the only json_schema hits are FastAPI internals in web/app.py and vis/app.py. + +**Verifier note.** Claim confirmed; could not refute. rg for output_schema/response_format/json_schema/structured-output across src/pythinker_code hits only FastAPI's separate_input_output_schemas (web/app.py:171, vis/app.py:52) and prose in skills/agent-creator/SKILL.md (a 'structured output contract' meaning headed markdown sections, not JSON Schema). The full CLI option list in src/pythinker_code/cli/__init__.py has no schema flag, and packages/pythinker-core has no response_format plumbing. + +**Adopt.** Add --output-schema FILE (print mode only): validate the file as JSON at startup (fail fast), then thread the schema into the final-turn prompt as a strict output contract (system-reminder instructing JSON-only final message conforming to schema) and validate the final assistant text against it with jsonschema, retrying once with the validation error before exiting nonzero. Provider-native structured-output can be layered later for models that support it. + +**Files.** `src/pythinker_code/cli/__init__.py`, `src/pythinker_code/ui/print/__init__.py`, `/exec/src/lib.rs` + +### `review-mode/interactive-review-command-with-target-presets-and-git-picke` — missing, M, high + +**Today.** No /review slash command exists in either command registry (src/pythinker_code/ui/shell/slash.py, src/pythinker_code/soul/slash.py — audited every @registry.command), and src/pythinker_code/ui/shell/selectors/ has no branch or commit picker. Review is reachable only via natural language, /skill:review-pr (src/pythinker_code/skills/review-pr/SKILL.md), or the separate `pythinker review` CLI (src/pythinker_code/cli/review.py, cli/_lazy_group.py). + +**Verifier note.** Claim CONFIRMED. No /review command is registered in either registry and no git branch/commit picker exists. One nuance the analyst missed: the Suggest tool's spec actively instructs the model to prefill '/review' (src/pythinker_code/tools/suggest/description.md:8, tools/suggest/__init__.py:14-19), yet typing it hits the unknown-command error path — so the shipped suggestion surface references a command that does not exist. Reachability list in the claim is otherwise accurate (natural language, /skill:review-pr, `pythinker review` CLI), plus reviewer subagents (agents/default/review.yaml, code_reviewer.yaml, security_reviewer.yaml) dispatched via the Agent tool. + +**Adopt.** Add a /review slash command that shows a 4-item selector (uncommitted / base branch / commit / custom). Branch and commit sub-pickers run `git branch --format` and `git log -100 --format=%h%x09%s` through the existing async git helper pattern (subagents/git_context.py:_run_git) and feed the shell's existing searchable selector component. The chosen target is resolved (see prompt-synthesis finding) and dispatched as a `review` subagent run, with results rendered through the existing report-block renderer. + +**Files.** `src/pythinker_code/ui/shell/slash.py`, `src/pythinker_code/soul/slash.py`, `src/pythinker_code/ui/shell/selectors/`, `src/pythinker_code/skills/review-pr/SKILL.md` + +### `skills-hooks-memories/posttooluse-hook-feedback-surfaced-to-the-model` — partial, M, high + +**Today.** toolset.py fires PostToolUse as fire_and_forget_trigger (soul/toolset.py:746) — results are discarded, so a hook that detects a broken build or lint failure can never tell the model. The additional_context plumbing exists in hooks/runner.py:112-126 but is only consumed for PostCompact/SessionStart-after-compact via build_hook_context_message (soul/pythinkersoul.py:2149, soul/compaction_restore.py:161); _stdout_adds_context (runner.py:121-126) explicitly restricts plain-stdout context to those two events. + +**Verifier note.** Claim confirmed (minor line drift only: PostToolUse fire-and-forget is toolset.py:743-754 not 746; consumer is pythinkersoul.py:2182-2188 not 2149). PostToolUse and PostToolUseFailure results are discarded via fire_and_forget_trigger; additional_context is consumed solely in the compaction path — build_hook_context_message (compaction_restore.py:161) over PostCompact + SessionStart(source=compact) results at pythinkersoul.py:2182-2188. _stdout_adds_context (runner.py:121-126) restricts plain-stdout context to exactly those two events. UserPromptSubmit (pythinkersoul.py:997-1010) and PreToolUse (toolset.py:625-633) results are checked for action=='block' only; their additional_context is dropped too. rg confirms additional_context consumers are only runner.py, pythinkersoul.py, compaction_restore.py. + +**Adopt.** Await PostToolUse results in toolset.py instead of fire-and-forget (keep a short timeout so slow hooks don't stall turns), and when any result carries additional_context or a block decision, append it to the ToolResult output (or as a follow-up user-role fragment) so the model sees hook feedback. Extend _stdout_adds_context / JSON parsing to honor additionalContext for PostToolUse and UserPromptSubmit, not just compaction events. + +**Files.** `src/pythinker_code/soul/toolset.py`, `src/pythinker_code/hooks/runner.py`, `src/pythinker_code/soul/pythinkersoul.py`, `/hooks/src/events/post_tool_use.rs` + +### `skills-hooks-memories/trust-gating-for-repo-scoped-hook-definitions` — missing, M, high + +**Today.** Hooks load from merged config including project scope: config.py:944 defines Config.hooks, load_config merges ~/.pythinker/config.toml → .pythinker/config.toml → config.local.toml (config.py:1030-1040), and app.py:388-391 builds HookEngine(config.hooks) unconditionally. SCOPE_LOCKED_PATHS (config.py:54-68) deliberately blocks repo-controlled statusline commands but does NOT lock ('hooks',), so a repo-controlled .pythinker/config.toml can register auto-executing shell hooks. Workspace-trust primitives exist (session_state.py TrustStateData, soul/approval.py) but are not consulted for hooks. + +**Verifier note.** Claim confirmed. Hooks merge from project scope with no trust check: Config.hooks at config.py:944, scoped merge user→project→local in load_config/_load_scoped (config.py:1030-1043), HookEngine(config.hooks) built unconditionally at app.py:388-390. SCOPE_LOCKED_PATHS (config.py:54-68) locks providers/services/feedback.api_key and four tui.statusline fields only — no ('hooks',) entry. TrustStateData exists (session_state.py:21, Field at :53) but is never consulted anywhere in hooks/* or the hook construction path (rg 'trust' over hooks/ and config.py: zero hits). + +**Adopt.** Either add ('hooks',) to SCOPE_LOCKED_PATHS (smallest fix, loses project hooks entirely) or port the trust model: hash each HookDef (event+matcher+command+timeout), persist trusted hashes in user-level state keyed by source scope, and have HookEngine skip non-user-scope handlers whose hash is untrusted/modified until the user approves via a one-time prompt (reuse the existing workspace-trust flow in soul/approval.py). Surface skipped-untrusted hooks in /hooks display. + +**Files.** `src/pythinker_code/config.py`, `src/pythinker_code/hooks/engine.py`, `src/pythinker_code/app.py`, `/hooks/src/engine/discovery.rs` + +### `tools-registry-codemode/deferred-tool-loading-with-on-demand-tool-search` — missing, M, high + +**Today.** Every connected MCP tool's full schema is always advertised via PythinkerToolset.tools (src/pythinker_code/soul/toolset.py:402-404), subject only to the policy visibility filter and manual hide/unhide. defer_mcp_tool_loading (toolset.py:825) defers only server connection at startup, not schema exposure; no tool-search facility exists (rg tool_search/defer_loading is empty in src). + +**Verifier note.** Claim survives. PythinkerToolset.tools advertises the full schema of every non-hidden, policy-visible tool each step; defer_mcp_tool_loading defers only the server connection until shell start, after which all tool schemas are exposed. No tool-search tool exists in tools/ and no schema-deferral mechanism exists. + +**Adopt.** Add a per-server defer flag (or auto-trigger above N tools): deferred tools register but enter _hidden_tools with a search-index entry (name + description + schema property names). A built-in ToolSearch tool matches the index, unhides the top matches, and returns their names/descriptions so the next step can call them. Pairs naturally with the existing hide/unhide machinery. + +**Files.** `src/pythinker_code/soul/toolset.py`, `/tools/src/tool_search.rs`, `/tools/src/tool_discovery.rs` + +### `tools-registry-codemode/foreign-tool-schema-sanitization-and-budgeted-compaction` — partial, M, high + +**Today.** mcp_tool.inputSchema is passed verbatim into the tool definition (src/pythinker_code/soul/toolset.py:1119, MCPTool.__init__) and flows raw into every provider request; WireExternalTool likewise (toolset.py:1200). No sanitize/prune/compact layer exists (rg sanitiz/inputSchema across src and pythinker-core). A malformed or 100KB schema from a third-party server can break provider calls or permanently tax context. + +**Verifier note.** REFUTED as 'missing' — a narrow sanitization layer already exists. ensure_property_types() fills missing `type` keywords in nested property schemas (explicitly to keep loose MCP-server schemas working) and is applied to every tool schema sent through the Pythinker platform chat provider; deref_json_schema() normalizes internal CallableTool2 schemas. What IS missing: this sanitization is provider-specific (provider adapters send tool.parameters verbatim), and there is no size budgeting, pruning, or compaction anywhere — a 100KB schema still passes through untaxed. Correct verdict: partial. + +**Adopt.** Add a normalize_tool_schema() applied at MCPTool/WireExternalTool registration: fill missing type, const→enum, strip keywords providers reject, prune unreferenced $defs, and when serialized size exceeds a budget run lossy passes (drop nested descriptions, then collapse deep structure to permissive objects) with a warning log. Pure function, easy to fixture-test against real server schemas. + +**Files.** `src/pythinker_code/soul/toolset.py`, `/tools/src/json_schema.rs` + +## Tier 2 — high value, L effort (6 items) + +### `exec-safety/os-sandboxed-command-execution-with-escalation-on-denial-lif` — missing, L, high + +**Today.** Missing. Shell commands are spawned directly via pythinker_host.exec with no OS confinement (src/pythinker_code/tools/shell/__init__.py:366); all containment is advisory pre-classification in permission.py plus approval gating, which the comments themselves call 'not a shell sandbox'. No sandbox integration exists anywhere in src/pythinker_code (grep confirmed). + +**Verifier note.** Claim confirmed. Commands are spawned directly via pythinker_host.exec with the shell binary and -c/-command — no seatbelt/landlock/bwrap/seccomp/firejail integration anywhere in src/pythinker_code (the only 'sandbox' hits are jinja2.sandbox imports for template rendering and the permission.py comment explicitly disclaiming sandbox status). Containment is purely advisory pre-classification (permission.py profile gates) plus approval gating; no escalation-on-denial retry lifecycle and no network-off default at the OS level. + +**Adopt.** Add a sandbox transform layer in the Shell tool: on macOS wrap the argv in /usr/bin/sandbox-exec with a generated profile (deny default; allow read broadly, write only to workspace/additional_dirs/tmp; deny network unless the profile allows it), on Linux wrap in bwrap with equivalent binds when available; degrade gracefully to today's direct exec when unsupported. On a sandbox-denial exit signature, return a structured error inviting one escalation: re-request approval flagged 'will run unsandboxed', then re-run without the wrapper. This converts the heuristic read-only/offline guarantees of restricted profiles into kernel enforcement. + +**Files.** `src/pythinker_code/tools/shell/__init__.py`, `src/pythinker_code/soul/permission.py`, `/sandboxing/src/manager.rs`, `/sandboxing/src/seatbelt.rs` + +### `exec-safety/user-extensible-declarative-exec-policy-with-allow-prompt-fo` — partial, L, high + +**Today.** Absent. All command classification is hardcoded Python (_MUTATING_COMMANDS, _GIT_MUTATIONS, _PACKAGE_MANAGER_MUTATIONS etc. in src/pythinker_code/soul/permission.py); config.py exposes no shell-command allow/forbid settings (only web.allowed_domains); execution_profiles.py gives coarse per-tool ask/deny/allow modes, not per-command rules. + +**Verifier note.** Overstated as 'Absent'. Pythinker DOES have a user-extensible forbidden tier: config.py:944 exposes hooks: list[HookDef]; a PreToolUse hook (regex matcher on tool name, full tool_input JSON on stdin) can block any Shell call with a reason via exit-2 or permissionDecision=deny, fired before tool execution in soul/toolset.py:611-633. Documented at docs/en/customization/hooks.md. However the rest of the claim is correct: it is imperative (arbitrary shell command), deny-only (no allow/auto-approve tier, no prompt tier), fail-open on error/timeout (hooks/runner.py:31,49,59), with no declarative per-command rules and no self-testing. Built-in classification is hardcoded Python and config.py has no shell allow/forbid lists (only web.allowed_domains at :567); execution_profiles.py gives only coarse per-tool deny/ask/allow. + +**Adopt.** Add a small policy schema (TOML/YAML under ~/.pythinker and project .pythinker, repo-scope locked like statusline.command) of token-prefix rules with decision allow|prompt|forbidden and optional justification; validate match/not_match examples at load and refuse the file on failure. Evaluate before the heuristic classifiers in check_shell_command_allowed/Approval.request: forbidden returns a ToolError carrying the justification, prompt forces a fresh approval even under session rules, allow feeds the prompt-elision path from the safelist finding. Strictest decision wins across matches; unmatched commands fall through to today's heuristics unchanged. + +**Files.** `src/pythinker_code/soul/permission.py`, `src/pythinker_code/config.py`, `/execpolicy/src/policy.rs`, `/execpolicy/README.md` + +### `multi-agent/mid-run-steering-of-live-child-agents-send-input-with-option` — missing, L, high + +**Today.** Missing for agents. src/pythinker_code/subagents/runner.py busy_resume_message() hard-rejects resume of any running instance; src/pythinker_code/tools/background/input.md states TaskInput is 'only for non-terminal bash background tasks'; src/pythinker_code/background/manager.py write_input() is bash-stdin only. The only steering path is the coarse TaskStop-kill then Agent(resume=...) after terminal state. + +**Verifier note.** Claim upheld for child agents. One nuance the analyst missed: a steering primitive DOES exist at root level — PythinkerSoul.steer() (src/pythinker_code/soul/pythinkersoul.py:923) with a pending-steer queue (_consume_pending_steers/_inject_steer, lines 927/946) wired from the user via wire/server.py _handle_steer and the shell UI. It is user->root only; nothing routes steers parent->child. + +**Adopt.** Add an AgentInput (or Agent param message_to=) path: for a running_background agent, enqueue the message into a per-agent inbox file in SubagentStore; BackgroundAgentRunner checks the inbox between soul steps (or via a cancellation+requeue wrapper around run_soul) and injects it as the next user message, with an optional interrupt flag that cancels the in-flight step first. Reuse the existing wire/SubagentEvent plumbing for begin/end visibility and keep busy_resume_message pointing at the new tool. + +**Files.** `src/pythinker_code/subagents/runner.py`, `src/pythinker_code/background/agent_runner.py`, `src/pythinker_code/background/manager.py`, `src/pythinker_code/tools/background/input.md` + +### `review-mode/llm-approval-guardian-auto-review-of-on-request-approvals-wi` — missing, L, high + +**Today.** Approval gating is deterministic: profile rules, never-auto-approve lists for boundary-crossing actions, and yolo/auto flags (src/pythinker_code/soul/approval.py — no LLM path). The nearest seed is the blind-first decision advisor used for auto-mode AskUserQuestion deliberation (src/pythinker_code/soul/deliberation.py), which proves the tool-less single-call advisor pattern but never gates tool approvals. + +**Verifier note.** Verdict CONFIRMED — no LLM ever reviews a tool-approval request — but the claimed state understates pythinker's deterministic coverage of this capability's fail-closed/backstop half. (1) Approval.deliberation_gate (soul/approval.py:384-447) is a destructive-action auto-approval guardian: under auto/yolo it bounces destructive calls once per (execution-context, generation) fingerprint with explicit fail-closed fallback when no deliberation scope is bound ('Fail CLOSED: keep bouncing'), gating AHEAD of the yolo bypass — the 'deliberation' is done by the main agent on re-issue, not a separate LLM. (2) PreToolUse hooks can deny tool calls with structured permissionDecision=deny and fail-closed sink handling (hooks/runner.py:83-86, hooks/engine.py:318-324) — user-scripted, not LLM. (3) The claim's deliberation.py characterization is accurate: blind_advisor_verdict is a tool-less single-call advisor scoped to auto-mode AskUserQuestion only (consumed by tools/ask_user), and it advises rather than gates. The only circuit breaker found is the degenerate-loop backstop (pythinkersoul.py:1898-1918, max_consecutive_failures) which counts all-error steps, not approval outcomes. So: LLM auto-review of approvals + approval-scoped circuit breakers = genuinely absent; fail-closed destructive backstop = present deterministically. + +**Adopt.** Extend the deliberation.py pattern into an opt-in approval advisor for auto/yolo modes: when a gated action would otherwise auto-approve (or would interrupt an unattended run), make one tool-less LLM call with a bounded transcript (reuse existing pruning utilities for caps) and a strict-JSON verdict; fail closed to the normal human prompt on timeout/malformed output. Add per-turn consecutive-denial and windowed-denial circuit breakers so a misfiring advisor degrades to human prompting instead of looping, and tag manual overrides of advisor denials in history so the model knows the user explicitly authorized that exact action. + +**Files.** `src/pythinker_code/soul/approval.py`, `src/pythinker_code/soul/deliberation.py`, `src/pythinker_code/soul/permission.py` + +### `skills-hooks-memories/llm-driven-two-phase-durable-memory-pipeline-extraction-cons` — missing, L, high + +**Today.** Pythinker's durable-memory profile is heuristic and synchronous: memory/harvest.py extracts only regex-prefixed lines (decision:/blocker:/next:/evidence:) from compaction-dropped assistant messages; memory/recap.py writes fixed-schema JOURNAL.md recaps; memory/consolidation.py only stages existing scratch/journal blocks into an approval-gated inbox (no LLM, no synthesis, no forgetting); config.py:470-519 gates these flags. Nothing reads past rollouts with a model, classifies outcomes, or consolidates/prunes MEMORY.md content. + +**Verifier note.** Claim confirmed for the stated capability; one nuance to record. Extraction is regex-only (memory/harvest.py _NOTE_RE matching decision:/blocker:/next:/evidence: in dropped assistant messages); recap.py writes fixed-schema journal recaps; consolidation.py generate_inbox_candidates (:56-99) only stages existing scratch/journal blocks into an approval-gated inbox — no model call anywhere in memory/ (rg confirms). Flags at config.py:470-519. Nuance: in-session model-driven curation DOES exist — the root-only Memory tool (tools/memory/__init__.py) supports add/replace/remove against MEMORY.md/USER.md, MEMORY_CHAR_LIMIT=2200 (project_memory.py:38) forces curation when full, and the journal is count-pruned to 100 entries (:315). But nothing reads past rollouts with a model, classifies outcomes, or runs background consolidation/forgetting, so the two-phase-pipeline verdict 'missing' stands. + +**Adopt.** Adopt incrementally: (1) port the Phase-1 prompt design (no-op gate, outcome triage, preference-signal/evidence rules, redaction) as an opt-in background extraction over recent idle sessions using the existing subagent runner, writing candidates into the existing approval-gated inbox so the consent model is preserved; (2) later add a consolidation pass (offline subagent profile, workspace-write-only) that merges approved candidates into MEMORY.md with diff-reviewable output, using session-store leases (multi-instance invariants already exist) and a budget guard that skips the pipeline when recent provider usage is high. Keep the inbox approval gate as the trust boundary instead of fully autonomous writes. + +**Files.** `src/pythinker_code/memory/harvest.py`, `src/pythinker_code/memory/consolidation.py`, `src/pythinker_code/memory/recap.py`, `/memories/README.md`, `/memories/write/templates/memories/stage_one_system.md`, `/memories/write/templates/memories/consolidation.md` + +### `tools-registry-codemode/unified-interactive-pty-exec-sessions-persistent-process-ids` — partial, L, high + +**Today.** The background subsystem (src/pythinker_code/background/worker.py, manager.py; src/pythinker_code/tools/background/__init__.py) provides TaskInput stdin writes, TaskOutput with byte-offset paging + blocking waits + anti-poll escalation hints, TaskStop/TaskList/TaskHandoff. But processes run on plain pipes (worker.py:181 asyncio.subprocess.PIPE), not a PTY; foreground Shell closes stdin immediately (src/pythinker_code/tools/shell/__init__.py:370); there is no interrupt-without-kill and no persistent interactive shell session — one command per task. + +**Verifier note.** Claim survives as stated. Background tasks have persistent ids, TaskInput stdin writes, TaskOutput paging, TaskStop — but on plain pipes, stop is SIGTERM with SIGKILL escalation (no interrupt-without-kill), foreground Shell closes stdin immediately, and each task is one command (no persistent shell session). No PTY code exists anywhere in src/ or packages/ (rg openpty|ptyprocess|TIOCSWINSZ|conpty: zero hits). + +**Adopt.** Add a PTY-backed task kind in background/worker.py (pty.openpty on POSIX, pywinpty on Windows) selected via a tty flag at creation; extend TaskInput with an interrupt/control-character path and clamp poll windows like the reference; cap concurrent interactive sessions with last-used pruning. This unlocks REPLs, debuggers, password prompts and watch UIs for the agent. + +**Files.** `src/pythinker_code/background/worker.py`, `src/pythinker_code/tools/background/__init__.py`, `src/pythinker_code/tools/shell/__init__.py`, `/core/src/unified_exec/process_manager.rs` + +## Tier 3 — medium value (62 items) + +### `config-features/per-key-runtime-config-override-layer` — partial, S, medium + +**Today.** Missing. --config replaces the entire config with inline text validated standalone (src/pythinker_code/cli/__init__.py:791 load_config_from_string, defaults for everything else), and --config-file bypasses the scope pipeline entirely (load_config explicit-path branch in config.py). There is no way to tweak one key for one run while keeping the resolved user/project config. + +**Verifier note.** REFUTED as 'missing' — downgrade to partial. The claim's core assertion ('no way to tweak one key for one run while keeping the resolved user/project config') is factually wrong: the PYTHINKER_* env overlay applies per-key overrides ON TOP of the fully resolved user/project/local merge for 17 mapped keys (including the nested tui.statusline.enabled), and dedicated CLI flags mutate individual loop_control keys after load_config. What IS missing: a generic mechanism for arbitrary dotted keys (no --set key=value); env coverage is limited to the ENV_FIELD_MAP allowlist. The claim's characterization of --config/--config-file is accurate but incomplete. + +**Adopt.** Add a repeatable `--set key=value` option parsed into a nested dict (dotted paths, TOML-ish value coercion) and merged in _load_scoped as the final, highest-precedence scope with provenance 'cli --set'; reuse the unknown-key warning machinery from the strict-config finding. + +**Files.** `src/pythinker_code/cli/__init__.py`, `src/pythinker_code/config.py`, `/config/src/overrides.rs` + +### `config-features/post-load-per-field-origin-map-exposed-to-the-user` — partial, S, medium + +**Today.** Partial. _type_based_merge in src/pythinker_code/config.py builds exactly this provenance map, but it is a local variable used only to enrich ValidationError messages and is discarded after validation; only source_scopes (scope -> file path) survives on Config, and the settings panel (ui/shell/selectors/settings.py:274) shows just source_file. + +**Verifier note.** Claim confirmed. The per-field provenance map is built during merge but is a local variable discarded after validation; only the coarse scope->file map survives on Config, and no UI surface shows per-field origins. + +**Adopt.** Retain the provenance dict on Config as an excluded field; show 'set by project scope (.pythinker/config.toml)' next to non-default values in the /settings panel and in a `config origin ` query. Nearly free since the map is already computed. + +**Files.** `src/pythinker_code/config.py`, `src/pythinker_code/ui/shell/selectors/settings.py`, `/config/src/fingerprint.rs` + +### `config-features/sanitize-and-warn-handling-for-denied-project-scope-config-k` — partial, S, medium + +**Today.** Partial. SCOPE_LOCKED_PATHS in src/pythinker_code/config.py covers the right keys (providers, services, statusline command) but _check_scope_locks hard-raises ConfigError, and _read_toml raises on invalid project TOML — a repo-controlled file can therefore prevent pythinker from launching in that directory (a friction/DoS vector the reference avoids). + +**Verifier note.** Claim confirmed. Scope locks exist and cover the claimed keys, but enforcement is hard-fail (ConfigError raise), not sanitize-and-warn; invalid project TOML also hard-fails, so a repo-controlled .pythinker/config.toml can block startup in that directory. + +**Adopt.** In the GUARD step, strip locked paths from project/local dicts and collect startup warnings ('ignored providers in .pythinker/config.toml; move to ~/.pythinker/config.toml') instead of raising; degrade unparseable project/local TOML to an empty scope with a warning. Keep hard failure for the user scope only. Surface accumulated warnings once in the shell banner. + +**Files.** `src/pythinker_code/config.py`, `/config/src/loader/mod.rs` + +### `context-mgmt/calibrated-token-estimation-for-non-text-payloads-tool-call-` — partial, S, medium + +**Today.** Partial. The pending-estimate mechanism exists (Context._pending_token_estimate, token_count_with_pending in src/pythinker_code/soul/context.py:29,92), but estimate_text_tokens counts only TextPart chars/4 (src/pythinker_code/soul/compaction.py:44-53): tool_call arguments and image/media parts contribute zero, so large un-sampled reads or media can silently undercount until the next API usage report, delaying compaction past the trigger. + +**Verifier note.** Claim confirmed as partial. Add: ThinkPart text is also uncounted, not just tool-call args and media. + +**Adopt.** Extend estimate_text_tokens to include len(tool_call.function.arguments)//4 per tool call and a flat per-media-part constant (e.g. ~1800 tokens per image, matching common provider downscaling), keeping the chars/4 heuristic elsewhere. One function, existing tests in tests/ cover the call sites. + +**Files.** `src/pythinker_code/soul/compaction.py`, `src/pythinker_code/soul/context.py` + +### `context-mgmt/model-visible-context-budget-signals-threshold-crossing-budg` — missing, S, medium + +**Today.** Missing. Context usage is surfaced only in the TUI footer (src/pythinker_code/ui/shell/components/footer.py:154-212) and StatusSnapshot (soul/pythinkersoul.py:864-879); no dynamic-injection provider in src/pythinker_code/soul/dynamic_injections/ exposes remaining tokens to the model, and no tool under src/pythinker_code/tools/ reports it. + +**Verifier note.** Claim confirmed. Do not confuse ContextBudget in dynamic_injection.py — that is the budget for sizing injections, not a signal exposed to the model. + +**Adopt.** Add a TokenBudgetInjectionProvider to soul/dynamic_injections/ that compares token_count_with_pending against max_context_size each step and, on first crossing of each threshold (state reset in on_context_compacted), emits a one-line system-reminder with tokens remaining. Optionally add a tiny read-only tool returning the same figure. Fits the existing InjectionCandidate budget framework directly. + +**Files.** `src/pythinker_code/soul/dynamic_injection.py`, `src/pythinker_code/soul/dynamic_injections/`, `src/pythinker_code/ui/shell/components/footer.py` + +### `context-mgmt/verbatim-user-intent-retention-across-compaction-within-a-to` — partial, S, medium + +**Today.** Partial. SimpleCompaction preserves only the last 2 user/assistant messages by count, summary-first (src/pythinker_code/soul/compaction.py:146-238); older user messages survive only as summarized bullets in the Goal/Constraints sections of prompts/compact.md. Post-compaction restore re-injects file names, skills, and task snapshots (src/pythinker_code/soul/compaction_restore.py) but not raw user text. + +**Verifier note.** Claim confirmed as partial. Two partial mitigations worth adding: (1) the cheap prune tier preserves EVERY non-tool message verbatim (it only elides stale tool-result bodies), deferring the lossy summary; (2) when /goal is active, GoalModeInjectionProvider re-injects the goal objective text verbatim after compaction. Neither is a token-budgeted verbatim retention of raw user messages through full compaction. + +**Adopt.** In SimpleCompaction.prepare/compact, collect user-role messages (excluding prior compaction-summary messages) from the to_compact slice and append them verbatim after the summary, newest-first within a configurable token budget (e.g. 8-20k via loop_control), truncating the oldest selected message to fit. Skip messages already inside the preserved tail. + +**Files.** `src/pythinker_code/soul/compaction.py`, `src/pythinker_code/prompts/compact.md` + +### `core-loop/model-visible-interrupted-turn-history-marker` — partial, S, medium + +**Today.** Partial. Cancellation during the tool phase persists the assistant message plus per-call synthetic 'Tool call interrupted by user' results via shielded writes (pythinkersoul.py:1761-1785). But cancellation during the LLM stream leaves no trace: run_soul cancels the soul task (soul/__init__.py:235-241), partial assistant text is dropped, and the next turn's model gets no signal the prior turn was aborted mid-answer. + +**Verifier note.** Claim confirmed exactly as stated. Tool-phase cancellation persists markers; LLM-stream-phase cancellation leaves no trace in model-visible history. The shell-side RunCancelled handler only prints 'Interrupted by user' and kills turn-spawned background tasks — it writes nothing to context. + +**Adopt.** On RunCancelled where no tool-phase marker was written, append a short system_reminder ('your previous response was interrupted by the user before completion; do not assume it was delivered') to the context via a shielded write before the turn unwinds. Optionally persist the partial streamed assistant text above the marker so work is not silently lost. + +**Files.** `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/soul/__init__.py` + +### `core-loop/token-budget-remaining-notices-injected-into-model-context-a` — missing, S, medium + +**Today.** Missing. Context usage is surfaced to the USER via StatusUpdate/statusline (pythinkersoul.py:1738-1752, ui/shell/stats_pricing.py) but the model itself is never told how much budget remains; it only experiences sudden prune/compaction. No injection provider covers this (checked soul/dynamic_injections/*). + +**Verifier note.** Claim confirmed. No injection provider or system-reminder path tells the model how much context budget remains. The registered providers are PlanMode, GoalMode, ModelDefense, InlineCommandReminder, Orchestration (new, orchestration-shape guidance only — not budget), and AutoMode. Context usage flows only to the USER via StatusUpdate/statusline. The closest-sounding feature, subagents/usage.py 'budget-visible' child token roll-ups, surfaces child SPEND to the parent agent, not remaining context budget. + +**Adopt.** Add a small self-filtering DynamicInjectionProvider that tracks the last context_usage ratio, and on crossing 25/50/75% emits a one-shot system_reminder with approximate tokens remaining and a hint to leave durable notes (todo/scratchpad) before compaction. Re-arm thresholds in on_context_compacted. + +**Files.** `src/pythinker_code/soul/dynamic_injection.py`, `src/pythinker_code/soul/dynamic_injections/orchestration.py`, `src/pythinker_code/soul/pythinkersoul.py` + +### `exec-safety/process-self-hardening-at-startup-core-dumps-off-debugger-at` — partial, S, medium + +**Today.** Partial. Child-process env hygiene exists — internal session tokens always dropped, secret-shaped vars scrubbed for restricted profiles, PyInstaller LD_ restoration (src/pythinker_code/utils/subprocess_env.py) — but the agent process itself is unhardened: no RLIMIT_CORE=0, no dumpable/ptrace controls, and LD_/DYLD_ are not stripped from its own environment outside the frozen-Linux case. + +**Verifier note.** Claim confirmed. Child-process hygiene exists in utils/subprocess_env.py (PYTHINKER_WEB/VIS session tokens always dropped at :22-27, scrub_secret_env heuristic at :102-119 for restricted profiles, PyInstaller LD_LIBRARY_PATH/LD_PRELOAD restore at :17-20/:53-59). But there is no self-hardening of the agent process: zero hits for RLIMIT_CORE/setrlimit/PR_SET_DUMPABLE/PT_DENY_ATTACH across src/; the only prctl usage is PR_SET_PDEATHSIG in utils/sleep_inhibitor.py:169-185, which is child-lifetime management, not anti-debug/core-dump hardening. + +**Adopt.** At CLI entry (before config/auth load): resource.setrlimit(RLIMIT_CORE, (0, 0)); on Linux call prctl(PR_SET_DUMPABLE, 0) via ctypes; strip DYLD_*/LD_* from os.environ (after capturing any PyInstaller _ORIG values subprocess_env needs). A core dump of the agent process contains API keys and conversation data, so this is cheap leak prevention; keep it best-effort (log, never abort) since Python startup differs from a native pre-main hook. + +**Files.** `src/pythinker_code/utils/subprocess_env.py`, `src/pythinker_code/cli/__init__.py`, `/process-hardening/src/lib.rs` + +### `mcp/bounded-retry-backoff-for-retryable-http-transport-initializ` — missing, S, medium + +**Today.** Missing. _connect_server (soul/toolset.py) makes a single connect attempt per server; any transient failure (gateway blip, 429) permanently marks the server 'failed' for the whole session with no retry and no per-server reconnect command (/reload restarts the session wholesale). + +**Verifier note.** Verdict stands as claimed, and the project's own progress log confirms it: _connect_server makes a single attempt (status pending -> connecting -> connected|failed, no retry loop; rg for retry/backoff/reconnect in toolset.py and all of src returns zero MCP hits). Granular per-server /mcp reconnect/disconnect was explicitly scoped as mcpext-2(b) and DEFERRED ('/reload covers coarsely') per tasks/agent-enhancement-remaining-plan.md. + +**Adopt.** Wrap the connect attempt for remote (http/sse) servers in a small retry loop (2 retries, 250ms/1s delays, all inside the startup-timeout deadline), retrying only on connection errors and 408/429/5xx-shaped exceptions, never on auth errors. Optionally add a `/mcp reconnect ` action to re-attempt a failed server without a session reload. + +**Files.** `src/pythinker_code/soul/toolset.py`, `/rmcp-client/src/streamable_http_retry.rs` + +### `mcp/mcp-resources-and-prompts-surfaced-to-the-model-list-read-wi` — partial, S, medium + +**Today.** Partial. tools/mcp_resource/__init__.py provides ListMcpResources and ReadMcpResource (with untrusted-data wrapping and binary placeholders), and resources/prompts are captured at connect with METHOD_NOT_FOUND-aware discovery (_discover_optional_capability in soul/toolset.py). Gaps: resource templates are not listed, prompts are listed but cannot be fetched/invoked (no get_prompt anywhere), and the inventory is frozen at connect time. + +**Verifier note.** Verdict and details fully confirmed — this is the mcpext-1 arc, landed 2026-06-08 (commit 4a8424e0 per tasks/agent-enhancement-remaining-plan.md:287). ListMcpResources/ReadMcpResource exist with untrusted-data wrapping (builder.mark_untrusted) and binary placeholders; resources/prompts captured at connect via _discover_optional_capability with METHOD_NOT_FOUND awareness. Confirmed gaps exactly as claimed: no list_resource_templates anywhere, prompts are listed (prompt.name + description) but never fetched — zero get_prompt hits in src — and the inventory is frozen at connect time. + +**Adopt.** Add a GetMcpPrompt tool (or surface server prompts as parameterized slash commands) using fastmcp's get_prompt, and include resource templates in ListMcpResources output; re-list on demand inside the tools rather than serving the connect-time snapshot so late-published resources appear. + +**Files.** `src/pythinker_code/tools/mcp_resource/__init__.py`, `src/pythinker_code/soul/toolset.py`, `/mcp/src/connection_manager.rs` + +### `mcp/required-server-gating-at-session-init` — missing, S, medium + +**Today.** Missing. MCPServerInfo (soul/toolset.py) has no required/optional distinction; _connect raises MCPRuntimeError if ANY server fails, but only when wait_for_mcp_tools is awaited — in normal background mode failures degrade to a toast and the session proceeds regardless. mcp.json schema (cli/mcp.py) has no required field. + +**Verifier note.** Verdict stands: no required/optional field exists anywhere (MCPServerInfo has only status/client/tools/resources/prompts; mcp.json schema in cli/mcp.py writes only command/args/env or url/transport/headers/auth). However the claimed mechanics are wrong in one material way: wait_for_mcp_tools IS awaited in normal background mode — PythinkerSoul._agent_loop awaits wait_for_background_mcp_loading() at the start of every turn, so a failed server does not merely toast; the MCPRuntimeError raised by _connect re-raises into the first agent turn (then the task is cleared in wait_for_mcp_tools' finally and subsequent turns proceed without the failed server). Effectively every server is hard-surfaced once, then silently optional — still no required/optional distinction. + +**Adopt.** Add an optional `required: true` per-server key in mcp.json; on session start (or first turn), await only required servers and surface a hard, aggregated error if any failed, leaving optional servers best-effort. Change MCPRuntimeError raising to cover only required servers so optional failures stop poisoning wait_for_mcp_tools. + +**Files.** `src/pythinker_code/soul/toolset.py`, `src/pythinker_code/cli/mcp.py`, `/mcp/src/connection_manager.rs` + +### `multi-agent/multi-target-wait-primitive-returning-per-agent-status-map` — partial, S, medium + +**Today.** Partial. TaskOutput(block=true) in src/pythinker_code/tools/background/__init__.py waits on exactly one task; BackgroundTaskManager.wait() (src/pythinker_code/background/manager.py:495) is single-task; a session-wide completion_event and automatic completion notifications already exist, plus poll-escalation counters (note_nonblocking_poll/note_blocking_timeout) that discourage serial polling. + +**Verifier note.** Claim upheld, but partial credit is larger than stated: (1) TaskList tool (src/pythinker_code/tools/background/__init__.py:239-270) returns a status snapshot of all active/terminal tasks in one call; (2) foreground RunAgents already does a multi-target gather — asyncio.gather over all children with per-child results returned inline (src/pythinker_code/tools/agent/__init__.py:756, summarized via subagents/usage.py summarize_batch/aggregate_findings). What is genuinely absent is a blocking multi-target wait over arbitrary already-running task ids. + +**Adopt.** Extend TaskOutput (or add TaskWait) to accept task_ids: list[str]; loop on the manager's completion_event with a deadline, return on the first task reaching a terminal status (collecting any others already terminal), and emit a per-task status map plus timed_out. Treat unknown ids as status=not_found entries rather than tool errors. + +**Files.** `src/pythinker_code/tools/background/__init__.py`, `src/pythinker_code/background/manager.py` + +### `observability-feedback/http-response-debug-context-extraction-on-provider-api-error` — partial, S, medium + +**Today.** Partial. The soul reads `request_id` off exceptions when the provider SDK exposes it and logs it (src/pythinker_code/soul/pythinkersoul.py:1450-1457), and api_error events already carry only error_type/status/model family — no bodies (pythinkersoul.py:1466-1476). But there is no generic header-level extraction for SDKs that don't surface request_id, no gateway ray capture, despite an httpx response hook already existing for the rate-limit cache (src/pythinker_code/llm.py:139). + +**Verifier note.** Verdict 'partial' stands but the claimed state understates what exists: the sub-claim 'no generic header-level extraction for SDKs that don't surface request_id' is wrong. pythinker-core's error conversion reads x-request-id directly off response headers in two places — for provider-SDK status errors AND for raw httpx errors that leak through streaming — and additionally captures the parsed response BODY on APIStatusError (.body), which the shell UI uses to surface structured 429 usage-limit detail. The soul then logs request_id. Still genuinely missing: gateway-ray (cf-ray) capture, and attaching request_id/headers to the api_error telemetry event (which carries only error_type/status/model family). + +**Adopt.** Extend the existing httpx response event hook to retain the last error response's request-id/ray/auth headers per provider in a tiny process cache; on classify_api_error, merge that context into the api_error event, the user-facing failure line, and the recent-errors ring entry. Keep the status-only message policy for exported telemetry. + +**Files.** `/response-debug-context/src/lib.rs`, `src/pythinker_code/llm.py`, `src/pythinker_code/soul/pythinkersoul.py` + +### `patch-file-tools/auto-create-missing-parent-directories-on-file-write` — partial, S, medium + +**Today.** src/pythinker_code/tools/file/write.py returns a hard ToolError ('parent directory does not exist') when the parent is missing, costing the model a mkdir round-trip; the mkdir(parents=True) courtesy exists only for plan-file writes (line 126). + +**Verifier note.** Claim confirmed as stated. WriteFile hard-errors on missing parents; mkdir(parents=True) exists only for the plan-file path. + +**Adopt.** In WriteFile, after approval (the prompt already displays the full path, mitigating typo risk), replace the parent-exists error with mkdir(parents=True, exist_ok=True) before writing; keep the error only when the parent path exists but is not a directory. Mention created directories in the success message so the transcript stays auditable. + +**Files.** `/apply-patch/src/lib.rs`, `src/pythinker_code/tools/file/write.py` + +### `persistence-resume/session-provenance-metadata-captured-at-creation-git-snapsho` — missing, S, medium + +**Today.** Missing. The wire.jsonl header records only protocol_version (src/pythinker_code/wire/file.py WireFileMetadata); SessionState has no provenance fields at all (src/pythinker_code/session_state.py); fork_session records lineage only as a 'Fork: ' prefix with no machine-readable forked_from id (src/pythinker_code/session_fork.py:333-343); subagent parentage exists only via the subagents/ dir layout. + +**Verifier note.** Claim confirmed. SessionState (src/pythinker_code/session_state.py:50-72) has no provenance fields; the wire header WireFileMetadata carries only protocol_version (src/pythinker_code/wire/file.py:20-26); fork_session sets lineage only as a title string f'{title_prefix}: {source_title}' with no machine-readable forked_from id (src/pythinker_code/session_fork.py:333-343); rg for forked_from/parent_session/lineage finds nothing. Two near-misses worth noting that do NOT satisfy the capability: (a) per-step model_name and provider_key are recorded in wire StatusUpdate records (src/pythinker_code/wire/types.py:233-235), so model identity is recoverable from the transcript per turn but is not creation-time session metadata; (b) the git branch is computed only for the welcome banner display via _safe_git_branch (src/pythinker_code/app.py:45-63, 789-791) and never persisted. Web Session model (web/models.py) also has no provenance fields. + +**Adopt.** Extend SessionState with created_at, cli_version, model_id, source (cli/web/acp), forked_from_id, and a git snapshot (branch, commit, origin URL via one subprocess at Session.create, skipped outside repos). Set forked_from_id in fork_session and parent linkage when subagent sessions are materialized. Surface branch/commit in the session picker, web list, and /recap; lineage enables fork grouping and provenance during debugging (which commit a session's edits were made against). + +**Files.** `src/pythinker_code/session_state.py`, `src/pythinker_code/session.py`, `src/pythinker_code/session_fork.py`, `<ref>/thread-store/src/thread_metadata_sync.rs`, `<ref>/rollout/src/recorder.rs` + +### `prompts-instructions/ambition-vs-precision-calibration-for-greenfield-vs-existing` — partial, S, medium + +**Today.** Only the precision half exists: Rule 7 'smallest complete change' and §6 'no features beyond what was asked' apply unconditionally (system.md), which can produce flat, minimal output on greenfield 'build me X' asks where users expect creative completeness. + +**Verifier note.** Claimed verdict label (partial) ends up right, but the claimed state is factually wrong: 'only the precision half exists' is refuted. The exact calibration ships verbatim in src/pythinker_code/prompts/best_practices.md line 134 (Final answers section): 'Ambition vs. precision: for brand-new projects, be ambitious and demonstrate creativity. In an existing codebase, do exactly what the user asks with surgical precision — no renaming files or variables, no relocating code, no unrequested improvements.' The real gap is narrower than claimed: this rule is opt-in — injected only when the user runs /best-practices (soul/slash.py lines 301-324, session-scoped system message; rg shows prompts.BEST_PRACTICES is referenced nowhere else) — while the always-on system.md applies Rule 7 'Smallest complete change' (line 30) and §6 'No features beyond what was asked' (line 150) unconditionally, with no greenfield carve-out in the default prompt. + +**Adopt.** Add 2-3 sentences to §3 or §6 scoping the minimalism rules to existing codebases, and granting judicious ambition (sensible extras, polished defaults, stated as assumptions) for from-scratch projects with vague scope. Pure prompt edit; re-pin phrase-guard tests. + +**Files.** `src/pythinker_code/agents/default/system.md` + +### `prompts-instructions/uniform-mode-transition-semantics-and-a-pair-programming-col` — partial, S, medium + +**Today.** Pythinker has plan mode (tool-enforced read-only, persisted, periodic reminders — soul/dynamic_injections/plan_mode.py), auto mode with an explicit disable-reminder canceling prior guidance (auto_mode.py AUTO_DISABLED_REMINDER), goal mode (goal_mode.py), and the new orchestration provider defers to stronger modes (orchestration.py _stronger_mode_active). But mode supersession wording is ad hoc per provider, there is no uniform 'mode X active, prior mode guidance void' contract, and no pair-programming/interactive style exists at all. + +**Verifier note.** Claim survives as stated. Supersession wording is per-provider and inconsistent; no pair-programming/interactive collaboration style exists anywhere in the prompt surface. + +**Adopt.** Standardize a one-line activation/deactivation preamble across mode injections ('<mode> is now active; guidance from previously active modes no longer applies'), emitted on every toggle so stale-mode bleed-through (especially across compaction) is impossible; optionally add a /pair style toggle that injects small-step pacing + ask-the-user-for-observations debugging guidance. + +**Files.** `src/pythinker_code/soul/dynamic_injections/plan_mode.py`, `src/pythinker_code/soul/dynamic_injections/auto_mode.py`, `src/pythinker_code/soul/dynamic_injections/orchestration.py` + +### `protocol-headless/final-message-to-file-output-o` — missing, S, medium + +**Today.** Missing. --final-message-only prints the final message to stdout (src/pythinker_code/ui/print/visualize.py FinalOnly*Printer) but there is no file-output option; rg for last_message/output-last-message in src/pythinker_code finds only unrelated feedback.py context fields. + +**Verifier note.** Claim confirmed; could not refute. FinalOnlyTextPrinter/FinalOnlyJsonPrinter write only to stdout (src/pythinker_code/ui/print/visualize.py:145-153, 189-197 via print/_print_final_text). The complete CLI flag inventory (src/pythinker_code/cli/__init__.py:332-638) has no -o/--output-file/--last-message option; rg for last_message/output-file/output_file across src/pythinker_code finds nothing relevant. + +**Adopt.** Add --output-file FILE for print mode: capture the final assistant text (the FinalOnly printers already isolate it) and write it on exit regardless of output format; write empty content with a stderr warning when the turn produced no final message. Composes with stream-json so callers get both the event stream and the answer artifact. + +**Files.** `src/pythinker_code/ui/print/visualize.py`, `src/pythinker_code/cli/__init__.py`, `<ref>/exec/src/event_processor.rs` + +### `protocol-headless/progress-to-stderr-final-answer-to-stdout-split-in-human-hea` — partial, S, medium + +**Today.** Partial. Default --print text mode (TextPrinter in src/pythinker_code/ui/print/visualize.py) rich-prints every wire message to stdout, so captured output mixes progress with the answer; --final-message-only/--quiet gives a clean answer but discards progress entirely instead of moving it to stderr. No config-summary header (model/work-dir/approval posture) is emitted at run start. + +**Verifier note.** Claim confirmed; could not refute. TextPrinter rich-prints every WireMessage object to stdout (src/pythinker_code/ui/print/visualize.py:38-43), mixing progress and answer in captured output. FinalOnlyTextPrinter/--quiet (visualize.py:132-153; cli/__init__.py:550-566, 707-716) buffer only ContentParts and discard all progress rather than routing it to stderr. No config-summary header (model/work-dir/approval) is emitted at print-run start — the print path (cli/__init__.py:1013-1019 run_print; Print.run in ui/print/__init__.py) prints only the command echo. --verbose (cli/__init__.py:339-345) is log verbosity, not a stderr progress channel. + +**Adopt.** Make TextPrinter write progress to the original stderr stream and only the final assistant text to stdout (unconditionally, or gated on stdout-not-a-tty like the reference); print a one-block run header (model, session id, work dir, approval mode) to stderr at start. Keeps --final-message-only as the fully quiet variant. + +**Files.** `src/pythinker_code/ui/print/visualize.py`, `<ref>/exec/src/event_processor_with_human_output.rs` + +### `protocol-headless/robust-stdin-prompt-contract-sentinel-append-as-context-enco` — partial, S, medium + +**Today.** Partial. src/pythinker_code/ui/print/__init__.py reads stdin only when no -p prompt was given and stdin is not a tty; when both -p and piped stdin are present, the piped data is silently ignored; there is no `-` sentinel and no encoding detection (sys.stdin.read() will raise or mojibake on UTF-16 input). cli/__init__.py rejects empty --prompt but has no stdin guidance message. + +**Verifier note.** Claim confirmed; could not refute. src/pythinker_code/ui/print/__init__.py:73-75 reads stdin only when `command is None and not sys.stdin.isatty() and input_format == "text"` — when -p is supplied, piped stdin is silently ignored (never combined as context). No '-' sentinel exists (rg for dash handling in cli/__init__.py and ui/print/__init__.py: nothing). sys.stdin.read() uses the default text decoder with no encoding detection. cli/__init__.py:766-767 raises BadParameter('Prompt cannot be empty') with no stdin guidance. + +**Adopt.** In Print.run: support prompt == '-' to force stdin; when a prompt is given and stdin is piped, append stdin wrapped in a clearly tagged context block (consistent with the existing untrusted-data conventions); read stdin as bytes and decode with BOM sniffing, exiting with a convert-to-UTF-8 hint on failure. + +**Files.** `src/pythinker_code/ui/print/__init__.py`, `<ref>/exec/src/lib.rs` + +### `review-mode/upstream-aware-merge-base-selection` — partial, S, medium + +**Today.** diff_source.py computes `git merge-base HEAD <chosen_ref>` with a static candidate chain (origin/main, main, master) and records fallback reasons, but never checks whether the chosen local branch's upstream is ahead (packages/pythinker-review/src/pythinker_review/engine/diff_source.py:144-167). subagents/git_context.py collects no merge-base at all. + +**Verifier note.** Claim CONFIRMED. The candidate chain is static (origin/main → main → master) and there is no upstream-tracking-ref logic anywhere in src/pythinker_code or packages/. The only @{u}-adjacent git calls are current-branch lookups (`rev-parse --abbrev-ref HEAD`), never upstream comparisons. Minor line drift only: the merge-base block actually spans diff_source.py L144-177 (merge-base call at L161), defaults at L94-95. + +**Adopt.** In diff_source.resolve_diff (and the new target resolver), after choosing a base ref, resolve `<ref>@{upstream}`; if `git rev-list --left-right --count <ref>...<upstream>` shows the upstream ahead, use the upstream for the merge-base and record it in the existing fallback_reason audit field. Two extra guarded git calls, fully covered by the existing PreflightError handling. + +**Files.** `packages/pythinker-review/src/pythinker_review/engine/diff_source.py`, `src/pythinker_code/subagents/git_context.py` + +### `skills-hooks-memories/hook-output-spill-to-disk-with-recovery-path` — partial, S, medium + +**Today.** hooks/engine.py:33-39 hard-truncates hook stdout/stderr at 12,000 chars with a '...[truncated]' suffix — the tail is lost and there is no recovery path. Pythinker already has disk-spill infrastructure for oversized tool output from a prior arc that this could reuse. + +**Verifier note.** Claim confirmed, with one precision fix: the 12,000-char truncation (_MAX_HOOK_OUTPUT_CHARS, engine.py:33-39) applies in _hook_outputs_for_wire (engine.py:42-58), i.e. the wire/UI display path; the in-memory HookResult keeps full stdout, and the post-compact additional_context path is separately bounded by MAX_RESTORED_SKILL_CHARS in build_hook_context_message (compaction_restore.py:161-183) with its own '...[truncated]' and no recovery either. The reusable disk-spill infrastructure the analyst referenced does exist: enable_spill / SPILL_MAX_CHARS=5_000_000 in tools/utils.py:58-139 (used by shell/web tools). Hooks use none of it. + +**Adopt.** Replace _truncate_hook_output with the existing tool-output disk-spill helper: write full text under the session dir, return head/tail preview plus the saved path. Applies to wire-visible outputs and (once adopted) model-injected hook context. + +**Files.** `src/pythinker_code/hooks/engine.py`, `<ref>/hooks/src/output_spill.rs` + +### `skills-hooks-memories/pretooluse-hook-input-rewriting-updated-input` — missing, S, medium + +**Today.** hooks/runner.py HookResult has only action/reason/additional_context; toolset.py:617-637 checks results solely for action=='block' and then executes the original arguments unchanged. No mechanism for a hook to modify tool input. + +**Verifier note.** Claim confirmed. HookResult (hooks/runner.py:11-21) carries only action/reason/stdout/stderr/exit_code/timed_out/additional_context — no updated_input. rg for 'updated_input|updatedInput' over src returns zero hits (only 'hookSpecificOutput' for permissionDecision parsing at runner.py:81-91). toolset.py:625-633 checks PreToolUse results only for block, then executes tool.call(arguments) with the original parsed arguments at toolset.py:655. No mechanism exists for a hook to mutate tool input. + +**Adopt.** Add an updated_input field to HookResult parsed from JSON stdout (hookSpecificOutput.updatedInput), and in toolset.py apply the first/last non-null rewrite to tool_input_dict before permission checks and tool.call — re-running check_tool_call_allowed on the rewritten input so a hook cannot widen permissions. + +**Files.** `src/pythinker_code/hooks/runner.py`, `src/pythinker_code/soul/toolset.py`, `<ref>/hooks/src/events/pre_tool_use.rs` + +### `skills-hooks-memories/skill-usage-doctrine-in-the-system-prompt-trigger-rules-prog` — partial, S, medium + +**Today.** agents/default/system.md §12 (lines 261-267) has a single paragraph: identify relevant skills, read SKILL.md before applying, -local companions, conserve context. No mandatory trigger rule, no read-to-EOF/no-subagent-delegation rule, no announce-usage or minimal-set/sequencing guidance, no missing-skill fallback wording. + +**Verifier note.** Verdict 'partial' stands but the claimed_state materially understates what exists — skill doctrine is NOT confined to the §12 paragraph. system.md:126 (§5 tool guidance) is a mandatory trigger rule: 'Load a skill's exact instructions before applying its workflow — mandatory for review-pr, diagnose-ci-failures, fix-errors, implement-specs, spec-driven-implementation, check-impl-against-spec, resolve-merge-conflicts, and create-pr.' system.md:112 is a read-to-EOF rule for skills ('when the file is a spec, skill, or checklist you are implementing against, keep reading to the end before acting on it'). Also present: inline /skill:<name> handling doctrine (:128), skill-content authority/prompt-injection framing (:181), and checklist-walking for skill compliance claims (:216). Still genuinely absent: announce-usage wording, an explicit no-subagent-delegation rule for skill reading, minimal-set/sequencing guidance, and missing-skill fallback wording. So 'no mandatory trigger rule, no read-to-EOF rule' in the claim is factually wrong. + +**Adopt.** Expand §12 with the doctrine bullets (trigger obligation when a skill is named or clearly matches, full-read rule, main-agent-reads-skills rule, minimal set + one-line announcement, fallback when a named skill is absent). Note: agent prompt text is test-pinned — update phrase pins/inline snapshots (pytest --inline-snapshot=fix) in the same change. + +**Files.** `src/pythinker_code/agents/default/system.md`, `<ref>/core-skills/src/render.rs` + +### `tools-registry-codemode/head-tail-capped-output-retention-drop-the-middle-keep-prefi` — partial, S, medium + +**Today.** ToolResultBuilder (src/pythinker_code/tools/utils.py:168-214) is head-only: after 50K chars everything later is dropped from the inline result. Mitigations exist — tail() surfaces the last 5 non-empty lines in error briefs (shell/__init__.py:221) and enable_spill writes the full output to disk with a ReadFile recovery hint — but the inline body still loses the suffix. + +**Verifier note.** Claim survives as stated. ToolResultBuilder.write is head-only (stops appending once max_chars=50_000 reached); suffix recovery exists only via tail() in error briefs and enable_spill disk spill with a ReadFile hint. The only middle-drop truncation in the repo is the TUI diff renderer (display-only, not the model-facing result). + +**Adopt.** Convert ToolResultBuilder's buffer to a head+tail budget (e.g. 60% head / 40% tail) with an explicit '[... N chars omitted ...]' marker between segments; keep disk spill as the full-output escape hatch. Touches only utils.py plus truncation-marker assertions in tests. + +**Files.** `src/pythinker_code/tools/utils.py`, `<ref>/core/src/unified_exec/head_tail_buffer.rs` + +### `config-features/central-feature-flag-registry-with-lifecycle-staging` — missing, M, medium + +**Today.** Missing. Toggles are scattered Pydantic booleans across config sections (MemoryConfig.lexical_recall/injection_bus/durable_memory, TUIConfig.turn_recaps/smooth_streaming, GoalConfig.auto_continue, etc. in src/pythinker_code/config.py) with no staging metadata, no experimental menu, no deprecation/removal pathway, and no warning when an experimental flag is on. + +**Verifier note.** Claim confirmed. No flag registry, staging metadata, experimental menu, or deprecation pathway exists; toggles are plain Pydantic booleans. + +**Adopt.** Add a small registry module (list of dataclasses: key, stage, default, description, optional announcement) backing a [features] table in config; route new experimental behaviors through it; render an /experimental toggle list in the settings selector; print a one-line warning when under-development flags are enabled; keep removed keys parseable as no-ops so old configs survive upgrades. + +**Files.** `src/pythinker_code/config.py`, `src/pythinker_code/ui/shell/selectors/settings.py`, `<ref>/features/src/lib.rs` + +### `config-features/comment-preserving-targeted-config-edits` — partial, M, medium + +**Today.** Partial. Writes correctly target the single user file by re-loading it first (e.g. src/pythinker_code/soul/pythinkersoul.py:848, auth/oauth.py, web/api/config.py), but save_config in src/pythinker_code/config.py dumps the entire validated model (model_dump -> tomlkit.dumps), destroying comments/ordering and materializing every default into the file on any single-field persist. + +**Verifier note.** Claim confirmed. All persist paths re-load the user file then call save_config, which serializes a fresh full model_dump through tomlkit.dumps — no round-trip document editing, so comments/ordering are destroyed and all defaults are materialized. + +**Adopt.** Add an edit helper that parses the user file as a tomlkit TOMLDocument, applies targeted set/delete of dotted keys, and writes back (tomlkit already round-trips comments); migrate the persist-one-setting call sites (thinking effort, auth tokens, web API) onto it, leaving full save_config for initial file creation. + +**Files.** `src/pythinker_code/config.py`, `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/web/api/config.py`, `<ref>/config/src/mcp_edit.rs` + +### `config-features/named-user-config-profile-overlays` — partial, M, medium + +**Today.** Partial. agent_execution_profile (src/pythinker_code/config.py _apply_agent_execution_profile + src/pythinker_code/execution_profiles.py) is a fixed enum of behavioral presets that fill unset fields — useful, but users cannot author arbitrary named config overlays (different model/provider/theme bundles) and switch with a flag. + +**Verifier note.** Claim confirmed as partial. agent_execution_profile is a fixed 5-value Literal filling unset behavioral fields; users cannot author arbitrary named config bundles. One adjacent escape hatch the claim omits: agent spec files (--agent/--agent-file) can pin a model per named agent, but that covers only the model field, not provider/theme/config overlays. + +**Adopt.** Support ~/.pythinker/<name>.config.toml selected via --profile or PYTHINKER_PROFILE, merged as an extra scope between user and project with provenance label 'profile <name>'; keep existing execution-profile presets unchanged. Settings writes while a profile is active should target the profile file. + +**Files.** `src/pythinker_code/config.py`, `src/pythinker_code/execution_profiles.py`, `<ref>/config/src/profile_toml.rs` + +### `context-mgmt/conversation-carry-over-on-model-switch-compact-with-the-out` — missing, M, medium + +**Today.** Missing. /model switching creates a brand-new session and discards the conversation entirely (src/pythinker_code/ui/shell/slash.py:359-366 'Starting fresh session for the new model...'); in-runtime LLM swaps (soul/pythinkersoul.py:825-835) would compact lazily with the NEW model, whose window may not fit the request. + +**Verifier note.** Claim confirmed, but fix the second citation: pythinkersoul.py:825-835 is the thinking-effort swap (same model recreated via create_llm), not a model switch. The real in-runtime model swap that keeps history is the ACP server's setSessionModel path — it replaces runtime.llm in place (acp/server.py:439-448), so any later compaction runs with the NEW model whose window may not fit; same conclusion, stronger evidence. + +**Adopt.** On model switch, instead of always forking an empty session, offer carry-over: while the old LLM is still live, run compact_context (optionally with a 'handoff to a different model' instruction) when the history exceeds the new model's trigger threshold, then swap the LLM in place (or seed the new session with the compaction summary). Keeps multi-hour threads usable across model changes. + +**Files.** `src/pythinker_code/ui/shell/slash.py`, `src/pythinker_code/soul/pythinkersoul.py` + +### `core-loop/per-turn-aggregated-diff-tracker-net-unified-diff-of-all-fil` — missing, M, medium + +**Today.** Missing. Pythinker has per-file restore points for undo (file_restore.py) and per-tool-call diff rendering in the TUI, but no turn-level aggregation: there is no 'what did this turn change overall' artifact for the UI, hooks, or review flows (rg for turn_diff/TurnDiff over src/pythinker_code is empty). + +**Verifier note.** Claim confirmed. No turn-level diff aggregation exists. Closest analogs found (none per-turn, none net-unified-diff): per-file restore points for undo; a session-level files-modified NAME list for the resume recap; a web API endpoint computing workdir-vs-HEAD git numstat (session/workdir scope, not agent-turn scope); and per-tool-call DiffDisplayBlock rendering. + +**Adopt.** Hook WriteFile/StrReplaceFile (and any patch tool) to report (path, before, after) into a turn-scoped baseline map keyed by first-seen content; at turn end render a net unified diff (difflib, with a size cutoff fallback to a stat summary) and emit it as a wire event plus a /diff slash command. The existing file_restore capture point already has before-content in hand. + +**Files.** `src/pythinker_code/file_restore.py`, `src/pythinker_code/tools/file/write.py`, `src/pythinker_code/soul/pythinkersoul.py` + +### `exec-safety/durable-always-allow-policy-amendment-across-sessions` — partial, M, medium + +**Today.** Partial. 'Approve for session' records a signature key in ApprovalState.auto_approve_actions which persists only with that session's state (src/pythinker_code/session_state.py:18, src/pythinker_code/soul/agent.py:311-332); there is no user- or project-durable grant, so the same approval is re-asked in every new session. + +**Verifier note.** Claim confirmed as stated. approve_for_session writes a shell_command_signature-scoped key into ApprovalState.auto_approve_actions, which is persisted and restored only with that session's state (so it survives resume, not new sessions). The only durable knobs are blanket ones: config default_yolo (config.py:858) and workspace trust/safe_mode (session.state.trust, saved in agent.py:320) — neither is a per-command grant. No user- or project-level per-command store exists. + +**Adopt.** Add an 'approve always' response tier to the approval runtime (models.py ApprovalResponseKind) that appends the command's signature prefix to a durable rules file (the policy file from the declarative-policy finding, or a minimal JSON-lines grants file as a first step) with fcntl locking and dedup, loaded and merged into auto-approve checks at startup. Keep the existing invariant: destructive and config-surface calls are never eligible, mirroring _is_session_approvable. + +**Files.** `src/pythinker_code/approval_runtime/models.py`, `src/pythinker_code/soul/approval.py`, `src/pythinker_code/session_state.py`, `<ref>/execpolicy/src/amend.rs` + +### `exec-safety/structural-ast-shell-parsing-for-safety-classification-of-co` — partial, M, medium + +**Today.** Partial. permission.py uses shlex with punctuation_chars plus targeted patches — a regex for $()/backticks/<()/>(), a glued-operator double-lex diff, and an unquoted-newline check (_shell_hidden_command_reason) — which is fail-closed but heuristic; consequently `bash -c '...'` is treated as opaque/mutating wholesale (interpreters listed in _MUTATING_COMMANDS), so wrapped read-only scripts can never be classified or safe-listed. + +**Verifier note.** Claim confirmed. All classification is shlex-token based with fail-closed heuristics: _shell_hidden_command_reason (permission.py:434) detects $()/backticks/<()/>() via regex plus a punctuation_chars double-lex diff (lines 449-462) and unquoted newlines; opaque commands get a self-scoped 'shell:opaque:' signature (permission.py:944-945). Shell interpreters and script runtimes (bash/sh/zsh/python/node/perl...) are listed wholesale in _MUTATING_COMMANDS (permission.py:140-157), so bash -c '<read-only script>' can never be classified safe. No AST parser in pythinker's own code: bashlex appears in uv.lock only as a transitive dep of an unrelated package (batrachian-toad, uv.lock:299,312) and is never imported under src/; no tree-sitter. + +**Adopt.** Introduce an optional structural parse (tree-sitter-bash via py-tree-sitter, or bashlex) used in two places: (a) replace/back up _shell_hidden_command_reason with whitelist-of-node-kinds rejection, eliminating the documented shlex blind-spot patches; (b) when a command is `bash|zsh|sh -c/-lc <script>`, extract word-only sub-command sequences and run each through the existing segment classifiers instead of blanket-blocking the interpreter, keeping today's behavior as the fallback when the parse rejects. Gate behind a parser-availability check so the dependency stays optional. + +**Files.** `src/pythinker_code/soul/permission.py`, `<ref>/shell-command/src/bash.rs` + +### `exec-safety/windows-powershell-aware-command-safety-classification` — partial, M, medium + +**Today.** Partial-to-missing. The Shell tool spawns PowerShell on Windows (src/pythinker_code/tools/shell/__init__.py:89-99, 390-393) but every classifier in src/pythinker_code/soul/permission.py is POSIX/shlex-based: no PowerShell cmdlets appear in _MUTATING_COMMANDS or shell_destructive_reason, so the read-only profile gate, destructive deliberation backstop, and session-approval signatures are largely blind for PowerShell commands (Remove-Item -Recurse -Force is neither mutating nor destructive today). + +**Verifier note.** Claim confirmed, with one small nuance. The Shell tool is PowerShell-aware only for spawn/description (shell/__init__.py:90-98 loads powershell.md; :392-393 spawns '<path> -command <cmd>'), while soul/permission.py contains zero PowerShell cmdlets — rg for Remove-Item/cmdlet/powershell across permission.py returns nothing — so the read-only mutation gate, destructive backstop, and network gate are blind to cmdlets. Nuance: session-approval signatures are not entirely blind — shell_command_signature still keys on the first token (e.g. 'remove-item'), so approvals are scoped per cmdlet family; the real hole is that shell_destructive_reason never matches cmdlets, so Remove-Item -Recurse -Force IS session-approvable and never deliberation-bounced. + +**Adopt.** Add a PowerShell branch to the permission classifiers: a light tokenizer (split on ;, |, && with quote awareness), Verb-Noun cmdlet classification by verb family (Get/Read/Select/Measure read-only; Remove/Set/New/Clear/Stop mutating; Remove-Item with -Recurse+-Force, Format-Volume etc. destructive), suffix/case normalization for .exe/.cmd/.bat, and alias mapping (rm/del/ri -> Remove-Item). Wire it in when the active shell is PowerShell so signatures, mutation, and destructive checks share the same data, mirroring how the POSIX path shares _unwrap_command. + +**Files.** `src/pythinker_code/soul/permission.py`, `src/pythinker_code/tools/shell/__init__.py`, `<ref>/shell-command/src/command_safety/windows_dangerous_commands.rs` + +### `mcp/mcp-elicitation-requests-with-policy-based-auto-accept-auto-` — missing, M, medium + +**Today.** Missing. No elicitation handling exists (grep for elicit across src/pythinker_code returns nothing); fastmcp 3.2.0 (pyproject.toml) supports an elicitation_handler on Client but pythinker never registers one, so servers that require elicitation fail or hang. + +**Verifier note.** Verdict stands as claimed. rg -i 'elicit' across src/pythinker_code, tests, tests_e2e, tasks returns zero hits; fastmcp is pinned at 3.2.0 (pyproject.toml:47) and no elicitation_handler is ever passed to fastmcp.Client (toolset.py:1038 constructs it bare). + +**Adopt.** Register a fastmcp elicitation_handler that consults the existing approval system: decline by default in non-interactive/never-ask modes, auto-accept only empty-schema confirmations when the active profile would auto-approve MCP actions anyway, and otherwise raise an approval request through runtime.approval with the server name and message. Fail closed (decline) on any doubt. + +**Files.** `src/pythinker_code/soul/toolset.py`, `src/pythinker_code/soul/permission.py`, `<ref>/mcp/src/elicitation.rs` + +### `mcp/model-visible-tool-name-normalization-charset-sanitization-d` — missing, M, medium + +**Today.** Missing. MCPTool (soul/toolset.py) exposes the raw mcp_tool.name to the model with no charset/length normalization anywhere (grep for sanitize/[A-Za-z0-9_-] found only unrelated hits); cross-server same-name collisions are last-wins with a warning log; mcp.json server names are unvalidated (only the stderr log filename is sanitized via _MCP_LOG_NAME_RE). + +**Verifier note.** Verdict stands as claimed. MCPTool.__init__ passes name=mcp_tool.name raw to the model-visible tool list; no charset/length normalization exists in toolset.py or llm.py (grep for sanitize/re.sub on tool names returned nothing relevant). Cross-server same-name collisions are last-wins with a warning (toolset.py:363-373); MCP-vs-builtin collisions skip the MCP tool. The only name sanitization is _MCP_LOG_NAME_RE for the stderr log filename, exactly as the claim says. + +**Adopt.** At registration, sanitize the model-visible name to the provider-safe charset, cap length (~64 chars), and on collision (cross-server or post-sanitize) append a short stable hash of (server, raw name) instead of silently shadowing; keep the raw name on MCPTool for the actual call_tool protocol request. Validate server names on `mcp add`. + +**Files.** `src/pythinker_code/soul/toolset.py`, `src/pythinker_code/cli/mcp.py`, `<ref>/mcp/src/tools.rs` + +### `observability-feedback/code-retention-outcome-analytics-accepted-line-counts-from-u` — missing, M, medium + +**Today.** Missing. No per-turn edit-volume telemetry exists; feedback.py collects diff stats only on explicit /feedback (src/pythinker_code/feedback.py:_collect_git_snapshot), and grep over src/pythinker_code shows no added/deleted-line tracking tied to turns. This is the main outcome metric (does the agent's code stick?) absent from the fleet view. + +**Verifier note.** Claim stands. Verified via full track()-event inventory (no edit-volume/line-count event exists) and grep for additions/deletions/numstat across src. Line stats exist only as on-demand display surfaces: the web API git-diff endpoint (GitDiffStats with total_additions/total_deletions via `git diff --numstat HEAD`) and /feedback's diff_stat snapshot. Nothing ties added/deleted lines to turns or reaches telemetry. + +**Adopt.** After each turn that ran file-mutating tools, compute `git diff --numstat` (or diff the Edit/Write tool results) in the workdir, derive added/deleted effective-line counts with the same normalization rules, and attach counts (never content or hashes) to the per-turn rollup event, plus an optional salted repo-remote hash for cohorting. Gate behind the existing telemetry kill switch. + +**Files.** `<ref>/analytics/src/accepted_lines.rs`, `src/pythinker_code/feedback.py`, `src/pythinker_code/soul/pythinkersoul.py` + +### `observability-feedback/per-attempt-api-request-and-stream-event-health-telemetry` — partial, M, medium + +**Today.** Partial. Pythinker meters whole LLM calls (duration/success/tokens, src/pythinker_code/soul/pythinkersoul.py:1607-1700; telemetry/metrics.py record_llm_call) and tracks api_error on final failure; tenacity retries are surfaced to the UI/log (_before_step_retry_sleep, pythinkersoul.py:1705-1710) but not metered, and there is no stream-event-level health accounting (parse failures, idle timeout, early close) for diagnosing flaky providers in a BYO-key multi-provider fleet. + +**Verifier note.** Verdict 'partial' stands but two details need correction: (1) per-attempt metering DOES exist at the span/metric layer — the pythinker.llm span plus record_llm_call(success=False) and record_error(kind='api_error') live inside _run_step_once, which is exactly the unit tenacity retries, so EVERY failed attempt (not just final failure) emits a failed-llm-call metric, an api_error metric, and an error-annotated llm span. (2) Only the track('api_error') analytics event is final-failure-only. Confirmed correct: retries are surfaced via wire StepRetry + log only (no retry counter metric), and there is no stream-event-level health accounting (no parse-failure/idle-timeout/early-close telemetry found in src or packages/pythinker-core). + +**Adopt.** Add a retry counter metric incremented in the tenacity before_sleep hook (tagged with error_type and attempt), and an attempt attribute on the llm span. In the provider streaming layer, count malformed/early-closed stream events into a low-cardinality `pythinker.llm.stream_anomalies` counter (kind = parse_error | idle_timeout | early_close). + +**Files.** `<ref>/otel/src/events/session_telemetry.rs`, `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/telemetry/metrics.py` + +### `observability-feedback/turn-latency-profile-decomposition-and-time-to-first-token-t` — partial, M, medium + +**Today.** Missing. Pythinker records whole-turn, whole-llm-call, and whole-tool durations as separate metrics (src/pythinker_code/telemetry/metrics.py, soul/pythinkersoul.py:1607-1700, soul/toolset.py:646), but no first-token timing exists anywhere (visualize UI computes display-only token rates: src/pythinker_code/ui/shell/visualize/_blocks.py:508) and no per-turn breakdown distinguishes harness overhead from sampling vs tool-blocking time; retry counts are logged but not metered. + +**Verifier note.** Claimed 'missing' is overstated; correct verdict is partial. TTFT is confirmed absent everywhere (only display-side token rates in the visualize UI). BUT per-turn latency decomposition exists at the trace level: start_span deliberately nests spans into a connected tree (pythinker.turn -> pythinker.llm -> pythinker.tool, with tool.duration_ms attributes), and the default trace sample rate is 1.0, so harness overhead vs sampling vs tool-blocking time is derivable per turn from exported traces. What's missing is metrics-level (aggregate histogram) decomposition, TTFT, and a retry counter — though note each failed retry attempt IS already metered as a failed llm call (see claim 7). + +**Adopt.** Instrument the streaming callback (on_message_part) to record monotonic first-token time per llm call and per turn; emit a pythinker.turn.ttft_seconds histogram. Accumulate per-turn buckets in the soul loop: time before first llm call, sum of llm-call durations, sum of tool-task durations, residual overhead; attach as turn span attributes and to the per-turn rollup event, with retry counts from the tenacity hooks. + +**Files.** `<ref>/analytics/src/facts.rs`, `<ref>/otel/src/events/session_telemetry.rs`, `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/telemetry/metrics.py`, `src/pythinker_code/ui/shell/visualize/_blocks.py` + +### `patch-file-tools/first-class-delete-rename-file-operations-with-diff-display-` — missing, M, medium + +**Today.** Pythinker has only WriteFile (src/pythinker_code/tools/file/write.py) and StrReplaceFile (replace.py); there is no audited delete or rename tool, so the model must shell out to rm/mv, which bypasses build_diff_blocks display and create_file_restore_point snapshots (file_restore.py is only invoked from write.py/replace.py). Multi-edit batches are validated in-memory before write, but only within a single file. + +**Verifier note.** Claim survives. No delete/rename/move file tool exists under any name; restore points are created only by WriteFile and StrReplaceFile. + +**Adopt.** Add delete and rename capabilities to the file toolset (either a small DeleteFile/MoveFile pair or ops on the existing tools) that run the same workspace/symlink validation, refuse directories, show the removed/moved content as a diff block in the approval prompt, snapshot via create_file_restore_point (capturing overwritten destination content on rename), and auto-create destination parents. Optionally accept a list of per-file ops validated fully before the first write for multi-file refactors. + +**Files.** `<ref>/apply-patch/src/parser.rs`, `<ref>/apply-patch/src/lib.rs`, `src/pythinker_code/tools/file/write.py`, `src/pythinker_code/file_restore.py` + +### `patch-file-tools/persistent-background-fuzzy-filename-search-session` — partial, M, medium + +**Today.** src/pythinker_code/ui/shell/prompt.py LocalFileMentionCompleter delivers fuzzy @-completion backed by utils/file_filter.py: git ls-files (or capped 1000-file walk) with a 2s TTL cache invalidated on .git/index mtime, scope-aware caching, and basename re-ranking — solid for small/medium repos, but the listing rebuild is synchronous on the UI thread path, untracked files are invisible in git mode only via ls-files flags used, results cap at 1000 in walk mode, and there are no highlight indices or progress/cancellation for huge trees. + +**Verifier note.** Verdict 'partial' stands, but two factual errors in the claimed state: (1) untracked files are NOT invisible in git mode — list_files_git defaults include_untracked=True and runs `git ls-files --others --exclude-standard`, also subtracting deleted working-tree files via `ls-files --deleted`; (2) fuzzy match highlighting exists in the completion menu, because the completer is prompt_toolkit's FuzzyCompleter, which styles matched characters in its display. Accurate parts: synchronous listing on the completion path (plain Completer via merge_completers, no ThreadedCompleter), 2.0s TTL (refresh_interval=2.0), .git/index mtime invalidation, scope-aware cache, 1000-entry cap in walk mode (and top-level mode), and no persistent background session/progress/cancellation. + +**Adopt.** Move candidate indexing to a background thread that walks once per session and serves re-queries from memory (invalidate on .git/index mtime or watcher events); raise the candidate cap, add cancellation when the fragment changes, and return match-character indices for highlight styling in the completion menu. Keep the existing file_filter ignore rules as walker overrides. + +**Files.** `<ref>/file-search/src/lib.rs`, `src/pythinker_code/ui/shell/prompt.py`, `src/pythinker_code/utils/file_filter.py` + +### `patch-file-tools/shared-file-watcher-service-with-subscriber-fan-out-and-watc` — missing, M, medium + +**Today.** rg over src/pythinker_code finds no watchdog/watchfiles/inotify usage anywhere; skills/ and skill/ have no reload/refresh path (manifest loaded at startup), and the mention completer polls with a 2s TTL + .git/index mtime check instead of event-driven invalidation. + +**Verifier note.** Claim survives, with one nuance: while skill DISCOVERY runs once at agent construction with no reload path, ReadSkill reads skill content from disk at call time (read_skill_text_with_local_specialization), so edits to an already-discovered skill's body are picked up; only the discovered set/manifest is startup-frozen. No event-driven watching anywhere. + +**Adopt.** Introduce a small watcher service (watchfiles handles the OS layer) owned by the app: subscribers register path sets and receive debounced, deduped path batches via asyncio queues; first consumers are (a) skills/AGENTS.md hot-reload with a changed-notification into the TUI, and (b) mention-completer index invalidation. Port the two non-obvious design points: refcount watches shared across subscribers, and ancestor-fallback for not-yet-existing paths with events renamed back to the requested path. + +**Files.** `<ref>/file-watcher/src/lib.rs`, `<ref>/app-server/src/skills_watcher.rs`, `<ref>/app-server/src/fs_watch.rs`, `src/pythinker_code/ui/shell/prompt.py` + +### `persistence-resume/full-content-transcript-search-for-session-discovery-grep-ac` — missing, M, medium + +**Today.** Missing. The TUI session picker has no search input at all — only a Ctrl+A scope toggle (src/pythinker_code/ui/shell/session_picker.py); the web list's q parameter filters title and work_dir only (src/pythinker_code/web/api/sessions.py:267-272). Raw transcripts are never content-searched for discovery; memory/recall.py retrieves distilled memory blocks, not transcripts, and serves injection rather than resume. + +**Verifier note.** Verdict confirmed, but the claimed_state mischaracterizes the recall landscape: besides memory/recall.py (distilled blocks), pythinker has a model-invocable cross-session Recall TOOL (src/pythinker_code/tools/recall/__init__.py, adoption arc memory-1/ctxmgmt-3) with mode='search' and mode='read' that reads full sanitized transcripts of prior sessions. However its search ranks by TITLE keyword overlap only (_rank_sessions, lines 51-66 — 'score = sum(1 for term in terms if term in title.lower())'), serves in-conversation context to the model, and is not a session-discovery/resume affordance. The TUI session picker has no search input (only Ctrl+A scope toggle, ui/shell/session_picker.py:177-183) and the web q parameter filters title/work_dir only (web/store/sessions.py:351-359). No grep/ripgrep acceleration, no transcript content matching, no match snippets anywhere. + +**Adopt.** Add a type-to-filter search to the session picker and a content search mode to the web list: JSON-escape the term (json.dumps slice) and run rg -l --fixed-strings over the sessions buckets' context.jsonl/wire.jsonl, with a pure-Python line scan fallback when rg is missing; merge matched session ids into the normal listing and show a ~140-char snippet around the first match. Pythinker already vendors ripgrep discovery logic in tools/file/grep_local.py to reuse. + +**Files.** `src/pythinker_code/ui/shell/session_picker.py`, `src/pythinker_code/web/api/sessions.py`, `src/pythinker_code/tools/file/grep_local.py`, `<ref>/rollout/src/search.rs` + +### `persistence-resume/structured-cross-agent-session-import-with-content-hash-dedu` — partial, M, medium + +**Today.** Partial. /import accepts a text file or a pythinker session id and injects flattened text as a single context message into the current session (src/pythinker_code/soul/slash.py import_context, src/pythinker_code/utils/export.py resolve_import_source/perform_import, with token-budget and sensitive-file guards). There is no structured conversion of a foreign agent transcript into a native resumable session, no detection of recent external sessions, and no dedup ledger — re-importing duplicates content. + +**Verifier note.** Claim confirmed as stated. /import (registry command import_context, soul/slash.py:459-491) resolves either a UTF-8 text file or a same-workspace pythinker session id (resolve_import_source, utils/export.py:708-789), flattens session history via stringify_context_history, and appends ONE wrapped user message (build_import_message '<imported_context source=...>') to the current context with token-budget and sensitive-file guards (perform_import, utils/export.py:809-858). There is no conversion of foreign-agent transcripts into a native resumable session, no detection of external agents' recent sessions (rg for external-agent session directories reference harness session paths finds nothing), and no dedup: perform_import appends unconditionally with no content hash or ledger (rg 'already imported|ledger|sha' in export.py: no hits in the import path). + +**Adopt.** Add an importer that parses a foreign JSONL transcript (role/message/timestamp records) into a new native session: user/assistant messages into context.jsonl, TurnBegin/TurnEnd records into wire.jsonl, cwd-matched to the current work dir, title from the first user line. Keep an import ledger (source path -> sha256 + imported session id) under the share dir so repeat invocations skip unchanged sources. Expose as a flag on /import (or a sessions import command); optional startup detection of recent foreign sessions for the cwd can come later. + +**Files.** `src/pythinker_code/utils/export.py`, `src/pythinker_code/soul/slash.py`, `<ref>/external-agent-sessions/src/detect.rs`, `<ref>/external-agent-sessions/src/ledger.rs`, `<ref>/external-agent-sessions/src/export.rs` + +### `prompts-instructions/first-class-review-target-resolution-lifecycle-preset-prompt` — partial, M, medium + +**Today.** The rubric and dispatch discipline are adopted (system.md §4.1, §5 review fan-out including model-side merge-base guidance; review.yaml/code_reviewer.yaml; review-pr skill; ```report rendering in §8), but there is no /review slash command (soul/slash.py registers init/recap/compact/clear/yolo/auto/plan/goal/learn/best-practices/add-dir/export/import only), no harness-computed diff target, and no structured selectable result re-entry or interrupted-review record — the model must orchestrate all of it ad hoc per request. + +**Verifier note.** Verdict correct, but the 'no /review slash command' framing undercounts the preset-prompt half: every bundled skill is auto-registered as a slash command, so /skill:review-pr IS a slash-invocable review preset prompt — soul/pythinkersoul.py SKILL_COMMAND_PREFIX (line 136) and _make_skill_runner (lines 1349-1370) inject the skill text as the user turn, and tests_e2e/test_wire_protocol.py (lines 164/364) pins 'skill:review-pr' in the advertised command list. The other two lifecycle pieces are genuinely absent: no harness-computed diff target (merge-base appears only as model-side guidance — system.md line 120 '$(git merge-base main HEAD)'; skills/review-pr/SKILL.md step 1 tells the model to 'Identify the exact diff under review and its base'; no .py code computes a review target — rg 'merge.base' hits no source besides system.md), and no structured selectable result re-entry (ui/shell/components/report.py renders ```report blocks display-only; the /reports command in ui/shell/slash.py lines 1932-1940 opens the Agent Tracing Visualizer, unrelated to review findings; no interrupted-review record exists). + +**Adopt.** Add a /review [uncommitted|branch <base>|commit <sha>] command: harness resolves the target (runs git merge-base itself, embeds the SHA), dispatches the review subagent with the preset prompt + rubric, and posts results as a report message whose findings the user can reference to request fixes; on interrupt, record an explicit 'review interrupted — re-run /review' marker in history. Update the wire-handshake slash-command snapshot (tests_e2e). + +**Files.** `src/pythinker_code/soul/slash.py`, `src/pythinker_code/agents/default/review.yaml`, `src/pythinker_code/agents/default/code_reviewer.yaml` + +### `prompts-instructions/per-model-system-prompt-variants-scaled-to-model-capability` — partial, M, medium + +**Today.** One ~41KB system.md serves every model (src/pythinker_code/agents/default/system.md, agent.yaml system_prompt_path; agentspec.py has a single system_prompt_path per agent, no model keying). Model-specific quirks are handled by a deliberate lightweight alternative: family-matched dynamic defense fragments (src/pythinker_code/soul/dynamic_injections/model_defense.py), whose docstring explicitly rejects per-model prompt cloning to keep the prompt cache-stable. + +**Verifier note.** Claim survives as stated. No per-model prompt keying exists anywhere; the family-matched defense-fragment channel is the only model-specific prompting. + +**Adopt.** Keep the canonical system.md but add an optional model-family→prompt-profile map (e.g. a condensed variant for small local models served via Ollama/llama.cpp that drowns in 41KB of instructions), resolved at agent load using the same substring-matching machinery as model_defense.py; default unchanged, gate behind config so prompt-cache stability is opt-out. + +**Files.** `src/pythinker_code/agents/default/system.md`, `src/pythinker_code/agentspec.py`, `src/pythinker_code/soul/dynamic_injections/model_defense.py` + +### `review-mode/interactive-findings-triage-select-findings-and-dispatch-fix` — partial, M, medium + +**Today.** Report rendering is display-only: findings are grouped by severity and pretty-printed but there is no selection surface and no path from a rendered finding to a dispatched fix task (src/pythinker_code/ui/shell/components/report.py). The user must re-describe findings in a new prompt. + +**Verifier note.** Verdict CORRECTED from missing to partial. The TUI half of the claim is right — src/pythinker_code/ui/shell/components/report.py is render-only (parse_report_block/render_report/render_finding, no selection or dispatch surface). But the claim 'no path from a finding to a dispatched fix task; the user must re-describe findings' is FALSE for the product: the pythinker-review reviewflow persists findings with stable IDs and ships a complete finding→fix dispatch loop — `pythinker review next` / `show-finding` / `triage --finding --status` / `fix --finding` / `revalidate` / `open-pr`. fix_project() plans a patch via LLM, applies it under an allowed-paths constraint, runs trusted+allowlisted validation commands, records a PatchAttempt linked to the finding, and updates finding lifecycle/history. What is genuinely missing is only an interactive picker-style selection surface (TUI or CLI — triage/fix are flag-driven, not menus). + +**Adopt.** After a review report renders, offer an optional follow-up selector listing findings (severity-ordered, checkbox multi-select using the existing selector component). On confirm, synthesize a fix prompt quoting only the selected findings (title, location, body) and feed it as the next user turn — or dispatch an implementer subagent in goal-style mode. Keep it additive: plain Enter dismisses with no behavior change. + +**Files.** `src/pythinker_code/ui/shell/components/report.py`, `src/pythinker_code/ui/shell/selector.py`, `src/pythinker_code/ui/shell/slash.py` + +### `skills-hooks-memories/memory-usage-feedback-loop-read-citation-tracking-drives-ret` — missing, M, medium + +**Today.** memory/retriever.py ranks recall purely by BM25 + 14-day recency half-life + label/path boosts; nothing records whether an injected recall block or a Recall-tool read was actually useful, and no usage signal influences future ranking or pruning (memory/recall.py, tools/recall/__init__.py). Durable MEMORY.md entries never expire. + +**Verifier note.** Claim confirmed. retriever.py ranks with hand-rolled BM25 (k1=1.5, b=0.75) x recency decay (_RECENCY_HALF_LIFE_DAYS=14.0) + _PATH_BOOST=0.5 (lines 12-15, 50-85); the only 'used' variables in retriever.py:96-105 and tools/recall/__init__.py:77-106 are token/char budget counters, not usage signals. Nothing in memory/recall.py or tools/recall records whether an injected block or Recall read helped, and no signal feeds back into ranking or pruning. Durable MEMORY.md entries have no expiry (project_memory.py has only the per-store char limit and the 100-entry journal cap; no time/usage-based eviction). + +**Adopt.** Record a last_used/use_count sidecar (e.g. memory/usage.json) bumped when (a) the Recall tool reads a session, (b) a recalled block's source entry text appears in later assistant output, or (c) the model Reads MEMORY.md/USER.md via file tools; fold use_count and last_used into LexicalRetriever scoring and have the approval-gated consolidation list never-used stale entries as prune candidates. + +**Files.** `src/pythinker_code/memory/retriever.py`, `src/pythinker_code/memory/recall.py`, `src/pythinker_code/tools/recall/__init__.py`, `<ref>/memories/read/src/usage.rs`, `<ref>/memories/read/src/citations.rs` + +### `skills-hooks-memories/memory-authored-skill-promotion-recurring-procedures-become-` — missing, M, medium + +**Today.** No path from memory to skills: memory/consolidation.py targets only MEMORY.md/USER.md entries; skill authoring exists solely as the interactive skill-creator bundled skill (src/pythinker_code/skills/skill-creator/). Recurring workflows learned across sessions never crystallize into reusable skill packages. + +**Verifier note.** Claim confirmed. memory/consolidation.py targets only the memory/user stores (InboxCandidate.target, approve path writes via ProjectMemoryStore); rg 'skill' over src/pythinker_code/memory/ and project_memory.py: zero functional hits. rg 'promote|crystalliz|instinct' over src *.py: only unrelated UI/shell matches (spinner words, markdown, background manager). Skill authoring exists only as interactive bundled skills (src/pythinker_code/skills/skill-creator/, plus agent-creator). No automated or suggested path from recurring memory content to a skill package. + +**Adopt.** Extend the inbox candidate model with a target='skill' kind: when consolidation (heuristic now, LLM later) sees the same procedure recur across journal recaps, stage a proposed skill directory (SKILL.md draft) in the inbox; on approval, write it under the user skills root where existing discovery picks it up. Reuse the quality-gate checklist from the reference template inside the skill-creator prompt. + +**Files.** `src/pythinker_code/memory/consolidation.py`, `src/pythinker_code/skills/skill-creator/SKILL.md`, `<ref>/memories/write/templates/memories/consolidation.md` + +### `skills-hooks-memories/per-skill-enable-disable-rules-and-invocation-policy` — missing, M, medium + +**Today.** No per-skill disable mechanism exists: config.py:945-960 only offers merge_all_available_skills and extra_skill_dirs; rg for disabled/disable over skill code finds nothing. parse_skill_text (skill/__init__.py:701-750) reads only name/description/type frontmatter — every discovered skill is always listed in the prompt, registered as a slash command, and readable via ReadSkill, including side-effectful ones (e.g. create-pr) the model can self-trigger. + +**Verifier note.** Claim confirmed. Skill config surface is only merge_all_available_skills (config.py:945-953) and extra_skill_dirs (:954-957); rg for disable/exclude/blocklist/denylist/allowed_skills over skill/__init__.py and lockfile.py finds nothing. parse_skill_text (skill/__init__.py:701-750) reads only name/description/type frontmatter. Every discovered standard/flow skill is auto-registered as /skill:<name> at pythinkersoul.py:1259-1276 (only filters: type check at :1260 and name-collision skip at :1263-1268), flow skills additionally as flow commands (:1279-1297), and all are model-readable via the ReadSkill tool (tools/skill/__init__.py). No invocation policy of any kind. + +**Adopt.** Add a [skills] config table with per-name/per-path enabled overrides applied after discovery, and honor a disable-model-invocation (or allow_implicit_invocation) frontmatter key: such skills stay out of PYTHINKER_SKILLS and ReadSkill but keep their /skill: slash command. Surface disabled skills in a /skills listing. + +**Files.** `src/pythinker_code/skill/__init__.py`, `src/pythinker_code/config.py`, `<ref>/core-skills/src/config_rules.rs`, `<ref>/core-skills/src/model.rs` + +### `skills-hooks-memories/permissionrequest-hook-event-programmatic-approval-decisions` — missing, M, medium + +**Today.** hooks/config.py HookEventType has 13 events but no PermissionRequest; the approval path (soul/permission.py check_tool_call_allowed, soul/approval.py) runs before PreToolUse hooks with no hook integration, so users cannot script auto-approve/deny policies beyond the static allowlist. + +**Verifier note.** Claim confirmed with one overstatement to trim. HookEventType (hooks/config.py:5-19) lists exactly 13 events; no PermissionRequest (rg over src+tests: zero hits; the ACP test matches are the unrelated ACP permission protocol). The permission gate check_tool_call_allowed (permission.py:412-426) runs at toolset.py:600-609 BEFORE the PreToolUse trigger (:614) and contains no hook integration; soul/approval.py has zero hook references. Nuance the analyst missed: auto-DENY is already scriptable — a PreToolUse hook can block via exit 2 or hookSpecificOutput.permissionDecision=='deny' (runner.py:66-91). What is genuinely missing is a hook in the approval path itself, i.e. programmatic auto-APPROVE / decision injection before the user prompt. + +**Adopt.** Add a PermissionRequest event fired from the approval flow just before an interactive prompt, passing tool_name/tool_input/permission context; honor hook decisions allow→skip prompt, deny→reject with reason, no-output→fall through to the normal prompt. Keep it fail-open to the interactive prompt (never fail-open to allow). + +**Files.** `src/pythinker_code/hooks/config.py`, `src/pythinker_code/soul/approval.py`, `<ref>/hooks/src/events/permission_request.rs` + +### `skills-hooks-memories/skills-prompt-listing-context-budget-with-graceful-degradati` — missing, M, medium + +**Today.** skill/__init__.py:354-389 format_skills_for_prompt renders every skill with full absolute path and full frontmatter description, unbounded; only the body-derived fallback description is capped (240 chars, line 693). soul/agent.py:259 injects the result verbatim into PYTHINKER_SKILLS. A user with many skills across brand+generic dirs (merge_all_available_skills) silently bloats every system prompt. + +**Verifier note.** Claim confirmed. format_skills_for_prompt (skill/__init__.py:354-389) renders every skill with name, full Path, and full description, grouped by scope, with no count cap, char budget, or degradation. Only the body-derived fallback description is truncated (_DESCRIPTION_FALLBACK_MAX_LEN=240 at :693, _truncate at :760); frontmatter descriptions are used verbatim — the docstring's mention of a 1024-char spec cap is not enforced in parse_skill_text (:701-750). soul/agent.py:259 formats and :351 injects verbatim as PYTHINKER_SKILLS into system.md:265. No skills-budget work found in tasks/ logs either. + +**Adopt.** Give format_skills_for_prompt a token budget derived from the model context window: cap per-skill descriptions proportionally when over budget, then fall back to name+path lines and an omitted-count note the model can see; emit a one-time UI warning when truncation occurs. Optionally add a roots-alias table ($PROJ/, $USER/) to shrink repeated path prefixes. + +**Files.** `src/pythinker_code/skill/__init__.py`, `src/pythinker_code/soul/agent.py`, `<ref>/core-skills/src/render.rs` + +### `tools-registry-codemode/hook-driven-tool-input-rewriting-pre-execution-hooks-can-mod` — partial, M, medium + +**Today.** HookResult carries only action allow|block, reason, and additional_context (src/pythinker_code/hooks/runner.py:12-21); the PreToolUse path in PythinkerToolset (soul/toolset.py:617-636) can veto a call but never amend its arguments, so policy hooks cannot e.g. rewrite a command to add flags or redirect a path. + +**Verifier note.** Claim survives as stated. HookResult carries only action allow|block, reason, stdout/stderr, exit_code, timed_out, additional_context. The JSON-output parser understands permissionDecision deny and additionalContext but has no updatedInput/modified-arguments channel; the PreToolUse path in toolset can only veto, never amend arguments. + +**Adopt.** Extend the hook output JSON with an optional updated_input object; in _call_with_lifecycle, when present, re-validate it through the tool's pydantic params model and substitute before tool.call, logging the rewrite. Reject rewrites that fail validation as a hook error rather than executing ambiguous input. + +**Files.** `src/pythinker_code/hooks/runner.py`, `src/pythinker_code/soul/toolset.py`, `<ref>/core/src/tools/registry.rs` + +### `tools-registry-codemode/namespaced-tool-identity-with-deterministic-flat-naming-and-` — partial, M, medium + +**Today.** MCP tools register under their raw server-side names; cross-server collisions are last-wins with a warning (_register_mcp_tools, src/pythinker_code/soul/toolset.py:354-376), and which server wins is nondeterministic because servers connect concurrently. A separate mcp__{server}__{tool} key scheme already exists in runtime.mcp_tools (toolset.py:983) but is not the model-facing name. + +**Verifier note.** Claim survives as stated. MCPTool registers under the raw server-side name (name=mcp_tool.name); cross-server collisions are last-wins with a logged warning and the winner is nondeterministic (servers connect concurrently, acknowledged in a code comment); the mcp__{server}__{tool} key exists only in runtime.mcp_tools bookkeeping, not as the model-facing name. + +**Adopt.** On collision (or always, behind a config flag), register the model-facing name as the existing mcp__server__tool flat form so both servers' tools stay addressable; update permission matchers and the dedup key derivation to accept the namespaced form. Deterministic and removes silent shadowing. + +**Files.** `src/pythinker_code/soul/toolset.py`, `<ref>/tools/src/code_mode.rs` + +### `mcp/agent-as-mcp-server-mode-expose-the-agent-itself-as-an-mcp-t` — partial, L, medium + +**Today.** Partial (different protocols). Pythinker can run as an ACP server (acp/server.py, `pythinker acp`) and an experimental Wire server (cli/__init__.py --wire), which serve the same embed-the-agent role for editors, but there is no MCP-protocol server mode, so MCP-only clients (other agents, MCP-capable IDEs) cannot drive pythinker as a tool. + +**Verifier note.** Claimed state is factually accurate. ACP server mode exists (src/pythinker_code/acp/server.py ACPServer; `pythinker acp` subcommand at cli/__init__.py:1556, plus deprecated --acp flag) and an experimental Wire server mode (--wire, UIMode 'wire', cli/__init__.py:174,525-529). No MCP-protocol server mode exists: cli/mcp.py has only add/remove/list/auth/reset-auth/test subcommands (no 'serve'), and rg for 'FastMCP(' as a server returns nothing. Whether 'partial' vs 'missing' is the right label depends on whether ACP/Wire count as the same capability, but the underlying facts as stated check out. + +**Adopt.** Add a `pythinker mcp serve` stdio mode reusing the existing wire/ACP session plumbing: one 'run agent' tool whose call starts a session and streams progress notifications, with the session id echoed in structured_content for follow-up calls, and approvals auto-resolved per a non-interactive policy flag. Only worth doing if agent-to-agent embedding becomes a goal. + +**Files.** `src/pythinker_code/acp/server.py`, `<ref>/mcp-server/src/tool_runner module`, `<ref>/mcp-server/src/lib.rs` + +### `mcp/server-initiated-notification-handling-server-log-messages-p` — missing, L, medium + +**Today.** Missing. Pythinker exits the client context after listing tools (_connect_server) and re-enters `async with self._client` per tool call (MCPTool.__call__), so no session persists between calls; no fastmcp message_handler/log_handler/progress_handler is registered anywhere (grep found zero hits). The tool/resource list is frozen at connect time; only child stderr is captured to a session log file. + +**Verifier note.** Verdict stands as claimed. _connect_server uses `async with server_info.client as client` and exits after list_tools/list_resources/list_prompts; MCPTool.__call__ re-enters `async with self._client` per call, so no persistent session. fastmcp.Client is constructed with no message_handler/log_handler/progress_handler/elicitation_handler kwargs (toolset.py:1038); grep for those handler names across src returns zero hits. Inventory frozen at connect time. mcpext-2(a) 'live tools/list_changed' explicitly deferred in the progress log. Only child stderr is captured (_configure_mcp_client_stderr_log). + +**Adopt.** Keep one persistent fastmcp client session per server (enter the context at connect, exit at cleanup — the close-timeout teardown already exists) and register fastmcp's log_handler/progress_handler/message_handler: route server logs to the pythinker logger with level mapping, and on tools/list_changed re-list and re-register that server's tools (with the conflict rules already in _register_mcp_tools). This also removes the per-call re-handshake latency for HTTP servers. + +**Files.** `src/pythinker_code/soul/toolset.py`, `<ref>/rmcp-client/src/logging_client_handler.rs`, `<ref>/mcp/src/rmcp_client.rs` + +### `multi-agent/best-of-n-parallel-attempts-on-one-task-with-comparison-and-` — missing, L, medium + +**Today.** Missing. RunAgents (src/pythinker_code/tools/agent/__init__.py) fans out different child tasks; a 'judge' subagent type with a verify permission profile exists (src/pythinker_code/soul/permission.py:96) but there is no built-in mode that runs N attempts of the same prompt in isolated trees and compares/selects results. + +**Verifier note.** Claim upheld. Also checked AgentLaunchSpec.variant (subagents/models.py:46) as a possible attempt-variant mechanism — it is persisted-only metadata (store.py:32,73) never set by the Agent tool, so not a best-of-N feature. The Implement->Review->Fix->Verify->Judge pipeline in agents/default/system.md is prompt guidance, not a built-in N-attempt compare/select mode. + +**Adopt.** Add an attempts: int (1-4) option to RunAgents (or a BestOf tool) that clones one child spec N times with distinct codenames, requires worktree isolation, runs attempts in parallel, then auto-launches a judge child fed each attempt's diff+report to rank them — surfacing the ranking and per-attempt worktree paths for the user/orchestrator to apply. Depends on the isolation finding landing first. + +**Files.** `src/pythinker_code/tools/agent/__init__.py`, `src/pythinker_code/subagents/usage.py` + +### `multi-agent/data-driven-batch-job-fan-out-with-templated-instructions-an` — missing, L, medium + +**Today.** Missing. RunAgents caps at 8 hand-written children with no data-driven templating, no per-item result schema, and no durable job record beyond individual background tasks (src/pythinker_code/tools/agent/__init__.py RunAgentsParams max_length=8; src/pythinker_code/background/store.py is per-task). + +**Verifier note.** Claim upheld. Closest existing pieces: base_prompt shared-prefix on RunAgents (a fixed prepend, not per-item templating) and capacity-overflow handling that launches a fitting prefix and reports the rest as 'deferred' (tools/agent/__init__.py:704-713) — there is no durable job/queue record, no per-item result schema, and no template-over-dataset expansion. + +**Adopt.** Add a RunAgentsOnRows tool: parse a CSV/JSONL, render an instruction template per row, run rows through the existing background agent pipeline under the session capacity semaphore with per-item timeout, give workers a ReportJobResult tool keyed by job_id/item_id, and write a job manifest plus output CSV under the session tasks dir for crash-safe resumption. + +**Files.** `src/pythinker_code/tools/agent/__init__.py`, `src/pythinker_code/background/store.py`, `src/pythinker_code/background/manager.py` + +### `observability-feedback/opt-in-local-raw-evidence-session-trace-bundle-with-offline-` — partial, L, medium + +**Today.** No equivalent. Sessions persist message history JSONL (src/pythinker_code/session.py) and checkpoints, and an httpx recording client exists only for eval cassettes (src/pythinker_code/llm.py:139 _build_recording_http_client); there is no raw-event spine with payload refs, no model-visible vs runtime separation, and no offline reducer. Subagent metadata lives in per-agent meta.json (src/pythinker_code/subagents/store.py) without interaction-edge linkage. + +**Verifier note.** Claimed 'missing' is wrong. Pythinker HAS a per-session raw-event spine, a model-visible vs runtime separation, an opt-in local export bundle, and offline reducers. Each session persists context.jsonl (model-visible message history) AND wire.jsonl (timestamped runtime event spine: TurnBegin/TurnEnd/StepBegin/StepRetry/ToolCall/ToolCallPart/ToolResult/ContentPart/ApprovalResponse/SubagentEvent...). Subagents get their own nested wire.jsonl under the parent session. `pythinker export` builds an opt-in ZIP (manifest.json system diagnostics, transcript.yaml reduction, all session files incl. subagents/, recent log files), and the vis app is an offline reducer that parses wire.jsonl into per-session timelines/summary stats and aggregate statistics, and accepts uploaded export ZIPs. What is genuinely absent: raw provider HTTP request/response payload capture with payload refs — the spine is post-parse wire events, and the httpx recording client only harvests rate-limit headers for the /usage panel (the analyst's 'eval cassettes' description of llm.py:139 is also wrong). + +**Adopt.** Add an env-gated TraceWriter (e.g. PYTHINKER_TRACE_ROOT) that the soul, toolset, and subagent runner call best-effort: append seq-numbered JSONL events referencing payload files written first; record LLM request/response payloads, tool dispatch boundaries, and subagent spawn/result edges. Ship a `pythinker debug trace-reduce` CLI that replays the bundle into a state.json graph keyed by stable IDs. Keep all writes wrapped so tracing can never fail a session, and document the bundle as sensitive local-only data. + +**Files.** `<ref>/rollout-trace/README.md`, `<ref>/rollout-trace/src/writer.rs`, `src/pythinker_code/session.py`, `src/pythinker_code/llm.py`, `src/pythinker_code/subagents/store.py` + +### `patch-file-tools/shell-invoked-file-edit-interception-into-the-structured-app` — missing, L, medium + +**Today.** src/pythinker_code/soul/permission.py classifies output redirection for gating (lines 507-514) and the memory-noted shlex tokenizer is blind to heredocs, so `cat <<EOF > file` style shell writes execute as opaque bash with no per-file diff, no create_file_restore_point snapshot, and coarse approval text; tools/shell has no edit-extraction layer. + +**Verifier note.** Claim survives. Shell redirection/in-place-edit detection exists only as a coarse mutation classifier for permission gating; there is no extraction of shell-mediated file edits into per-file diffs, restore points, or structured approval display. + +**Adopt.** Add a pre-execution inspector on the shell tool that detects simple heredoc/redirection write forms (a real bash parser e.g. tree-sitter-bash or bashlex, not shlex), extracts target path + content, and either (a) renders a proper diff block in the approval prompt and creates a restore point before running, or (b) returns a corrective error steering the model to WriteFile/StrReplaceFile. Start with the steer-to-tool variant (matches the reference's ImplicitInvocation pattern) as a cheap first step. + +**Files.** `<ref>/apply-patch/src/invocation.rs`, `src/pythinker_code/soul/permission.py`, `src/pythinker_code/tools/shell/__init__.py` + +### `persistence-resume/persisted-session-metadata-index-with-self-repairing-backfil` — partial, L, medium + +**Today.** Partial. CLI Session.list/list_all re-scan every session directory and re-read wire heads to derive titles on every call (src/pythinker_code/session.py:272-331); the web store builds an index only as an in-memory TTL cache from a full disk scan, lost on restart, with limit/offset (not cursor) pagination (src/pythinker_code/web/store/sessions.py _build_sessions_index/_load_sessions_index_cached). Nothing is persisted, so launch-time listing cost grows linearly with history and titles/previews are recomputed repeatedly. + +**Verifier note.** Claim confirmed as stated. There is an index abstraction (SessionIndexEntry) but it is built by a full disk scan and held only in module-global TTL caches (_sessions_cache/_sessions_index_cache, CACHE_TTL), lost on restart; nothing is ever persisted to disk (no json.dump/write of the index anywhere in web/store/sessions.py). Pagination is limit/offset list slicing, not cursor-based. CLI Session.list/list_all rescan every session dir and re-read wire heads via refresh() to derive titles on each call. The only 'self-repair' that exists is unrelated: legacy metadata.json->state.json migration (session_state.py _migrate_legacy_metadata) and torn-line tolerance in Session.is_empty. + +**Adopt.** Persist an index (SQLite or a single index.jsonl) under the share dir keyed by session id with title, preview, updated_at, work_dir, archived state, and token count. Write-through from Session.save_state/refresh and fork/archive mutations; on startup run a flock-lease-guarded backfill that only scans sessions newer than a stored watermark and upserts. List paths read the index, drop/repair entries whose directories are missing, and fall back to today's full scan when the index is absent or corrupt (fail-open). Reuse web's SessionIndexEntry shape so web and TUI picker share one source. + +**Files.** `src/pythinker_code/session.py`, `src/pythinker_code/web/store/sessions.py`, `<ref>/rollout/src/metadata.rs`, `<ref>/rollout/src/state_db.rs`, `<ref>/rollout/src/list.rs` + +### `tools-registry-codemode/tools-as-code-orchestration-mode-script-cell-that-composes-n` — missing, L, medium + +**Today.** No equivalent exists (rg for code mode / exec-cell / tool-script across src/pythinker_code is empty); every tool composition costs one model round-trip per call through PythinkerToolset.handle (src/pythinker_code/soul/toolset.py). + +**Verifier note.** Claim survives. No code-mode/exec-cell/tool-script facility exists; searches for code_mode, exec_cell, script_cell, tool_script, RunCode/ExecuteCode across src/pythinker_code and packages/ return nothing relevant (only pythinker-review's run_code_review_pass, unrelated). The tools/ package contains no python-exec orchestration tool; every composition is one model round-trip through PythinkerToolset.handle. + +**Adopt.** Opt-in RunToolScript tool: execute a model-authored Python snippet in a separate sandboxed interpreter process whose only capability is an RPC proxy back into PythinkerToolset (permission/approval checks still applied per nested call), with long scripts parked as background cells reusing the existing task store + TaskOutput wait path. Major token-efficiency win for MCP-heavy and data-shuttling work, but the sandbox boundary is the hard part in Python — keep it process-isolated, no ambient FS/network. + +**Files.** `src/pythinker_code/soul/toolset.py`, `<ref>/code-mode/src/description.rs`, `<ref>/code-mode/src/service.rs`, `<ref>/tools/src/code_mode.rs` + +## Tier 4 — low value (29 items) + +### `config-features/config-json-schema-export-for-editor-ci-validation` — missing, S, low + +**Today.** Missing. Config is Pydantic so model_json_schema() exists for free, but nothing exports or ships it (no json_schema usage found in src/pythinker_code). + +**Verifier note.** Claim confirmed. Nothing exports or ships a JSON schema for the Config model; the only jsonschema code in the monorepo is pythinker_core tool-schema dereferencing, unrelated to config. + +**Adopt.** Add a hidden CLI subcommand (or build step) that writes Config.model_json_schema() to a published schema file; document a taplo/even-better-toml association so ~/.pythinker/config.toml gets editor validation. Pairs with the unknown-key finding. + +**Files.** `src/pythinker_code/config.py`, `<ref>/config/src/schema.rs` + +### `config-features/configurable-project-root-markers` — missing, S, low + +**Today.** Missing. _find_project_root in src/pythinker_code/config.py hardcodes .git and returns None otherwise, so non-git directories get no project/local config scope at all. + +**Verifier note.** Claim confirmed. Root detection hardcodes .git everywhere; the config-scope variant returns None for non-git dirs so project/local scopes are skipped entirely, and no marker configuration exists. + +**Adopt.** Add a `project_root_markers` list setting (default ['.git']) read from the user scope, consulted by _find_project_root; first ancestor containing any marker wins, falling back to None as today. + +**Files.** `src/pythinker_code/config.py`, `<ref>/config/src/project_root_markers.rs` + +### `config-features/legacy-key-aliasing-with-user-facing-deprecation-notices` — partial, S, low + +**Today.** Partial. One ad-hoc AliasChoices (LoopControl.max_steps_per_turn) and a normalizing validator (FeedbackConfig.github_repo) exist in src/pythinker_code/config.py, but aliases are silent and there is no general mechanism or deprecation messaging when config keys get renamed. + +**Verifier note.** Claim confirmed. Exactly one validation alias and one normalizing validator exist; both are silent, and there is no general rename/deprecation mechanism for config keys. + +**Adopt.** Add a small (old_path -> new_path) alias table applied to the raw merged dict before validation, recording usages and printing 'X is deprecated, use Y' once at startup; gives a safe runway for future config renames. + +**Files.** `src/pythinker_code/config.py`, `<ref>/config/src/key_aliases.rs` + +### `context-mgmt/diff-based-mid-session-settings-environment-reinjection-agai` — partial, S, low + +**Today.** Partial. Mode toggles are covered by dedicated injection providers with rearm semantics (src/pythinker_code/soul/dynamic_injections/auto_mode.py, plan_mode.py, goal_mode.py; provider lifecycle hooks on_context_compacted/on_auto_changed in soul/dynamic_injection.py:138-182), and injections are budgeted deterministically. There is no generic diffed environment/permission baseline, but pythinker's static-per-session system prompt plus fresh-session-on-model-switch makes most reference cases moot. + +**Verifier note.** Claim confirmed as partial; the cited evidence is accurate (hook line numbers: on_context_compacted at dynamic_injection.py:153, on_auto_changed at :162). + +**Adopt.** Low priority: if mid-session environment changes become possible (cwd change, --add-dir at runtime, approval-policy edits), add a small 'environment changed' injection provider that snapshots the relevant fields per turn and emits a one-line diff reminder when they change. Reuse the existing provider framework; no new architecture. + +**Files.** `src/pythinker_code/soul/dynamic_injection.py`, `src/pythinker_code/soul/dynamic_injections/auto_mode.py` + +### `context-mgmt/hardened-cross-session-prompt-input-history-file-byte-cap-so` — partial, S, low + +**Today.** Partial. Per-workdir JSONL with O_APPEND single-write, 0600 enforcement, secret redaction, and consecutive-dup skip exists (src/pythinker_code/ui/shell/prompt.py:1526-1611, 3588-3615), but the file grows unbounded (no max-bytes/soft-cap trim) and there is no lock against concurrent pythinker instances on the same workdir. + +**Verifier note.** Claim confirmed as partial. One softener: each entry is a single write through an O_APPEND fd, so concurrent-instance appends are mostly atomic at the POSIX level even without a lock — but the claim's core points (no max-bytes/soft-cap trim, no explicit lock) are factually right. + +**Adopt.** After append, if file size exceeds a configurable cap (e.g. 1 MiB), rewrite to ~80% of the cap by dropping oldest lines via the existing tempfile+os.replace pattern from soul/context.py. Locking is optional given single-syscall appends under PIPE_BUF; skip unless interleaving is observed. + +**Files.** `src/pythinker_code/ui/shell/prompt.py` + +### `core-loop/turn-level-timing-telemetry-ttft-sampling-retry-counts-per-t` — partial, S, low + +**Today.** Partial. Turn duration/step-count/stop-reason and per-LLM-call duration+token metrics exist (pythinkersoul.py _turn and _run_step_once via telemetry.metrics), but TTFT is not measured (no first-token timestamp in the stream path; rg ttft empty) and per-turn token deltas are not recorded (only cumulative usage). + +**Verifier note.** Claim confirmed as stated, with one nuance: step retries ARE surfaced as StepRetry wire events (attempt number, max attempts, wait, error type/status) so the UI sees them, but they are not aggregated into a telemetry retry-count metric. TTFT is genuinely unmeasured (the only first-token timestamps are local TUI tokens-per-second sparkline math, not telemetry), and record_turn carries no token fields — token counters are per-LLM-call plus a cumulative session total only. + +**Adopt.** Capture a first-content-part timestamp in the on_message_part path of _run_step_once and emit a ttft metric on the existing pythinker.llm span; snapshot cumulative_usage at _turn entry to record a per-turn token delta attribute on the pythinker.turn span. + +**Files.** `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/telemetry/metrics.py` + +### `mcp/tool-metadata-trust-hygiene-strip-privileged-meta-keys-from-` — missing, S, low + +**Today.** Missing, but mostly inapplicable: pythinker has no first-party connector ecosystem to spoof, and it already prepends its own trusted framing to every MCP tool description (MCPTool.__init__ in soul/toolset.py). Tool _meta is otherwise ignored entirely. + +**Verifier note.** Verdict stands as claimed. No _meta handling anywhere (rg '_meta' in toolset.py/mcp_resource returns nothing); tool annotations/visibility metadata are ignored — MCPTool consumes only name, description, and inputSchema. The trusted framing prefix the claim mentions is confirmed: MCPTool.__init__ prepends pythinker's own 'This is an MCP tool from the already-connected MCP server `X`...' text before the server-supplied description. + +**Adopt.** Low priority: if tool _meta ever starts influencing behavior or rendering, honor the visibility convention (hide tools whose meta marks them non-model-visible) and ignore/strip unrecognized privileged-looking meta keys rather than forwarding them. + +**Files.** `src/pythinker_code/soul/toolset.py`, `<ref>/mcp/src/rmcp_client.rs` + +### `multi-agent/collaboration-mode-presets-bundling-model-reasoning-effort-i` — partial, S, low + +**Today.** Largely present in different shape: plan mode is a first-class toggle with dynamic injection and permission gating (src/pythinker_code/soul/dynamic_injections/plan_mode.py, src/pythinker_code/app.py:381-383), execution profiles gate tools and subagent types per mode (src/pythinker_code/execution_profiles.py), and goal mode covers autonomous execution. The only delta is that modes do not bundle a model/reasoning-effort switch. + +**Verifier note.** Claim upheld with one refinement: the 'modes do not bundle a model/reasoning-effort switch' delta is true only for session-level modes (plan/auto/goal — no model/effort refs in soul/dynamic_injections/plan_mode.py, auto_mode.py, goal_mode.py, or the plan-mode toggle path). Per-SUBAGENT-TYPE presets already bundle model + instructions + tool policy: AgentTypeDefinition.default_model (subagents/models.py:33), markdown agent frontmatter `model:` with validation fallback (subagents/discovery.py:128,186-192), and AgentLaunchSpec carries thinking/thinking_effort applied at build time (subagents/models.py:44-45; subagents/builder.py:26-27,44). So model+effort bundling exists in the subagent preset layer, just not as session collaboration modes. + +**Adopt.** Optionally let plan mode (and execution profiles) carry a model/thinking-effort override applied on mode entry and restored on exit — a small config field plus a swap in the mode toggle path. Low urgency; current modes already gate behavior correctly. + +**Files.** `src/pythinker_code/soul/dynamic_injections/plan_mode.py`, `src/pythinker_code/execution_profiles.py` + +### `multi-agent/interrupted-turn-guidance-marker-in-child-history` — partial, S, low + +**Today.** Partial. Pending tool calls get synthetic 'Tool call interrupted by user.' results on interrupt (src/pythinker_code/soul/pythinkersoul.py:1768-1784), which makes interruption visible at tool-result granularity, but there is no turn-level guidance message; mostly relevant once mid-run steering (interrupt+redirect) exists for children. + +**Verifier note.** Claim upheld; minor line-ref drift only — the synthetic-marker block sits at src/pythinker_code/soul/pythinkersoul.py:1801-1817 in the current (locally modified) working tree rather than 1768-1784. Mechanism matches the claim. + +**Adopt.** When the steering tool interrupts a child, append a short system-reminder style marker ('previous turn was interrupted by the orchestrator; new instructions follow') ahead of the injected message so the child does not treat the truncation as its own failure. Piggyback on the existing synthetic-marker write path. + +**Files.** `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/subagents/runner.py` + +### `multi-agent/persisted-parent-child-spawn-edge-graph-with-open-closed-lif` — partial, S, low + +**Today.** Partial. parent_agent_id is persisted in each AgentLaunchSpec (src/pythinker_code/subagents/models.py:47, store.py:33,74) and crash recovery reconciles instance statuses (src/pythinker_code/background/manager.py recover()/reconcile_stale_agent_record), but there is no edge-status concept, no descendant/tree query, and the graph is flat anyway because only root may spawn. + +**Verifier note.** Claim upheld as stated, including the flat-graph point: nested spawning is hard-blocked ('Subagents cannot launch other subagents.', src/pythinker_code/tools/agent/__init__.py:264), so persisted parent_agent_id edges are depth-1. SubagentStore.list_instances (subagents/store.py:181) is a flat list with no parent filter or tree/descendant query, and there is no edge open/closed status distinct from instance status. + +**Adopt.** Low priority while depth is capped at 1. If deeper delegation ever lands, add list_children(parent_agent_id, status) to SubagentStore using the existing meta files, plus an open/closed flag flipped on terminal reconciliation, keeping deterministic ordering by created_at then id. + +**Files.** `src/pythinker_code/subagents/store.py`, `src/pythinker_code/subagents/models.py` + +### `observability-feedback/metric-tag-hygiene-validation-and-bounded-cardinality-normal` — partial, S, low + +**Today.** Partial. The sink asserts primitive-only attributes (src/pythinker_code/telemetry/sink.py:_assert_primitive), model names are bucketed into a bounded family dimension (telemetry/metrics.py classify_model_family), and resource attrs carry version/ui_mode/device. But wire/ACP client names/versions pass through verbatim with validation explicitly deferred to the backend (telemetry/__init__.py set_client_info), and no charset/allowlist normalization exists for free-form tag values. + +**Verifier note.** Claim stands; all cited details verified. _assert_primitive enforces primitive-only event properties/context and the sink drops schema violations without retry; classify_model_family buckets model names into a bounded family dimension; set_client_info passes wire/ACP client name/version through verbatim with an explicit comment deferring validation/normalization to the backend; no charset/allowlist normalization for free-form tag values exists (only Sentry path/message scrubbing and error-ring message redaction, which are privacy redaction, not cardinality hygiene). + +**Adopt.** Add a small sanitize_tag_value helper (charset filter + length cap) plus an allowlist mapping for client names (known editors -> canonical value, else 'other') applied in set_client_info and track_session_started_once; optionally assert tag-value length in _assert_primitive. + +**Files.** `<ref>/otel/src/metrics/validation.rs`, `<ref>/otel/src/metrics/tags.rs`, `src/pythinker_code/telemetry/sink.py`, `src/pythinker_code/telemetry/__init__.py` + +### `prompts-instructions/configurable-personality-presets-injected-into-a-prompt-slot` — partial, S, low + +**Today.** The slot mechanism exists — system.md has ${ROLE_ADDITIONAL} wired through agent.yaml system_prompt_args — but it ships empty; there are no curated personality presets and no config key or slash command to select one. Communication style is fixed in system.md §8 (src/pythinker_code/agents/default/system.md lines 18, 183-189; agent.yaml lines 5-6). + +**Verifier note.** Claim survives with one nuance: the ${ROLE_ADDITIONAL} slot is not dormant — it is the active persona slot for every subagent yaml (plan.yaml, review.yaml, coder.yaml, etc. all fill it with role personas) and is fillable by user-authored agents via --agent-file (documented in skills/customize-pythinker/SKILL.md and skills/agent-creator/SKILL.md). What is genuinely absent is curated end-user personality presets and any runtime selector: the root default agent ships ROLE_ADDITIONAL: "" (agents/default/agent.yaml line 6), the only builtin agent variants are --agent {default,ask,debug,okabe} (cli/__init__.py lines 565-571, 745-755) and okabe/agent.yaml is a tool-subset only with no persona text, and no config key or slash command selects a personality (rg 'personality|persona|output style' across src; config.py has no persona/system_prompt key; soul/slash.py registers init/recap/compact/clear/yolo/auto/plan/goal/learn/best-practices/add-dir/export/import only). + +**Adopt.** Ship 2-3 curated personality overlay files (e.g. pragmatic default, supportive/pairing, terse) with Values/Tone/Escalation structure; add a config key (and optionally a slash command) that fills ROLE_ADDITIONAL from the chosen preset. Default stays empty so existing prompt pins/tests are unaffected. + +**Files.** `src/pythinker_code/agents/default/agent.yaml`, `src/pythinker_code/agents/default/system.md` + +### `prompts-instructions/frontend-design-anti-genericism-guidance` — missing, S, low + +**Today.** Absent: no frontend/design guidance in system.md, best_practices.md, or any subagent spec (rg for frontend/design/UI across src/pythinker_code/agents and prompts finds only unrelated 'Design and implementation' engineering headings). + +**Verifier note.** Claim survives. No frontend/design-quality guidance exists in any prompt surface. + +**Adopt.** Add a short conditional 'Frontend work' subsection to system.md §6 or best_practices.md (5-7 rules + the existing-design-system exception), or ship it as a built-in skill the model loads for UI tasks to avoid bloating the cached prompt for a review-first product. + +**Files.** `src/pythinker_code/agents/default/system.md`, `src/pythinker_code/prompts/best_practices.md` + +### `protocol-headless/exported-protocol-schemas-for-external-client-codegen` — partial, S, low + +**Today.** Partial. Wire protocol types are pydantic models with a versioned initialize handshake (src/pythinker_code/wire/jsonrpc.py protocol_version + ClientCapabilities; types.py WireMessageEnvelope with a v1 back-compat alias), and an e2e handshake snapshot pins the slash-command list (tests_e2e), but no JSON Schema fixtures are generated/checked in for wire or ACP types — external clients must read Python source. + +**Verifier note.** Claim confirmed with one naming nit. Versioned handshake exists: src/pythinker_code/wire/jsonrpc.py:85 ClientCapabilities, :109-113 InitializeParams.protocol_version. WireMessageEnvelope exists (src/pythinker_code/wire/types.py:722-749, untagged {type, payload}); the 'v1 back-compat alias' the claim cites is actually the _compat_legacy_fields validator (types.py:304-310) normalizing task_tool_call_id -> parent_tool_call_id — there is no literal 'v1' tag. The e2e handshake inline-snapshot pin is real (tests_e2e/test_wire_protocol.py:test_initialize_handshake, snapshot includes slash_commands). The core gap stands: no JSON Schema fixtures are generated or checked in for wire/ACP types — find for *.schema.json hits only blackbox/agent_x (the vendored upstream clone, not pythinker), and rg for model_json_schema across src/tests/tests_e2e/docs returns nothing. + +**Adopt.** Add a small generator (make target) that dumps model_json_schema() for the WireMessage envelope union and JSON-RPC message types into a checked-in schema/ dir, plus a snapshot test that regeneration is clean — giving wire clients a codegen artifact and CI drift detection for protocol changes. + +**Files.** `src/pythinker_code/wire/types.py`, `src/pythinker_code/wire/jsonrpc.py`, `<ref>/app-server-protocol/src/schema_fixtures.rs` + +### `protocol-headless/version-control-trust-gate-before-unattended-runs` — partial, S, low + +**Today.** Partial. Pythinker runs --print anywhere; the mitigation is per-session safe_mode defaulting to True (src/pythinker_code/session_state.py, soul/approval.py) which fail-closes approval-required actions, but auto-approvable edits in a non-VCS directory remain unrecoverable; file_restore.py provides checkpoint-based undo which softens this further. No git-repo check exists in cli/__init__.py or ui/print. + +**Verifier note.** Every fact in the claim verified; could not refute. There is no VCS-awareness anywhere in the run gate: the only .git checks in cli/__init__.py (lines 187-197, 218-225) walk to the repo root for MCP-config discovery, not trust gating; is_git_repo exists only in web/models.py for the web UI. Mitigations are as claimed: safe_mode defaults True (src/pythinker_code/session_state.py:23) and fail-closes approval-required actions (src/pythinker_code/soul/approval.py:226-245), and file_restore.py implements per-session FileRestorePoint checkpoints (file_restore_points dir). Caveat on grading: the named capability (a version-control check) is absent in any form — 'partial' is defensible only under the claim's explicit framing that safe_mode/file_restore partially cover the same risk; under a literal reading the verdict would be 'missing'. + +**Adopt.** On --print (and --auto/--yolo) startup, detect absence of a git repo at work_dir and emit a prominent stderr warning naming the risk and the file-restore checkpoint fallback; optionally a config knob to escalate the warning to a refusal for unattended profiles. Avoid hard-blocking by default to preserve existing UX. + +**Files.** `src/pythinker_code/cli/__init__.py`, `src/pythinker_code/session_state.py`, `<ref>/exec/src/lib.rs` + +### `review-mode/review-mode-ui-state-signaling-banner-token-usage-snapshot-r` — partial, S, low + +**Today.** Subagent activity is surfaced through the task browser/status feed and per-subagent usage accounting (src/pythinker_code/ui/shell/task_browser.py, src/pythinker_code/subagents/usage.py), but there is no dedicated review-mode banner or usage snapshot/restore — acceptable because pythinker reviews are ordinary subagent runs rather than a modal takeover. + +**Verifier note.** Claim CONFIRMED factually. Cited surfaces exist (task_browser.py status/preview feed; subagents/usage.py per-subagent accumulate_usage/format_usage_lines/summarize_batch). No review-mode banner: statusline segments are cwd/git/model/context/tokens/effort (statusline.py line1/line2 segment sets at L97-101, effort badge L265-279, git badge L282+); prompt.py carries a plan-mode status indicator but nothing review-specific. No usage snapshot/restore mechanism found (no usage_snapshot/restore_usage/snapshot_usage symbols anywhere in src). + +**Adopt.** Only worth doing as part of the /review command: show a transient 'review in progress — <hint>' status line scoped to the dispatched review run, reusing the existing status feed. Skip token snapshotting; pythinker already accounts subagent usage separately. + +**Files.** `src/pythinker_code/ui/shell/task_browser.py`, `src/pythinker_code/subagents/usage.py` + +### `skills-hooks-memories/explicit-inline-skill-mention-resolution-with-ambiguity-guar` — partial, S, low + +**Today.** Pythinker resolves /skill:<name> slash commands at message start (soul/pythinkersoul.py:1259-1370) and handles mid-message /command references via agent-mediated guidance (soul/dynamic_injections/inline_commands.py) — a deliberate repo decision (mechanical mid-message splitting was proven to discard the surrounding task). No $-style mention sigil or pre-turn body injection for inline references. + +**Verifier note.** Claim confirmed as stated. /skill:<name> resolves as a message-start slash command (SKILL_COMMAND_PREFIX at pythinkersoul.py:136, registration :1259-1276, active-skill persistence :1337-1346). Mid-message references are handled agent-mediated via InlineCommandReminderProvider (soul/dynamic_injections/inline_commands.py: _TOKEN_RE + _REMINDER_TEMPLATE instructing ReadSkill-and-apply), wired at pythinkersoul.py:448, reinforced by system.md:128 — a deliberate design (mechanical splitting was rejected; see tests/core and repo memory). rg for $-style mention sigils or pre-turn body injection of skill content: nothing found. Verdict 'partial' is right. + +**Adopt.** Keep agent mediation as the execution path, but borrow the detection rigor: extend inline_commands.py to also recognize skill names referenced mid-message, with the env-var blocklist and only-when-unambiguous name matching, and have the injected guidance name the exact matched skill (path included) so the agent reliably reads it. Do not auto-split or auto-inject bodies. + +**Files.** `src/pythinker_code/soul/dynamic_injections/inline_commands.py`, `src/pythinker_code/soul/pythinkersoul.py`, `<ref>/core-skills/src/injection.rs` + +### `tools-registry-codemode/cancellation-contract-with-teardown-aware-standardized-abort` — partial, S, low + +**Today.** Interruption cancels tool futures; pythinkersoul.py (~1787-1800) preserves real outputs of already-completed calls and synthesizes interrupted markers only for pending ones, and Shell kills its child inside its CancelledError handler (src/pythinker_code/tools/shell/__init__.py:381-384). But teardown-waiting is per-tool convention rather than a registry contract, and abort texts are ad hoc without elapsed-time info. + +**Verifier note.** Claim survives as stated. On interrupt, pythinkersoul keeps real outputs of completed calls and synthesizes a static 'Tool call interrupted by user.' ToolRuntimeError for pending ones (no elapsed-time info), with shielded context writes so no unanswered tool_calls persist. Per-tool teardown (Shell kill-on-CancelledError, grep_local, agent) is convention, not a registry contract; abort texts are ad hoc ('Tool call interrupted by user.' vs background 'Interrupted by user'). + +**Adopt.** Add an optional waits_for_cancellation attribute honored by the interrupt path (shield the tool task and await it with a bounded timeout before substituting the marker) and standardize the aborted-result text to include wall time. Small change to the CancelledError branch in pythinkersoul plus the marker constant. + +**Files.** `src/pythinker_code/soul/pythinkersoul.py`, `src/pythinker_code/tools/shell/__init__.py`, `<ref>/core/src/tools/parallel.rs` + +### `config-features/nested-project-config-layers-from-project-root-down-to-cwd` — missing, M, low + +**Today.** Missing. _load_scoped in src/pythinker_code/config.py reads exactly one project file at the .git root (.pythinker/config.toml + config.local.toml); subdirectory .pythinker dirs are ignored. Precedent exists elsewhere: AGENTS.md context already merges root->cwd. + +**Verifier note.** Claim confirmed. Exactly one project config file pair is read at the .git root; subdirectory .pythinker/ configs are ignored. The cited AGENTS.md root->cwd precedent is real. + +**Adopt.** Walk cwd ancestors up to the project root collecting .pythinker/config.toml files, merge root-first so deeper dirs override, applying the same scope-lock/sanitize guard and provenance label per directory. Should land after (or together with) trust gating since it widens the repo-controlled surface. + +**Files.** `src/pythinker_code/config.py`, `<ref>/config/src/loader/mod.rs` + +### `context-mgmt/model-initiated-fresh-context-window-tool-requested-history-` — missing, M, low + +**Today.** Missing. /clear exists as a user command (src/pythinker_code/soul/slash.py clear) and checkpoints exist (soul/context.py), but the model has no way to declare 'this exploration is done, give me a clean window'; nothing in src/pythinker_code/tools/ or soul/toolset.py exposes a reset. + +**Verifier note.** Claim confirmed. Adjacent-but-different capability that exists: subagent delegation (tools/agent) gives the model fresh child windows for delegated work, but there is no way for the model to reset ITS OWN history. + +**Adopt.** Add a tool that sets a 'fresh-window requested' flag; after the current step's tool results are appended, rebuild context as system prompt + a model-authored handoff note (the tool's required argument) + compaction-restore reminders, reusing the clear+rebuild primitive from compact_context. Gate behind config since it is destructive. + +**Files.** `src/pythinker_code/soul/slash.py`, `src/pythinker_code/soul/context.py` + +### `exec-safety/per-host-network-rules-with-deny-tier-and-justification` — partial, M, low + +**Today.** Partial. First-class web tools enforce a configurable host allowlist plus an SSRF guard with per-redirect re-validation (src/pythinker_code/tools/web/fetch.py, _allowlist.py, config.py web.allowed_domains); shell-level network access is binary — _NETWORK_COMMANDS blocked under restricted profiles, unrestricted otherwise (src/pythinker_code/soul/permission.py:166-178) — with no deny-with-justification tier and no per-host control for shell commands. + +**Verifier note.** Claim confirmed. Web tools have per-host control: config web.allowed_domains (config.py:567-593 with hostname validation), label-aware subdomain matching in tools/web/_allowlist.py (host_in_allowlist; None/empty = unrestricted for these tools), and fetch.py's SSRF guard with manual redirect-following that re-validates every hop against both the SSRF check and the allowlist (_get_revalidating_redirects, fetch.py:93-109; private/link-local/multicast/reserved blocked at :61). Shell-level network is binary exactly as claimed: _NETWORK_COMMANDS (permission.py:166-178) is blocked only under restricted (no-shell-mutation) profiles via shell_mutation_reason (:536) inside check_shell_command_allowed (:341-354); unrestricted profiles get normal approval with no per-host rules and no deny-with-justification tier. + +**Adopt.** Extend web config with deny entries carrying justifications (surfaced in the ToolError so the model learns why), and reuse the same host rules to scope obvious shell network commands (curl/wget URL arguments are parseable from the existing token stream) in 'ask' profiles. Full enforcement for arbitrary shell programs requires the sandbox/proxy layer and should ride that finding instead. + +**Files.** `src/pythinker_code/tools/web/fetch.py`, `src/pythinker_code/tools/web/_allowlist.py`, `src/pythinker_code/config.py` + +### `exec-safety/semantic-command-summarization-for-approval-history-display` — missing, M, low + +**Today.** Missing. Approval prompts and transcripts show the raw command string via ShellDisplayBlock (src/pythinker_code/tools/display.py:31, src/pythinker_code/tools/shell/__init__.py:158-163); no semantic classification exists (grep for parsed/summarize found only unrelated session-recap code). + +**Verifier note.** Claim confirmed, with one minor nuance. Approval prompts carry the raw command via ShellDisplayBlock(language, command) (display.py:31-36; shell/__init__.py:158-164 foreground, :271-276 background); no parser-derived semantic summary exists. Nuance worth recording: background tasks accept a model-supplied 'description' param (shell/__init__.py:65-71, default auto-filled) surfaced in BackgroundTaskDisplayBlock/task views — that is model-authored free text for task listings, not semantic classification, and the approval prompt still shows the raw command. + +**Adopt.** A small classifier over the already-tokenized segments (permission.py's tokenizer) mapping common commands to one-line intents ('Search for TODO in src/', 'Read file X', 'List files') attached to ShellDisplayBlock as an optional subtitle; keep raw command primary. Low safety impact for a developer audience, so only worth doing opportunistically. + +**Files.** `src/pythinker_code/tools/display.py`, `<ref>/shell-command/src/parse_command.rs` + +### `mcp/cached-tool-list-snapshots-served-while-a-server-is-still-co` — missing, M, low + +**Today.** Missing. MCP tools only become visible to the model after the background connect finishes (_register_mcp_tools in soul/toolset.py); on the first turn of every session the model plans without knowledge of MCP tools that will appear seconds later. + +**Verifier note.** The capability itself (a persisted/cached tool list served during connect) is indeed absent — no MCP tool-list cache exists anywhere (rg -i cache in toolset.py: zero hits; tools register only in _register_mcp_tools after connect). BUT the claim's stated failure mode is FALSE: the model never plans without MCP tools, because PythinkerSoul._agent_loop calls start_background_mcp_loading() and then AWAITS wait_for_background_mcp_loading() before step 1 of every turn (emitting MCPLoadingBegin/End to the UI while blocking). The real gap a cache would address in pythinker is first-turn latency / hung-connect blocking, not first-turn tool blindness. + +**Adopt.** Persist each server's last successful tool list (name/description/schema) under the share dir; at startup register provisional MCPTool entries from the snapshot whose __call__ awaits server readiness (wait-for-connect with timeout) before invoking, then reconcile/replace once the live list arrives. Invalidate the snapshot when mcp.json for that server changes. + +**Files.** `src/pythinker_code/soul/toolset.py`, `<ref>/mcp/src/rmcp_client.rs` + +### `observability-feedback/cross-process-trace-context-propagation-and-user-configurabl` — partial, M, low + +**Today.** Partial. The sampler is ParentBased anticipating upstream wire/ACP parents (src/pythinker_code/telemetry/otel.py:147), but no traceparent extraction/injection exists anywhere (rg 'traceparent|tracestate' over src/pythinker_code is empty); subagents run in-process so their spans already nest. Endpoint/token/sample-rate are env-overridable (telemetry/config.py) but custom span attributes are not. + +**Verifier note.** Claim stands as 'partial'; core facts verified: ParentBased(TraceIdRatioBased) sampler exists, and no traceparent/tracestate extraction or injection exists anywhere in src/pythinker_code (the only context.attach is local span nesting in otel.start_span). Endpoint/token/sample-rate are env-overridable as claimed. One nuance on 'custom span attributes are not [configurable]': resources are built with Resource.create(), and the OTel SDK merges the standard OTEL_RESOURCE_ATTRIBUTES env var there, so users can inject custom resource-level attributes onto all spans via standard OTel env — incidental SDK behavior, not a pythinker feature, and there is no pythinker-specific span-attribute config. + +**Adopt.** Accept an optional traceparent field in the wire/ACP initialize message and attach it as the parent context for turn spans of that session; expose a PYTHINKER_OTEL_SPAN_ATTRIBUTES env (k=v,k=v) merged into the Resource. Low urgency until external embedders ask for joined traces. + +**Files.** `<ref>/otel/src/trace_context.rs`, `src/pythinker_code/telemetry/otel.py`, `src/pythinker_code/telemetry/config.py` + +### `patch-file-tools/committed-change-delta-with-exactness-tracking-for-failure-s` — partial, M, low + +**Today.** src/pythinker_code/file_restore.py snapshots the pre-image before each WriteFile/StrReplaceFile mutation and /restore (ui/shell/slash.py:1652-1798) replays it, covering the main undo need; per-call edits are atomic-per-file because replace.py validates the whole batch in memory first. But results don't report old/overwritten content, there is no aggregated session 'what changed this turn' delta, and exactness of failed writes is untracked. + +**Verifier note.** Verdict 'partial' is right, but the claimed state overstates one gap: an aggregated per-turn 'what changed this turn' summary DOES exist at path/count granularity — the TUI turn recap collects every DiffDisplayBlock path from tool results and reports 'N files changed'; subagent blocks similarly aggregate changed-file lists. What is genuinely absent: content-level deltas (old/overwritten text is not in model-facing tool result messages — WriteFile reports only byte size, StrReplaceFile only replacement counts; diffs are UI display blocks), and any exactness/failure tracking on deltas. + +**Adopt.** Extend tool results (or a session-side ledger keyed off restore points) with the applied-change record {path, op, old_digest, new_digest, overwritten} and an exact/inexact bit set when a write raises after partially executing; surface it in session recap and /restore listings so multi-file work and failures are precisely reconstructable. Low urgency while edits stay one-file-per-call. + +**Files.** `<ref>/apply-patch/src/lib.rs`, `src/pythinker_code/file_restore.py`, `src/pythinker_code/ui/shell/slash.py` + +### `persistence-resume/cold-session-compression-with-transparent-readers-and-atomic` — missing, M, low + +**Today.** Missing. Pythinker resolves disk growth by deleting archived sessions older than 30 days outright (src/pythinker_code/session_cleanup.py sweep_old_sessions), making retention vs disk a hard tradeoff; context/wire files are always plain JSONL (src/pythinker_code/session.py, src/pythinker_code/wire/file.py). + +**Verifier note.** Claim confirmed. Disk growth is handled by deletion, not compression: sweep_old_sessions removes archived session dirs older than max_age_days (default doc says 30-day retention) via shutil.rmtree, and only archived=True sessions are eligible. context.jsonl/wire.jsonl are always plain JSONL; rg for gzip/zstd/lzma/bz2/zlib across src finds only FastAPI GZipMiddleware (web/vis HTTP responses) and archive file-extension lists in tools/file — nothing compresses session storage, and there is no compressed-transparent reader or materialize-on-append path. + +**Adopt.** Optional startup job (after the existing sweep) gzip-compresses context.jsonl/wire.jsonl of archived or long-idle sessions, guarded by a marker file; Session.find/Context.restore and the picker's title derivation transparently open .gz, and resuming decompresses atomically (mkstemp + os.replace, never clobbering an existing plain file) before append. Lets the retention sweep keep sessions resumable far longer at low disk cost. Low priority given recall + cleanup already manage history. + +**Files.** `src/pythinker_code/session_cleanup.py`, `src/pythinker_code/session.py`, `<ref>/rollout/src/compression.rs` + +### `protocol-headless/ephemeral-no-persistence-headless-runs` — missing, M, low + +**Today.** Missing. Every run creates a persisted Session with context/wire files (src/pythinker_code/session.py is always disk-backed; rg for ephemeral/no-save in src/pythinker_code finds only unrelated hits); cleanup relies on empty-session deletion and session_cleanup.py pruning rather than an opt-out. + +**Verifier note.** Claim confirmed for agent runs; could not refute. Session.create always mkdirs and persists under the session dir (src/pythinker_code/session.py:88-91 dir property mkdir, 188-203 create with metadata write), and the --print path in cli/__init__.py always constructs a Session before run_print (used for SessionStart/SessionEnd hooks at lines ~995-1060). No --ephemeral/--no-save flag exists on the main CLI (flag inventory lines 332-638). Cleanup is indeed deletion-based: _delete_empty_session (cli/__init__.py:~1119) plus session_cleanup.py sweeps. Nearest miss found while trying to refute: `pythinker review diff --no-save` / `pythinker secscan diff --no-save` (referenced in agents/default/code_reviewer.yaml:59 and security_reviewer.yaml:33, implemented in the delegated pythinker_review package per src/pythinker_code/cli/review.py) — but that is the standalone review pipeline's run-state, not a headless agent session, so it does not overturn the verdict. + +**Adopt.** Add --ephemeral for print mode: create the session under a temp dir (or a null wire-file backend) and delete it in _post_run regardless of exit code, skipping metadata last_session_id updates and journal recap. Useful for CI fan-out runs that would otherwise pollute session listings and persist prompt contents containing secrets. + +**Files.** `src/pythinker_code/session.py`, `src/pythinker_code/cli/__init__.py`, `<ref>/exec/src/cli.rs` + +### `config-features/machine-level-config-plus-admin-constraint-requirements-laye` — missing, L, low + +**Today.** Missing. Pythinker has no layer below user scope and no value-set constraints; SCOPE_LOCKED_PATHS only restricts which scope may set a key, not what values are allowed (src/pythinker_code/config.py). + +**Verifier note.** Claim confirmed. Scope chain is user -> project -> local -> env only; no system/machine layer (no /etc or platform-wide path) and no value-set constraint mechanism — SCOPE_LOCKED_PATHS only restricts which scope may set a key. + +**Adopt.** Only worth it for shared/CI machines: an optional /etc/pythinker/config.toml lowest-precedence scope plus a tiny requirements file that pins values like default_yolo=false or an approval-policy allowlist, rejecting overrides with an error naming the requirements file. Defer unless team/enterprise deployment becomes a goal. + +**Files.** `src/pythinker_code/config.py`, `<ref>/config/src/config_requirements.rs`, `<ref>/config/src/constraint.rs` + +### `patch-file-tools/streaming-edit-argument-parser-for-live-diff-preview` — partial, L, low + +**Today.** rg over src/pythinker_code finds no partial tool-argument (input-json-delta) handling; diffs are computed only after the full tool call arrives, at approval time (utils/diff.py build_diff_blocks), so long writes render nothing until complete. + +**Verifier note.** Claimed 'missing' is wrong on its core premise: pythinker DOES have partial tool-argument streaming parsing. streamingjson.Lexer is fed argument deltas as they stream and the parsed partial JSON live-updates the tool card's key argument (e.g. file path, command) in both the TUI and ACP frontends. What is missing is only the last mile: streamed old/new content is not turned into a live diff preview — build_diff_blocks runs after the full call arrives, at approval time. Correct verdict: partial. + +**Adopt.** If/when the wire layer exposes streaming tool-arg deltas, add an incremental JSON-prefix parser for WriteFile/StrReplaceFile args that extracts path and growing content to render a live 'writing path (+N lines)' preview in the worklog. Requires wire-level plumbing first; cosmetic payoff only. + +**Files.** `<ref>/apply-patch/src/streaming_parser.rs`, `src/pythinker_code/utils/diff.py` + +## Refuted claims — do NOT implement (already covered) + +- `core-loop/review-mode-with-machine-parsable-structured-findings`: Claim REFUTED on its core assertion. Pythinker DOES have a machine-parsable structured findings schema parsed with fallback: a fenced ```report JSON block (title, severity from the fixed critical|high|medium|low|info set, location 'path:line-range', body) defined as a base contract in agents/default + +- `observability-feedback/startup-phase-duration-telemetry`: Claim REFUTED. 'No boot timing reaches telemetry' is factually wrong: PythinkerCLI.create() times the boot end-to-end and per phase, then emits track('startup_perf', duration_ms=..., config_ms=..., init_ms=..., mcp_ms=...) at the end of startup. The phase decomposition (config load / runtime init / + +- `tools-registry-codemode/streamed-tool-argument-diff-consumers-live-ui-rendering-from`: REFUTED — the TUI does have a per-tool partial-argument renderer. _live_view.append_tool_call_part feeds each ToolCallPart.arguments_part into _ToolCallBlock.append_args_part, which runs a streaming-JSON repair lexer (streamingjson.Lexer), live-updates the row's argument summary (_extract_worklog_ar + +## Cluster design summaries (reference architecture, for orientation) + +### config-features + +The reference harness builds configuration as an ordered stack of layers (machine/system -> managed -> base user -> named profile overlay -> project layers from root to cwd -> per-key CLI/session overrides), each carried as a ConfigLayerEntry with a sha256 content fingerprint, per-field origin tracking, and an optional disabled_reason so layers can be loaded-but-inactive. Project-scope config is security-gated: a persisted per-project trust map decides whether repo-controlled config/hooks/exec-policies apply (untrusted layers are shown disabled with an actionable reason, never executed), and a denylist of sensitive keys is sanitized out of project config with a startup warning rather than a fatal error. A separate admin "requirements" layer expresses constraints (allowed value sets via Constrained<T> with source attribution) rather than values. Features are governed by a single registry of FeatureSpec entries with lifecycle stages (UnderDevelopment/Experimental/Stable/Deprecated/Removed), default-enabled bits, dependency normalization, legacy-alias deprecation notices, unknown-key diagnostics with file:line ranges (strict mode), and JSON-schema export for editors. Pythinker already has a solid simplified core (user/project/local scopes, type-based merge with provenance, env overlay, scope-locked secret paths), so the meaningful gaps are trust gating, graceful degradation, feature staging, and config observability/editing quality. + +### context-mgmt + +The reference harness treats the context window as a managed resource with explicit invariants and a full compaction state machine. A history manager (core/src/context_manager/history.rs, normalize.rs) records items with record-time truncation, enforces call/output pairing and modality invariants at prompt build, and estimates tokens from serialized bytes with calibrated discounts for images and encrypted content. Compaction (core/src/compact.rs, session/turn.rs) runs in distinct phases (pre-turn, mid-turn, manual) for distinct reasons (token limit under two scopes, model downshift, instruction-hash change), preserves verbatim user messages within a token budget plus a summary, self-trims and retries when the compaction request itself overflows, and re-injects initial context at model-expected positions. The model is made context-aware via injected token-budget fragments at usage thresholds, a remaining-context query tool, and a fresh-context-window request tool; settings/environment changes between turns are re-injected as diffs against a baseline snapshot (context_manager/updates.rs, context-fragments/). A hardened append-only JSONL file (message-history/src/lib.rs) persists cross-session prompt history with locking and byte caps. Pythinker already covers the compaction lifecycle, a cheap prune tier, hooks, rollback safety, and post-compaction restoration well; the meaningful gaps are overflow recovery, restore-time pairing repair, model-visible budget signals, verbatim user-message retention, and context carry-over on model switch. + +### core-loop + +The reference harness structures its core loop as session-owned tasks (regular/review/compact/user-shell kinds) driving a turn loop: each iteration drains queued mid-turn user input and inter-agent mail, builds a sampling request from history, streams the model response, executes tools, then re-evaluates token status. Robustness is layered in at every joint: pre-turn and mid-turn auto-compaction keyed to server-observed token usage (including compaction with the PREVIOUS model when switching models or downshifting context windows), retry/backoff with user-visible reconnect notices and transport fallback, reactive recovery from hard context-overflow and invalid-image errors, model-visible interrupted-turn markers, token-budget-remaining notices injected at usage thresholds, a per-turn aggregated diff tracker, and stop-hook continuation governance (key files: session/turn.rs, session/input_queue.rs, tasks/mod.rs, tasks/review.rs, session/token_budget.rs, responses_retry.rs, turn_diff_tracker.rs, state/auto_compact_window.rs). Pythinker's soul loop already matches most of this design — its gaps are concentrated in reactive (error-driven) context recovery, model-visible interruption/budget signals, model-switch continuity, and turn-level diff aggregation. + +### exec-safety + +The reference harness layers shell-exec safety in five tiers: (1) a declarative, user-extensible execution-policy engine (prefix rules with allow/prompt/forbidden decisions, per-rule justifications, load-time-validated match/not_match example tests, host-executable path pinning, strictest-decision aggregation, heuristics fallback when no rule matches); (2) a structural bash parser (tree-sitter grammar, whitelisted node kinds only) that auto-approves provably read-only commands — including `bash -lc "safe && safe"` composites — against a curated safelist with per-flag escape hatches (find -exec, rg --pre, git global-option bypass hardening); (3) OS sandboxes (seatbelt/landlock+bwrap/restricted token) with network disabled by default and a run-sandboxed-first, escalate-to-approval-on-denial lifecycle, plus an in-sandbox execve-interception protocol (Run/Escalate/Deny per nested command); (4) durable policy amendment — "always allow" appends a dedup-checked, file-locked allow rule (command prefix or network host) to the user's rules file; (5) pre-main process hardening (core dumps off, ptrace-attach denied, LD_/DYLD_ stripped). Pythinker already has a strong heuristic counterpart (profile-gated mutation/destructive/network/workspace-escape classifiers, signature-scoped session approval, deliberation backstop, secret-env scrubbing) but lacks the safe-command prompt-elision tier, any user-extensible policy language, durable cross-session rules, OS-level enforcement, and Windows/PowerShell-aware classification. + +### mcp + +The reference harness treats MCP as a managed connection fleet: a connection manager starts every enabled server concurrently under a cancellation token, emits per-server startup status events plus an aggregate completion summary, serves cached tool snapshots while servers are still connecting, and gates session start only on servers explicitly marked required. Robustness is layered into the client itself — per-server startup/tool timeouts with actionable error text, bounded retry/backoff for retryable HTTP initialize failures, OAuth discovery/scope resolution, per-server tool allow/deny filters enforced at both list and call time, and model-visible tool-name normalization (charset sanitize, collision hashing, 64-char cap) that preserves raw names for protocol routing. It also keeps persistent sessions so server-initiated traffic (elicitation requests with policy-based auto-accept/deny, log/progress/list-changed notifications) is handled, and it ships a reverse mode exposing the whole agent as an MCP tool over stdio. Pythinker (fastmcp-based) already covers non-blocking startup with status snapshots, OAuth login/pre-check, resource/prompt listing+reading, output budgeting, and hardened teardown, but connects once, freezes the tool list, drops the session between calls, and lacks required-server gating, startup timeouts, tool filtering, name normalization, and any server-initiated request handling. + +### multi-agent + +The reference harness treats child agents as long-lived, addressable threads rather than one-shot workers: a v1/v2 collaboration tool suite (spawn_agent, send_input with optional interrupt, wait on multiple targets, list/close/resume) lets the orchestrator steer running children mid-task; spawn supports role overlays (config layers with model/effort/instructions), full-history context forking, and depth/thread guardrails enforced by a registry with friendly model-facing errors. Around that core sit batch primitives (CSV-driven job fan-out with bounded concurrency and a worker-side result-reporting tool), a cloud best-of-N attempt surface (1-4 parallel attempts of the same prompt with sibling diff comparison and selection), a storage-neutral persisted parent/child spawn-edge graph with open/closed lifecycle and BFS descendant queries, and collaboration-mode presets (Plan/Default masks bundling model, reasoning effort, and developer instructions, with per-mode capability gates). Pythinker already covers most of the structural ground (typed subagents with permission profiles, foreground/background runners with resume, RunAgents fan-out with capacity slots and orchestration approval, crash recovery, completion notifications); the real gaps are interactivity (steering a live child), context forking at spawn, enforced workspace isolation for parallel writers, and multi-target waiting. + +### observability-feedback + +The reference harness splits observability into four layers: (1) an OTel layer with a session-scoped business-event emitter that stamps every event/metric with session metadata tags (auth mode, originator, session source, model, app version), a rich event taxonomy (conversation_starts, per-attempt api_request, per-stream-event sse/websocket health, tool_decision, sandbox_outcome, tool_result, startup_phase, turn TTFT), and strict metric tag validation/bounded-cardinality normalization; (2) a centralized analytics fact-reducer that consumes the protocol event stream on a bounded queue and emits consolidated per-turn rollup events (tool-type counts, token usage, steer count, error kind, resolved config, and a five-segment turn latency profile) plus privacy-hashed accepted-line counts parsed from unified diffs as a code-retention outcome metric; (3) a feedback subsystem with a process-wide full-fidelity log ring buffer independent of the console filter, a structured feedback-tags layer accumulating session diagnostics tags, connectivity diagnostics (proxy env detection), and consent-gated uploads with ordered attachments; and (4) opt-in local raw-evidence trace bundles (seq-ordered event log + payload files + offline reducer) that separate model-visible conversation from runtime observations for deep failure forensics, plus HTTP response debug-context extraction (request-id/gateway headers) with deliberately coarse telemetry error messages. Pythinker already has a mature equivalent of layer 1's core (OTel traces/metrics/logs with nested turn→llm→tool spans, GenAI semconv usage attrs, crash handlers, expected-error classification, recent-errors ring, broad track() taxonomy including approval decisions and compaction) and a strong consent-gated /feedback + /report-error pipeline with redaction; the meaningful gaps are per-turn rollup facts, turn latency decomposition/TTFT, code-retention analytics, feedback log/diagnostics attachments, and the local trace bundle. + +### patch-file-tools + +The reference harness ships a four-part file-manipulation stack: (1) apply-patch — a dedicated multi-file patch DSL (Add/Update/Delete/Move hunks) parsed strictly-but-leniently, located in files via a graduated fuzzy context matcher (exact → rstrip → strip → Unicode-punctuation normalization, EOF-anchored), applied with committed-change delta tracking (old/overwritten content + an `exact` flag that flips when a failed write may have mutated state), shell-invocation interception (AST-parses bash/pwsh/cmd heredoc forms incl. `cd dir &&` prefixes), and a streaming parser for live patch preview; (2) file-system — an async FS trait abstracting local/remote execution; (3) file-search — a persistent background walker + fuzzy-matcher session with cheap re-query, debounced top-N snapshots, cancellation, and highlight indices; (4) file-watcher — a refcounted shared OS watcher with per-subscriber coalesced/debounced/throttled receivers and missing-path ancestor fallback, used for skills hot-reload and fs-change notifications. Pythinker's str-replace-based editing stack is robust on single-file paths (batch in-memory validation, CRLF fallback, restore points, arg-shape normalization, symlink-resolved workspace checks) but lacks fuzzy edit-location recovery, first-class delete/rename, shell-mediated-edit interception, any file watcher, and large-repo-scalable fuzzy file search. + +### persistence-resume + +The reference harness persists each session as an append-only JSONL rollout of canonical model items plus a whitelisted subset of UI events (explicit persistence policy), written by a dedicated writer task with flush acks and latched terminal failure. A SQLite state runtime indexes thread metadata (title, preview, cwd, git provenance, token usage, archive state) and is kept honest by a lease-guarded, watermark-checkpointed backfill that re-extracts metadata from rollout files and self-repairs the index; listing supports stable timestamp+uuid cursors, sort/filter, and ripgrep-accelerated full-content search with snippets. Cold rollouts are transparently zstd-compressed and atomically rematerialized on append. Sessions created by a different agent harness are detected in that agent's home dir, converted into native rollout items as resumable threads, and deduplicated via a content-sha256 import ledger. Corrupt runtime SQLite stores are quarantined (DB + WAL/SHM moved to a timestamped backup dir) and rebuilt without touching sibling stores. Pythinker already has strong parity on the core log (context.jsonl/wire.jsonl with control records, torn-line resilience, flock single-writer), fork/undo, archive lifecycle, and resume replay; the real gaps are a persisted metadata index, session provenance (git/version/lineage), transcript content search, structured cross-agent import, and cold-session compression. + +### prompts-instructions + +The reference harness treats prompting as a layered, state-synchronized system rather than one static file: (1) distinct base system prompts per model family, scaled to model capability (a ~68-line minimal prompt for agent-tuned models vs ~300-line detailed prompts with worked planning examples for general models), plus a templated model-instructions file with a {{personality}} slot filled from curated personality presets; (2) switchable collaboration-mode developer-message templates (default / plan / pair-programming / execute), each explicitly voiding prior mode guidance, with mode changes driven only by developer messages and never user intent; plan mode is a conversational, decision-complete protocol with an explicit taxonomy for discoverable-fact vs preference unknowns and plan-document compactness rules; (3) dynamically composed permissions instructions rendered from the live sandbox/approval config — sandbox mode, writable roots, denied reads, approval-policy variant, already-approved command prefixes, command-segmentation semantics — plus a model-initiated escalation protocol (justification question + suggested scoped reusable allow-prefix with banned-prefix guidance); (4) hierarchical AGENTS.md scope/precedence guidance; (5) a review lifecycle with harness-resolved targets (uncommitted / base-branch with precomputed merge-base SHA / commit) feeding a strict bug-qualification rubric, and structured result re-entry into main history; (6) compaction and goal-mode prompt templates. Pythinker has already adopted the AGENTS.md hierarchy, the reviewer rubric, compaction/goal templates, todo discipline, dirty-worktree guardrails, and a stronger tool-enforced plan mode; the real gaps are the dynamic permissions-state prompt block, model-initiated escalation with justification/suggested rules, decision-complete plan interviewing, and the review-target command lifecycle. + +### protocol-headless + +The reference harness treats headless (non-interactive) runs as a first-class automation surface built on the same typed protocol as its interactive clients. A thin exec runner is a JSON-RPC client of an in-process app server (thread/start, turn/start, turn/interrupt, thread/read) and renders events through one of two strictly separated processors: a JSONL mode where stdout carries exactly one stable, versioned event per line (thread.started → turn.started → item.started/updated/completed → turn.completed{usage} / turn.failed / error, with normalized sequential item ids and typed item payloads), and a human mode where ALL progress goes to stderr and the final answer goes to stdout only when piped — making `$(...)` capture safe. Automation affordances include: structured final output validated against a caller-supplied JSON Schema (--output-schema), final-message-to-file (-o), nonzero exit on turn failure/interrupt, approvals forced to never with every interactive server request auto-rejected/auto-cancelled with an explanatory reason, a git-repo trust gate before unattended runs, --ephemeral no-persistence runs, resume-by-id/name/--last with cwd filtering, robust stdin prompt contract (`-` sentinel, piped-stdin-appended-as-tagged-block, BOM/UTF-16 detection), and turn-completed item backfill via thread/read when event delivery dropped items. Protocol types are exported as checked-in JSON Schema/TS fixtures so external clients can codegen against a versioned API. + +### review-mode + +The reference harness ships code review as a first-class session mode. A /review command opens preset pickers (uncommitted changes, base branch via a searchable branch list, one of the last 100 commits, or custom instructions); the chosen target is deterministically resolved into a synthesized prompt whose merge-base SHA is precomputed (preferring the branch's upstream when the remote is ahead), with a fallback prompt that teaches the model to compute the base itself. The review runs as a one-shot child thread that clones the parent config but force-disables web/collab/image tools, clamps approvals to never-ask, swaps in a dedicated review rubric as base instructions, and optionally uses a dedicated review model. The reviewer must emit strict JSON findings (priority-tagged titles P0-P3, per-finding confidence, file+line-range anchors, overall_correctness verdict); parsing is lenient (first {...} substring, then plain-text fallback into overall_explanation, never lost). On exit — including abort — the parent history records a synthetic findings block plus an explicit "review was interrupted, re-run" message, the UI flips a review-mode banner and restores the pre-review token display, and findings can be checkbox-selected to dispatch fixes. A separate "guardian" mechanism reuses the same delegate pattern as an LLM approval reviewer: it rebuilds a token-capped transcript, assesses the exact planned action into strict JSON {risk_level, user_authorization, outcome, rationale}, fails closed on 90s timeout or malformed output, and has circuit breakers (max 3 consecutive denials/turn, denial windows, special developer prefix when the user manually overrides a denial). Pythinker already covers most of the reviewer-quality surface (rubric-calibrated reviewer subagents, offline fail-closed review profiles, a full deterministic diff-review engine with confidence-scored schema-validated findings, report-block rendering, per-agent model overrides); the real gaps are the interactive TUI entry point with git pickers, deterministic target-to-prompt synthesis for agent-mediated review dispatch, upstream-aware merge-base, findings triage-to-fix, and the LLM approval guardian. + +### skills-hooks-memories + +The reference harness treats skills, hooks, and durable memories as three tightly engineered subsystems. Skills: layered SKILL.md discovery (repo/user/system/admin scopes, bounded scan, frontmatter + metadata sidecar with per-skill policy), a token-budgeted prompt listing (2% of context window, graceful description truncation, aliased root paths, omission warnings + telemetry), explicit $-mention resolution with ambiguity guards, implicit-invocation detection (model reading a skill doc or running its scripts counts as invocation), and a detailed "how to use skills" doctrine (trigger rules, read-to-EOF, no subagent delegation of skill reading, minimal-set + announce). Hooks: 10 lifecycle events with matchers and a JSON-schema stdin/stdout contract whose outputs can block, rewrite tool input, decide permission requests, inject additional model context, return post-tool feedback, and force turn continuation from Stop hooks; handlers carry a hash-based trust identity (managed/trusted/modified/untrusted) so repo/plugin-provided hooks never execute until trusted, and oversized hook output spills to disk with a head/tail preview plus recovery path. Memories: a two-phase background pipeline — Phase 1 LLM extraction per past session (DB-leased jobs, concurrency caps, retry backoff, strict no-op gate, outcome triage, preference-signal-first prompts, secret redaction) and Phase 2 a sandboxed consolidation agent operating on a git-baselined memory workspace whose diff drives incremental update and forgetting, producing an always-loaded summary, a greppable handbook, rollout summaries, and even auto-authored skills — with a read-path usage/citation tracker that feeds usage counts back into retention ranking, and a rate-limit guard that skips background work when quota is low. + +### tools-registry-codemode + +The reference harness factors model-visible tooling into a dedicated layer: ToolSpec/ToolName models with namespacing, a registry of typed tool executors carrying per-tool metadata (exposure, parallel-safety, cancellation semantics, hook payload contracts, search text), and a router that dispatches calls under a read/write concurrency gate with teardown-aware abort handling. Around it sit three robustness subsystems: JSON-schema sanitization plus budgeted compaction for foreign (MCP/dynamic) tool schemas, deferred tool loading with an on-demand tool-search loader, and a unified interactive PTY exec manager (persistent process ids, stdin writes with clamped yield windows, head+tail capped transcripts, LRU pruning of up to 64 sessions). On top is a tools-as-code mode: an exec tool runs model-written script in an isolated runtime where every nested tool is a callable, with resumable long-running cells (wait/terminate), cross-cell store/load, and incremental notify streaming. Pythinker already has a solid registry (dedup, hooks, telemetry, policy-based visibility) and a strong background-task subsystem, but lacks the concurrency policy, schema sanitization, tool-search deferral, PTY interactivity, head+tail retention, and any code mode. diff --git a/tasks/todo.md b/tasks/todo.md index 7a9e06e4..7539c922 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -3,11 +3,13 @@ ## Active - [ ] Agent-harness adoption arc (`feat/agent-harness-enhancements`): port the - reference harness's remaining coding-agent design (blackbox/agent_x) into - pythinker, generically framed. Checkpoint 0 committed (`047a0b29`): - orchestration injection provider + name-scrub. Gap-map workflow running - (14 clusters, map+verify); next: synthesize adoption plan → - checkpointed TDD implementation, clean-code-guard per checkpoint. + reference harness's remaining coding-agent design into pythinker, + generically framed. Gap map DONE: 124 verified items (3 refuted) ranked in + `tasks/agent-harness-adoption-plan.md` (tiers 1-4, execution discipline + inside). Done so far: checkpoint 0 `047a0b29` (orchestration injection + provider + name-scrub), checkpoint 1 `b40cdb71` (ACP hides + AskUserQuestion). Now executing Tier 1 checkpoints: TDD per item, + clean-code-guard per checkpoint, make check + pytest green per commit. - [ ] Windows shell hardening (researched, not yet implemented): bash-first shell policy (Git Bash probe → pwsh → powershell, never cmd), Windows tool-description guidance (`;` not `&&` on PS 5.1, `$env:`, quoting), From e722278ccc85563dc9229279d7716262b725976b Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:23:44 -0400 Subject: [PATCH 09/49] feat(soul): repair tool call/result pairing at context restore A crash between persisting an assistant tool-call message and its tool results leaves context.jsonl with a dangling call or an orphaned result; after resume every provider request then fails with a pairing error. restore() and revert_to() now run a pure repair pass that synthesizes an explicit lost-result message for unpaired calls and drops orphaned or duplicate tool results, logging each repair. The file is never rewritten, so re-repair on each restore stays idempotent. Plan item: context-mgmt/history-invariant-repair (Tier 1). --- src/pythinker_code/soul/context.py | 56 ++++++- tests/core/test_context_history_repair.py | 170 ++++++++++++++++++++++ 2 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 tests/core/test_context_history_repair.py diff --git a/src/pythinker_code/soul/context.py b/src/pythinker_code/soul/context.py index 53b54171..80f20052 100644 --- a/src/pythinker_code/soul/context.py +++ b/src/pythinker_code/soul/context.py @@ -12,7 +12,7 @@ import aiofiles import aiofiles.os from pydantic import ValidationError -from pythinker_core.message import Message +from pythinker_core.message import Message, TextPart from pythinker_code.soul.compaction import estimate_text_tokens from pythinker_code.soul.message import system @@ -20,6 +20,58 @@ from pythinker_code.utils.logging import logger from pythinker_code.utils.path import next_available_rotation +_LOST_RESULT_NOTE = ( + "Tool call result was lost before it could be recorded (the session ended " + "unexpectedly). Re-run the tool if its output is still needed." +) + + +def repair_history_invariants(history: Sequence[Message]) -> list[Message]: + """Restore tool call/result pairing broken by a crash mid-persistence. + + Synthesizes a lost-result message for every assistant tool call without a + recorded result and drops tool results with no matching open call — + either anomaly makes every subsequent provider request fail with a + pairing error. Runtime appends are pair-shielded, so this runs only at + the restore boundary; it never rewrites the file, so re-repair on each + restore is idempotent. + """ + repaired: list[Message] = [] + open_call_ids: list[str] = [] + + def _synthesize_lost_results() -> None: + for call_id in open_call_ids: + logger.warning( + "Context repair: synthesizing lost result for tool call {call_id}", + call_id=call_id, + ) + repaired.append( + Message( + role="tool", + content=[TextPart(text=_LOST_RESULT_NOTE)], + tool_call_id=call_id, + ) + ) + open_call_ids.clear() + + for message in history: + if message.role == "tool": + if message.tool_call_id in open_call_ids: + open_call_ids.remove(message.tool_call_id) + repaired.append(message) + else: + logger.warning( + "Context repair: dropping orphaned tool result for {call_id}", + call_id=message.tool_call_id, + ) + continue + _synthesize_lost_results() + repaired.append(message) + if message.role == "assistant" and message.tool_calls: + open_call_ids.extend(call.id for call in message.tool_calls) + _synthesize_lost_results() + return repaired + class Context: def __init__(self, file_backend: Path): @@ -77,6 +129,7 @@ async def restore(self) -> bool: line_no=line_no, ) + self._history[:] = repair_history_invariants(self._history) self._pending_token_estimate = estimate_text_tokens(messages_after_last_usage) return True @@ -228,6 +281,7 @@ async def revert_to(self, checkpoint_id: int): if keep_line: await new_file.write(line) + self._history[:] = repair_history_invariants(self._history) self._pending_token_estimate = estimate_text_tokens(messages_after_last_usage) async def clear(self): diff --git a/tests/core/test_context_history_repair.py b/tests/core/test_context_history_repair.py new file mode 100644 index 00000000..51df4949 --- /dev/null +++ b/tests/core/test_context_history_repair.py @@ -0,0 +1,170 @@ +"""Restore-time history invariant repair. + +A crash between persisting an assistant tool-call message and its tool +results leaves a dangling call (or an orphaned result) in context.jsonl. +Without repair, every subsequent API call fails with a call/result pairing +error. Repair happens only at the restore boundary — runtime appends are +already pair-shielded — and never rewrites the file, so it must be +idempotent across restores. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pythinker_core.message import Message, ToolCall + +from pythinker_code.soul.context import Context +from pythinker_code.wire.types import TextPart + + +def _tool_call(call_id: str, name: str = "Shell") -> ToolCall: + return ToolCall.model_validate( + {"type": "function", "id": call_id, "function": {"name": name, "arguments": "{}"}} + ) + + +def _user(text: str) -> dict: + return json.loads( + Message(role="user", content=[TextPart(text=text)]).model_dump_json(exclude_none=True) + ) + + +def _assistant_with_calls(*call_ids: str) -> dict: + message = Message( + role="assistant", + content=[TextPart(text="running tools")], + tool_calls=[_tool_call(cid) for cid in call_ids], + ) + return json.loads(message.model_dump_json(exclude_none=True)) + + +def _tool_result(call_id: str, text: str = "done") -> dict: + message = Message(role="tool", content=[TextPart(text=text)], tool_call_id=call_id) + return json.loads(message.model_dump_json(exclude_none=True)) + + +def _write_lines(path: Path, lines: list[dict]) -> None: + path.write_text("".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8") + + +async def _restore(path: Path) -> Context: + ctx = Context(file_backend=path) + assert await ctx.restore() + return ctx + + +@pytest.mark.asyncio +async def test_intact_history_is_unchanged(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines( + path, + [_user("hi"), _assistant_with_calls("call_1"), _tool_result("call_1")], + ) + + ctx = await _restore(path) + + assert [m.role for m in ctx.history] == ["user", "assistant", "tool"] + + +@pytest.mark.asyncio +async def test_dangling_call_at_end_gets_synthetic_result(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines(path, [_user("hi"), _assistant_with_calls("call_1")]) + + ctx = await _restore(path) + + assert [m.role for m in ctx.history] == ["user", "assistant", "tool"] + synthetic = ctx.history[-1] + assert synthetic.tool_call_id == "call_1" + assert "was lost" in synthetic.extract_text(" ") + + +@pytest.mark.asyncio +async def test_partially_recorded_results_are_completed(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines( + path, + [_assistant_with_calls("call_1", "call_2"), _tool_result("call_1")], + ) + + ctx = await _restore(path) + + tool_ids = [m.tool_call_id for m in ctx.history if m.role == "tool"] + assert tool_ids == ["call_1", "call_2"] + + +@pytest.mark.asyncio +async def test_dangling_call_followed_by_user_message(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines(path, [_assistant_with_calls("call_1"), _user("next task")]) + + ctx = await _restore(path) + + assert [m.role for m in ctx.history] == ["assistant", "tool", "user"] + assert ctx.history[1].tool_call_id == "call_1" + + +@pytest.mark.asyncio +async def test_orphaned_tool_result_is_dropped(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines(path, [_user("hi"), _tool_result("call_ghost")]) + + ctx = await _restore(path) + + assert [m.role for m in ctx.history] == ["user"] + + +@pytest.mark.asyncio +async def test_duplicate_tool_result_is_dropped(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines( + path, + [ + _assistant_with_calls("call_1"), + _tool_result("call_1"), + _tool_result("call_1", text="duplicate"), + ], + ) + + ctx = await _restore(path) + + tool_messages = [m for m in ctx.history if m.role == "tool"] + assert len(tool_messages) == 1 + assert tool_messages[0].extract_text(" ") == "done" + + +@pytest.mark.asyncio +async def test_repair_is_idempotent_across_restores(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines(path, [_assistant_with_calls("call_1"), _user("next")]) + before = path.read_text(encoding="utf-8") + + first = await _restore(path) + second = await _restore(path) + + assert path.read_text(encoding="utf-8") == before + assert [m.role for m in first.history] == [m.role for m in second.history] + assert [m.tool_call_id for m in first.history] == [m.tool_call_id for m in second.history] + + +@pytest.mark.asyncio +async def test_revert_to_also_repairs(tmp_path: Path) -> None: + path = tmp_path / "context.jsonl" + _write_lines( + path, + [ + {"role": "_checkpoint", "id": 0}, + _assistant_with_calls("call_1"), + {"role": "_checkpoint", "id": 1}, + _user("after"), + ], + ) + + ctx = await _restore(path) + await ctx.revert_to(1) + + assert [m.role for m in ctx.history] == ["assistant", "tool"] + assert ctx.history[-1].tool_call_id == "call_1" From 388da2d3610040b9f8d20e5bb2bd881a52e3e2c4 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:25:46 -0400 Subject: [PATCH 10/49] feat(soul): decision-complete planning protocol in plan-mode reminders Plan mode now teaches a two-kinds-of-unknowns rule (explore repo-discoverable facts yourself; surface preference/scope decisions early via AskUserQuestion with a recommended default), records unanswered defaults under an Assumptions section, gates ExitPlanMode on a decision-complete plan (no decisions left to the implementer), and adds a plan-file brevity rubric (3-5 short sections, subsystem-grouped bullets). Phrase pins lock the new clauses into all three reminder variants. Plan item: prompts-instructions/decision-complete-plan-mode (Tier 1). --- .../soul/dynamic_injections/plan_mode.py | 21 ++++++++++-- .../core/test_plan_mode_injection_provider.py | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/pythinker_code/soul/dynamic_injections/plan_mode.py b/src/pythinker_code/soul/dynamic_injections/plan_mode.py index c31213a8..bd090e0c 100644 --- a/src/pythinker_code/soul/dynamic_injections/plan_mode.py +++ b/src/pythinker_code/soul/dynamic_injections/plan_mode.py @@ -151,12 +151,26 @@ def _full_reminder( "4. Write Plan — modify the plan file with WriteFile or StrReplaceFile " "(use WriteFile if the plan file does not exist yet). The plan MUST include " "a Verification section: for each change, the smallest command, test, or " - "check that would prove it worked end-to-end", - "5. Exit — call ExitPlanMode for user approval", + "check that would prove it worked end-to-end. Keep the plan file " + "skimmable: 3-5 short sections (including Assumptions and Verification), " + "bullets grouped by subsystem, naming only load-bearing files", + "5. Exit — call ExitPlanMode for user approval, only once the plan is " + "decision-complete: an implementer could execute it without making any " + 'decision themselves. "Figure out X during implementation" is not a plan ' + "step — resolve it now by exploring, or ask", ] ) lines.extend( [ + "", + "## Resolving unknowns", + "Unknowns come in two kinds. Repo-discoverable facts (current behavior, " + "existing patterns, file locations): explore and answer them yourself — " + "never ask the user. Preference or scope decisions (product behavior, " + "tradeoff priorities, rollout): surface them early with AskUserQuestion, " + "offering 2-4 concrete options with your recommended default first.", + "If a preference question stays unanswered, proceed with your recommended " + "default and record it in the plan's Assumptions section.", "", "## Handling multiple approaches", "Keep it focused: at most 2-3 meaningfully different approaches. " @@ -205,6 +219,8 @@ def _sparse_reminder(plan_file_path: str | None = None) -> str: ) parts.extend( [ + "Exit only with a decision-complete plan; " + "record unconfirmed defaults under Assumptions.", "Use AskUserQuestion to clarify user preferences " "when it helps you write a better plan.", "If the plan has multiple approaches, " @@ -246,6 +262,7 @@ def _reentry_reminder(plan_file_path: str | None = None) -> str: "or user preferences that affect the plan.", "6. Always edit the plan file before calling ExitPlanMode.", "", + "Exit only when the plan is decision-complete (no decisions left to the implementer).", "Your turn must end with either AskUserQuestion (to clarify requirements) " "or ExitPlanMode (to request plan approval).", ] diff --git a/tests/core/test_plan_mode_injection_provider.py b/tests/core/test_plan_mode_injection_provider.py index 140278cf..36a34f90 100644 --- a/tests/core/test_plan_mode_injection_provider.py +++ b/tests/core/test_plan_mode_injection_provider.py @@ -170,3 +170,37 @@ def test_sparse_reminder_requires_verification_section(self) -> None: def test_reentry_reminder_requires_verification_section(self) -> None: assert "Verification section" in _reentry_reminder("/tmp/plan.md") + + +class TestPlanModeDecisionCompleteness: + """Lock the unknowns-resolution protocol, the decision-complete exit bar, + and the plan-file shape rubric into the plan-mode reminders so they cannot + silently drift out of the authoring instructions.""" + + def test_full_reminder_distinguishes_unknown_kinds(self) -> None: + text = _full_reminder("/tmp/plan.md", False) + assert "Repo-discoverable facts" in text + assert "never ask the user" in text + assert "AskUserQuestion" in text + assert "recommended default" in text + + def test_full_reminder_records_defaults_as_assumptions(self) -> None: + assert "Assumptions" in _full_reminder("/tmp/plan.md", False) + + def test_full_reminder_requires_decision_complete_exit(self) -> None: + text = _full_reminder("/tmp/plan.md", False) + assert "decision-complete" in text + assert "Figure out" in text + + def test_full_reminder_includes_plan_shape_rubric(self) -> None: + text = _full_reminder("/tmp/plan.md", False) + assert "skimmable" in text + assert "3-5 short sections" in text + + def test_sparse_reminder_mentions_decision_complete_and_assumptions(self) -> None: + text = _sparse_reminder("/tmp/plan.md") + assert "decision-complete" in text + assert "Assumptions" in text + + def test_reentry_reminder_requires_decision_complete_exit(self) -> None: + assert "decision-complete" in _reentry_reminder("/tmp/plan.md") From f5b9b06a43718da183d322c77a3d25ddd07a732d Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:29:41 -0400 Subject: [PATCH 11/49] feat(print): strict stdout/stderr channel discipline on failure paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Headless failure diagnostics (provider errors, max-steps + handoff, interrupt, unknown errors) printed plain text to stdout, corrupting the stream-json channel for machine parsers. All diagnostics now route to the pre-redirect stderr fd (falling back to sys.stderr), and stream-json mode additionally emits one structured error record — a Notification with category=run, type=error, the failure class, and the exit code — written via raw stdout so rich cannot soft-wrap the JSON line. Plan item: protocol-headless/channel-discipline (Tier 1). --- src/pythinker_code/ui/print/__init__.py | 58 ++++++-- .../test_print_channel_discipline.py | 126 ++++++++++++++++++ 2 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 tests/ui_and_conv/test_print_channel_discipline.py diff --git a/src/pythinker_code/ui/print/__init__.py b/src/pythinker_code/ui/print/__init__.py index 71d40be5..c6939bf2 100644 --- a/src/pythinker_code/ui/print/__init__.py +++ b/src/pythinker_code/ui/print/__init__.py @@ -6,6 +6,7 @@ import time from functools import partial from pathlib import Path +from uuid import uuid4 from pythinker_core.chat_provider import ( APIConnectionError, @@ -31,6 +32,7 @@ from pythinker_code.ui.print.visualize import visualize from pythinker_code.utils.logging import logger, open_original_stderr from pythinker_code.utils.signals import install_sigint_handler +from pythinker_code.wire.types import Notification class Print: @@ -409,19 +411,20 @@ def _handler(): command = None except LLMNotSet as e: logger.warning("LLM not set — user has no provider configured") - print(str(e)) + self._emit_failure(str(e), error_type="LLMNotSet", exit_code=ExitCode.FAILURE) return ExitCode.FAILURE except LLMNotSupported as e: logger.exception("LLM not supported:") - print(str(e)) + self._emit_failure(str(e), error_type="LLMNotSupported", exit_code=ExitCode.FAILURE) return ExitCode.FAILURE except ChatProviderError as e: logger.exception("LLM provider error:") - print(str(e)) - return self._classify_provider_error(e) + exit_code = self._classify_provider_error(e) + self._emit_failure(str(e), error_type=type(e).__name__, exit_code=exit_code) + return exit_code except MaxStepsReached as e: logger.warning("Max steps reached: {n_steps}", n_steps=e.n_steps) - print(str(e)) + diagnostic = str(e) # Graceful handoff: a tools-disabled summary of progress / next steps # for the human who resumes (best-effort; falls back to the line above). if isinstance(self.soul, PythinkerSoul): @@ -433,20 +436,59 @@ def _handler(): logger.warning("Max-steps handoff failed", exc_info=True) handoff = None if handoff: - print(f"\n── handoff ──\n{handoff}") + diagnostic += f"\n\n── handoff ──\n{handoff}" + self._emit_failure(diagnostic, error_type="MaxStepsReached", exit_code=ExitCode.FAILURE) return ExitCode.FAILURE except RunCancelled: logger.error("Interrupted by user") - print("Interrupted by user") + self._emit_failure( + "Interrupted by user", error_type="RunCancelled", exit_code=ExitCode.FAILURE + ) return ExitCode.FAILURE except BaseException as e: logger.exception("Unknown error:") - print(f"Unknown error: {e}") + self._emit_failure( + f"Unknown error: {e}", error_type=type(e).__name__, exit_code=ExitCode.FAILURE + ) raise finally: remove_sigint() return ExitCode.FAILURE + def _emit_failure(self, diagnostic: str, *, error_type: str, exit_code: int) -> None: + """Route a failure diagnostic to stderr, keeping stdout a data channel. + + ``sys.stderr`` may already be redirected to the logger pipe at this + point in the CLI lifecycle, so write through the pre-redirect fd when + one exists. In stream-json mode every stdout line must parse as JSON, + so the human text is followed by one structured error record on + stdout for machine parsers. + """ + notice = diagnostic if diagnostic.endswith("\n") else diagnostic + "\n" + with open_original_stderr() as stream: + if stream is not None: + stream.write(notice.encode("utf-8", errors="replace")) + stream.flush() + else: + sys.stderr.write(notice) + if self.output_format == "stream-json": + record = Notification( + id=str(uuid4()), + category="run", + type="error", + source_kind="print", + source_id="print", + title=error_type, + body=diagnostic, + severity="error", + created_at=time.time(), + payload={"exit_code": int(exit_code)}, + ) + # Builtin stdout write, NOT the module's rich print: rich soft-wraps + # long lines, which would tear the JSON record across lines. + sys.stdout.write(record.model_dump_json(exclude_none=True) + "\n") + sys.stdout.flush() + _RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} @staticmethod diff --git a/tests/ui_and_conv/test_print_channel_discipline.py b/tests/ui_and_conv/test_print_channel_discipline.py new file mode 100644 index 00000000..7ae7419f --- /dev/null +++ b/tests/ui_and_conv/test_print_channel_discipline.py @@ -0,0 +1,126 @@ +"""Headless channel discipline: stdout is a data channel, stderr is for humans. + +In stream-json mode every stdout line must parse as JSON — plain-text +failure diagnostics corrupt the stream for machine parsers. Diagnostics +route to stderr in every mode, and stream-json additionally emits one +structured error record (a Notification with category="run", +type="error") so parsers observe the terminal failure and its exit code. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +from pythinker_core.chat_provider import APIConnectionError, APIStatusError, ChatProviderError + +from pythinker_code.cli import ExitCode, OutputFormat +from pythinker_code.soul import LLMNotSet, MaxStepsReached, RunCancelled +from pythinker_code.ui.print import Print + + +def _make_print(tmp_path: Path, output_format: OutputFormat) -> Print: + from unittest.mock import AsyncMock + + soul = AsyncMock() + soul.runtime = None + return Print( + soul=soul, + input_format="text", + output_format=output_format, + context_file=tmp_path / "context.json", + ) + + +def _run_failing(p: Print, monkeypatch: pytest.MonkeyPatch, exception: BaseException) -> int: + async def _raise(*args: object, **kwargs: object) -> object: + raise exception + + monkeypatch.setattr("pythinker_code.ui.print.run_soul", _raise) + return asyncio.run(p.run(command="do something")) + + +def _parsed_stdout_lines(out: str) -> list[dict]: + lines = [line for line in out.splitlines() if line.strip()] + return [json.loads(line) for line in lines] + + +def _error_events(records: list[dict]) -> list[dict]: + return [r for r in records if r.get("category") == "run" and r.get("type") == "error"] + + +FAILURES = [ + pytest.param(LLMNotSet(), id="llm-not-set"), + pytest.param(ChatProviderError("provider exploded"), id="provider-error"), + pytest.param(RunCancelled(), id="cancelled"), + pytest.param(MaxStepsReached(10), id="max-steps"), +] + + +class TestStreamJsonChannelDiscipline: + @pytest.mark.parametrize("exception", FAILURES) + def test_every_stdout_line_parses_as_json( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys, exception + ) -> None: + p = _make_print(tmp_path, "stream-json") + + _run_failing(p, monkeypatch, exception) + + records = _parsed_stdout_lines(capsys.readouterr().out) + assert _error_events(records), "expected a structured error record on stdout" + + @pytest.mark.parametrize("exception", FAILURES) + def test_diagnostic_text_reaches_stderr( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys, exception + ) -> None: + p = _make_print(tmp_path, "stream-json") + + _run_failing(p, monkeypatch, exception) + + assert capsys.readouterr().err.strip() + + def test_error_event_carries_exit_code( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys + ) -> None: + p = _make_print(tmp_path, "stream-json") + + code = _run_failing(p, monkeypatch, APIStatusError(401, "no auth")) + + events = _error_events(_parsed_stdout_lines(capsys.readouterr().out)) + assert events[0]["payload"]["exit_code"] == int(code) == int(ExitCode.FAILURE) + + def test_retryable_error_event_exit_code( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys + ) -> None: + p = _make_print(tmp_path, "stream-json") + + code = _run_failing(p, monkeypatch, APIConnectionError("connection refused")) + + events = _error_events(_parsed_stdout_lines(capsys.readouterr().out)) + assert events[0]["payload"]["exit_code"] == int(code) == int(ExitCode.RETRYABLE) + + def test_unknown_error_emits_event_then_reraises( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys + ) -> None: + p = _make_print(tmp_path, "stream-json") + + with pytest.raises(RuntimeError): + _run_failing(p, monkeypatch, RuntimeError("boom")) + + records = _parsed_stdout_lines(capsys.readouterr().out) + assert _error_events(records) + + +class TestTextModeChannelDiscipline: + def test_diagnostic_goes_to_stderr_not_stdout( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys + ) -> None: + p = _make_print(tmp_path, "text") + + _run_failing(p, monkeypatch, LLMNotSet()) + + captured = capsys.readouterr() + assert str(LLMNotSet()) not in captured.out + assert captured.err.strip() From c8d82d384e31006b1d33ac361b78b8fc748f96cc Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:33:01 -0400 Subject: [PATCH 12/49] feat(subagents): inject merge-base-scoped git context into review agents Review-class subagents (review, code_reviewer, security_reviewer) received scope purely via the parent's prompt text and burned their first turns rediscovering branch, dirty files, and the merge base. The git-context prefix now also resolves the merge base against the first existing base ref (origin/main, main, master), names the exact review scope (git diff <sha>...HEAD), omits it when HEAD is the base, and is injected for reviewer-class agents alongside explore. Plan item: review-mode/deterministic-review-target-resolution (Tier 1, agent-dispatch slice). --- src/pythinker_code/subagents/core.py | 12 ++++- src/pythinker_code/subagents/git_context.py | 31 +++++++++++-- tests/subagents/test_git_context_gate.py | 18 ++++++++ tests/test_git_context.py | 49 +++++++++++++++++++++ 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 tests/subagents/test_git_context_gate.py diff --git a/src/pythinker_code/subagents/core.py b/src/pythinker_code/subagents/core.py index dfab4fa9..e7f5fc98 100644 --- a/src/pythinker_code/subagents/core.py +++ b/src/pythinker_code/subagents/core.py @@ -18,6 +18,14 @@ from pythinker_code.subagents.models import AgentLaunchSpec, AgentTypeDefinition from pythinker_code.subagents.store import SubagentStore +GIT_CONTEXT_AGENT_TYPES = frozenset({"explore", "review", "code_reviewer", "security_reviewer"}) +"""Read-oriented agent types whose first prompt gets a git-context prefix. + +Exploration and review both orient on repo state (branch, dirty files, +merge base); injecting it up front saves the turns each run would spend +rediscovering its scope. Write-capable types derive state themselves as +part of their task.""" + SUBAGENT_OUTPUT_LANGUAGE_INSTRUCTION = """\ <output-language> Write natural-language output in the same language as the original user request or task @@ -85,9 +93,9 @@ async def prepare_soul( if on_stage: on_stage("context_ready") - # 4. For new (non-resumed) explore agents, prepend git context to the prompt + # 4. For new (non-resumed) read-oriented agents, prepend git context to the prompt prompt = spec.prompt - if spec.type_def.name == "explore" and not spec.resumed: + if spec.type_def.name in GIT_CONTEXT_AGENT_TYPES and not spec.resumed: from pythinker_code.subagents.git_context import collect_git_context git_ctx = await collect_git_context(runtime.builtin_args.PYTHINKER_WORK_DIR) diff --git a/src/pythinker_code/subagents/git_context.py b/src/pythinker_code/subagents/git_context.py index af4765e7..e640bf38 100644 --- a/src/pythinker_code/subagents/git_context.py +++ b/src/pythinker_code/subagents/git_context.py @@ -1,4 +1,4 @@ -"""Collect git repository context for explore subagents.""" +"""Collect git repository context for read-oriented subagents.""" from __future__ import annotations @@ -13,10 +13,11 @@ _TIMEOUT = 5.0 _MAX_DIRTY_FILES = 20 +_BASE_REF_CANDIDATES = ("origin/main", "main", "master") async def collect_git_context(work_dir: HostPath) -> str: - """Collect git context information for the explore agent. + """Collect git context information for exploration and review agents. Returns a formatted ``<git-context>`` block, or an empty string if the directory is not a git repository or all git commands fail. Every git @@ -30,11 +31,12 @@ async def collect_git_context(work_dir: HostPath) -> str: return "" # Run all git commands in parallel for speed - remote_url, branch, dirty_raw, log_raw = await asyncio.gather( + remote_url, branch, dirty_raw, log_raw, head_sha = await asyncio.gather( _run_git(["remote", "get-url", "origin"], cwd), _run_git(["branch", "--show-current"], cwd), _run_git(["status", "--porcelain"], cwd), _run_git(["log", "-3", "--format=%h %s"], cwd), + _run_git(["rev-parse", "HEAD"], cwd), ) sections: list[str] = [] @@ -53,6 +55,12 @@ async def collect_git_context(work_dir: HostPath) -> str: if branch: sections.append(f"Branch: {branch}") + # Merge base — names the diff scope so review-style agents can run + # `git diff <sha>...HEAD` without rediscovering the base ref. + merge_base_line = await _merge_base_section(cwd, head_sha) + if merge_base_line: + sections.append(merge_base_line) + # Dirty files if dirty_raw is not None: dirty_lines = [line for line in dirty_raw.splitlines() if line.strip()] @@ -80,6 +88,23 @@ async def collect_git_context(work_dir: HostPath) -> str: return f"<git-context>\n{content}\n</git-context>" +async def _merge_base_section(cwd: str, head_sha: str | None) -> str | None: + """Resolve the merge base against the first base ref that exists. + + Returns ``None`` when no candidate resolves or when HEAD *is* the base + (reviewing on the base branch itself leaves nothing to scope). + """ + for base_ref in _BASE_REF_CANDIDATES: + merge_base = await _run_git(["merge-base", "HEAD", base_ref], cwd) + if not merge_base: + continue + if head_sha and merge_base == head_sha: + return None + short = merge_base[:12] + return f"Merge base vs {base_ref}: {short} (review scope: git diff {short}...HEAD)" + return None + + async def _run_git(args: list[str], cwd: str, timeout: float = _TIMEOUT) -> str | None: """Run one git command via pythinker_host.exec. diff --git a/tests/subagents/test_git_context_gate.py b/tests/subagents/test_git_context_gate.py new file mode 100644 index 00000000..0772c8d2 --- /dev/null +++ b/tests/subagents/test_git_context_gate.py @@ -0,0 +1,18 @@ +"""Which subagent types get the git-context prompt prefix. + +Reviewer-class agents need branch/dirty/merge-base orientation as much as +explore does — without it every review run burns turns rediscovering its +diff scope from a bare prompt. +""" + +from __future__ import annotations + +from pythinker_code.subagents.core import GIT_CONTEXT_AGENT_TYPES + + +def test_explore_and_reviewer_types_receive_git_context() -> None: + assert {"explore", "review", "code_reviewer", "security_reviewer"} <= GIT_CONTEXT_AGENT_TYPES + + +def test_write_capable_types_do_not() -> None: + assert {"coder", "implementer", "agent"}.isdisjoint(GIT_CONTEXT_AGENT_TYPES) diff --git a/tests/test_git_context.py b/tests/test_git_context.py index bcd400ce..f3d682eb 100644 --- a/tests/test_git_context.py +++ b/tests/test_git_context.py @@ -336,3 +336,52 @@ async def test_dirty_files_capped(self, tmp_path: Path) -> None: result = await collect_git_context(_host_path(tmp_path)) assert "Dirty files (25):" in result assert "... and 5 more" in result + + +async def _git(cwd: Path, *args: str) -> str: + proc = await asyncio.create_subprocess_exec( + "git", + *args, + cwd=cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout.decode().strip() + + +async def _init_repo(tmp_path: Path) -> None: + await _git(tmp_path, "init", "-b", "main") + await _git(tmp_path, "config", "user.email", "test@test.com") + await _git(tmp_path, "config", "user.name", "Test") + (tmp_path / "base.txt").write_text("base") + await _git(tmp_path, "add", ".") + await _git(tmp_path, "commit", "-m", "base commit") + + +class TestMergeBaseContext: + """Review scoping: the context names the merge base so a reviewer agent + can run `git diff <sha>...HEAD` without rediscovering the base ref.""" + + @pytest.mark.asyncio + async def test_merge_base_shown_on_feature_branch(self, tmp_path: Path) -> None: + await _init_repo(tmp_path) + base_sha = await _git(tmp_path, "rev-parse", "HEAD") + await _git(tmp_path, "checkout", "-b", "feature") + (tmp_path / "feat.txt").write_text("feature") + await _git(tmp_path, "add", ".") + await _git(tmp_path, "commit", "-m", "feature commit") + + result = await collect_git_context(_host_path(tmp_path)) + + assert "Merge base vs main:" in result + assert base_sha[:12] in result + assert "git diff" in result + + @pytest.mark.asyncio + async def test_merge_base_omitted_on_base_branch(self, tmp_path: Path) -> None: + await _init_repo(tmp_path) + + result = await collect_git_context(_host_path(tmp_path)) + + assert "Merge base" not in result From e2e74b70a69e22ee3f23fd394c113b9739f44b4d Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:39:39 -0400 Subject: [PATCH 13/49] feat(soul): same-step concurrency policy for parallel tool calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provider-emitted parallel tool calls all executed concurrently — two mutating tools (WriteFile + Shell from one assistant message) could race with no ordering guarantee. Tool dispatch now runs through a reader-writer gate: tools declaring supports_parallel (read-only builtins: ReadFile, ReadMediaFile, Glob, Grep, SmartSearch, Think, Recall, ListMcpResources, ReadMcpResource, SearchWeb, FetchURL) overlap freely, while everything else — including unflagged plugin/MCP tools, the safe default — executes exclusively in dispatch order. Writers drain in-flight readers and cannot be starved. Plan item: tools-registry-codemode/concurrency-policy (Tier 1). --- src/pythinker_code/soul/toolset.py | 53 ++++++- src/pythinker_code/tools/file/glob.py | 1 + src/pythinker_code/tools/file/grep_local.py | 2 + src/pythinker_code/tools/file/read.py | 1 + src/pythinker_code/tools/file/read_media.py | 1 + .../tools/mcp_resource/__init__.py | 2 + src/pythinker_code/tools/recall/__init__.py | 1 + src/pythinker_code/tools/think/__init__.py | 1 + src/pythinker_code/tools/web/fetch.py | 1 + src/pythinker_code/tools/web/search.py | 1 + tests/core/test_toolset_concurrency.py | 135 ++++++++++++++++++ 11 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 tests/core/test_toolset_concurrency.py diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index 1f99d13f..eff7a8b8 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -9,7 +9,7 @@ import json import re import time -from collections.abc import Awaitable, Callable, Iterable +from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable from contextvars import ContextVar from dataclasses import dataclass from datetime import timedelta @@ -300,6 +300,41 @@ def _append_reminder_to_return_value(return_value: Any, reminder_text: str) -> A return return_value.model_copy(update={"output": new_output}) +class _ReadWriteGate: + """Async reader-writer gate for same-step parallel tool calls. + + Parallel-safe tools (readers) overlap freely; a mutating tool (writer) + waits for in-flight readers to drain and excludes everything while it + runs. Writers hold the lock while draining, which also blocks new + readers behind a queued writer — dispatch order stays deterministic + and writers cannot starve. + """ + + def __init__(self) -> None: + self._writer_lock = asyncio.Lock() + self._active_readers = 0 + self._readers_drained = asyncio.Event() + self._readers_drained.set() + + @contextlib.asynccontextmanager + async def shared(self) -> AsyncGenerator[None]: + async with self._writer_lock: + self._active_readers += 1 + self._readers_drained.clear() + try: + yield + finally: + self._active_readers -= 1 + if self._active_readers == 0: + self._readers_drained.set() + + @contextlib.asynccontextmanager + async def exclusive(self) -> AsyncGenerator[None]: + async with self._writer_lock: + await self._readers_drained.wait() + yield + + class PythinkerToolset: def __init__(self, runtime: Runtime | None = None) -> None: self._runtime = runtime @@ -309,6 +344,7 @@ def __init__(self, runtime: Runtime | None = None) -> None: self._mcp_loading_task: asyncio.Task[None] | None = None self._deferred_mcp_load: tuple[list[MCPConfig], Runtime] | None = None self._hook_engine: HookEngine = HookEngine() + self._concurrency_gate = _ReadWriteGate() # Deduplication state self._previous_step_calls: list[ToolCallKey] = [] @@ -450,6 +486,19 @@ def _is_tool_visible(self, tool: ToolType) -> bool: return True + async def _gated_call(self, tool: ToolType, arguments: JsonType) -> ToolReturnValue: + """Execute under the same-step concurrency policy. + + Tools declaring ``supports_parallel`` share the gate; everything + else (including unflagged plugin/MCP tools — the safe default) + runs exclusively so same-step mutations stay ordered. + """ + if getattr(tool, "supports_parallel", False): + async with self._concurrency_gate.shared(): + return await tool.call(arguments) + async with self._concurrency_gate.exclusive(): + return await tool.call(arguments) + def begin_step( self, previous_calls: list[ToolCallKey], @@ -652,7 +701,7 @@ async def _call_with_lifecycle(): ) _tool_span = _tool_span_cm.__enter__() try: - ret = await tool.call(arguments) + ret = await self._gated_call(tool, arguments) except Exception as e: tool_elapsed = time.monotonic() - t0 _tool_span.set_attribute("tool.success", False) diff --git a/src/pythinker_code/tools/file/glob.py b/src/pythinker_code/tools/file/glob.py index c72040eb..b70130c4 100644 --- a/src/pythinker_code/tools/file/glob.py +++ b/src/pythinker_code/tools/file/glob.py @@ -31,6 +31,7 @@ class Params(BaseModel): class Glob(CallableTool2[Params]): name: str = "Glob" + supports_parallel = True description: str = load_desc( Path(__file__).parent / "glob.md", { diff --git a/src/pythinker_code/tools/file/grep_local.py b/src/pythinker_code/tools/file/grep_local.py index 24d9a707..edf546ba 100644 --- a/src/pythinker_code/tools/file/grep_local.py +++ b/src/pythinker_code/tools/file/grep_local.py @@ -719,6 +719,7 @@ def _smart_search_patterns(query: str) -> list[tuple[str, str]]: class SmartSearch(CallableTool2[SmartSearchParams]): name: str = "SmartSearch" + supports_parallel = True description: str = ( "Plan and run a small set of bounded local grep passes for a symbol or concept. " "Returns cited file/line spans and truncation guidance; use for exploration before " @@ -788,6 +789,7 @@ async def __call__(self, params: SmartSearchParams) -> ToolReturnValue: class Grep(CallableTool2[Params]): name: str = "Grep" + supports_parallel = True description: str = load_desc(Path(__file__).parent / "grep.md") params: type[Params] = Params diff --git a/src/pythinker_code/tools/file/read.py b/src/pythinker_code/tools/file/read.py index 2527e1a8..1264cbd8 100644 --- a/src/pythinker_code/tools/file/read.py +++ b/src/pythinker_code/tools/file/read.py @@ -63,6 +63,7 @@ def _validate_line_offset(self) -> "Params": class ReadFile(CallableTool2[Params]): name: str = "ReadFile" + supports_parallel = True params: type[Params] = Params def __init__(self, runtime: Runtime) -> None: diff --git a/src/pythinker_code/tools/file/read_media.py b/src/pythinker_code/tools/file/read_media.py index 68ceac50..fa30340f 100644 --- a/src/pythinker_code/tools/file/read_media.py +++ b/src/pythinker_code/tools/file/read_media.py @@ -49,6 +49,7 @@ class Params(BaseModel): class ReadMediaFile(CallableTool2[Params]): name: str = "ReadMediaFile" + supports_parallel = True params: type[Params] = Params def __init__(self, runtime: Runtime) -> None: diff --git a/src/pythinker_code/tools/mcp_resource/__init__.py b/src/pythinker_code/tools/mcp_resource/__init__.py index 26035ebb..dbdda56d 100644 --- a/src/pythinker_code/tools/mcp_resource/__init__.py +++ b/src/pythinker_code/tools/mcp_resource/__init__.py @@ -26,6 +26,7 @@ class ListParams(BaseModel): class ListMcpResources(CallableTool2[ListParams]): name: str = "ListMcpResources" + supports_parallel = True params: type[ListParams] = ListParams def __init__(self, toolset: PythinkerToolset) -> None: @@ -74,6 +75,7 @@ class ReadParams(BaseModel): class ReadMcpResource(CallableTool2[ReadParams]): name: str = "ReadMcpResource" + supports_parallel = True params: type[ReadParams] = ReadParams def __init__(self, toolset: PythinkerToolset) -> None: diff --git a/src/pythinker_code/tools/recall/__init__.py b/src/pythinker_code/tools/recall/__init__.py index 4e5257c8..2f8db506 100644 --- a/src/pythinker_code/tools/recall/__init__.py +++ b/src/pythinker_code/tools/recall/__init__.py @@ -111,6 +111,7 @@ def _render_transcript(context_file: Path, budget: int) -> str: class Recall(CallableTool2[Params]): name: str = NAME + supports_parallel = True params: type[Params] = Params def __init__(self, runtime: Runtime): diff --git a/src/pythinker_code/tools/think/__init__.py b/src/pythinker_code/tools/think/__init__.py index 7672ab74..d2ddeb23 100644 --- a/src/pythinker_code/tools/think/__init__.py +++ b/src/pythinker_code/tools/think/__init__.py @@ -13,6 +13,7 @@ class Params(BaseModel): class Think(CallableTool2[Params]): name: str = "Think" + supports_parallel = True description: str = load_desc(Path(__file__).parent / "think.md", {}) params: type[Params] = Params diff --git a/src/pythinker_code/tools/web/fetch.py b/src/pythinker_code/tools/web/fetch.py index 9292e6fc..eda83848 100644 --- a/src/pythinker_code/tools/web/fetch.py +++ b/src/pythinker_code/tools/web/fetch.py @@ -130,6 +130,7 @@ class Params(BaseModel): class FetchURL(CallableTool2[Params]): name: str = "FetchURL" + supports_parallel = True description: str = load_desc(Path(__file__).parent / "fetch.md", {}) params: type[Params] = Params diff --git a/src/pythinker_code/tools/web/search.py b/src/pythinker_code/tools/web/search.py index 0e681b88..9c0fe8dd 100644 --- a/src/pythinker_code/tools/web/search.py +++ b/src/pythinker_code/tools/web/search.py @@ -43,6 +43,7 @@ class Params(BaseModel): class SearchWeb(CallableTool2[Params]): name: str = "SearchWeb" + supports_parallel = True description: str = load_desc(Path(__file__).parent / "search.md", {}) params: type[Params] = Params diff --git a/tests/core/test_toolset_concurrency.py b/tests/core/test_toolset_concurrency.py new file mode 100644 index 00000000..ecc3cbee --- /dev/null +++ b/tests/core/test_toolset_concurrency.py @@ -0,0 +1,135 @@ +"""Same-step tool-call concurrency policy. + +When a provider emits parallel tool calls, mutating tools (WriteFile + +Shell from one assistant message) used to execute concurrently with no +race protection. Policy: parallel-safe tools overlap freely; everything +else (including unflagged plugin/MCP tools — safe default) runs +exclusively, keeping same-step mutation ordering deterministic. +""" + +from __future__ import annotations + +import asyncio + +from pythinker_core.tooling import ToolReturnValue + +from pythinker_code.hooks.engine import HookEngine +from pythinker_code.soul.toolset import PythinkerToolset +from pythinker_code.wire.types import ToolCall + + +class _RecordingTool: + base = None + + def __init__( + self, name: str, events: list[tuple[str, str]], *, parallel: bool, delay: float = 0.05 + ) -> None: + self.name = name + self._events = events + self._delay = delay + if parallel: + self.supports_parallel = True + + async def call(self, arguments: object) -> ToolReturnValue: + self._events.append(("enter", self.name)) + await asyncio.sleep(self._delay) + self._events.append(("exit", self.name)) + return ToolReturnValue(is_error=False, output="ok", message="ok", display=[]) + + +def _toolset(*tools: _RecordingTool) -> PythinkerToolset: + toolset = PythinkerToolset() + toolset._hook_engine = HookEngine([], cwd="/tmp") + for tool in tools: + toolset._tool_dict[tool.name] = tool # type: ignore[assignment] + return toolset + + +async def _dispatch(toolset: PythinkerToolset, *names: str) -> None: + tasks = [] + for index, name in enumerate(names): + result = toolset.handle( + ToolCall(id=f"tc_{index}", function=ToolCall.FunctionBody(name=name, arguments="{}")) + ) + assert isinstance(result, asyncio.Task) + tasks.append(result) + await asyncio.gather(*tasks) + + +class TestSameStepConcurrencyPolicy: + async def test_mutating_tools_serialize_in_dispatch_order(self) -> None: + events: list[tuple[str, str]] = [] + toolset = _toolset( + _RecordingTool("WriteA", events, parallel=False), + _RecordingTool("WriteB", events, parallel=False), + ) + + await _dispatch(toolset, "WriteA", "WriteB") + + assert events == [ + ("enter", "WriteA"), + ("exit", "WriteA"), + ("enter", "WriteB"), + ("exit", "WriteB"), + ] + + async def test_parallel_safe_tools_overlap(self) -> None: + events: list[tuple[str, str]] = [] + toolset = _toolset( + _RecordingTool("ReadA", events, parallel=True), + _RecordingTool("ReadB", events, parallel=True), + ) + + await _dispatch(toolset, "ReadA", "ReadB") + + assert {events[0][0], events[1][0]} == {"enter"}, events + + async def test_reader_waits_for_earlier_writer(self) -> None: + events: list[tuple[str, str]] = [] + toolset = _toolset( + _RecordingTool("Write", events, parallel=False), + _RecordingTool("Read", events, parallel=True), + ) + + await _dispatch(toolset, "Write", "Read") + + assert events.index(("exit", "Write")) < events.index(("enter", "Read")) + + async def test_writer_waits_for_inflight_readers(self) -> None: + events: list[tuple[str, str]] = [] + toolset = _toolset( + _RecordingTool("Read", events, parallel=True), + _RecordingTool("Write", events, parallel=False), + ) + + await _dispatch(toolset, "Read", "Write") + + assert events.index(("exit", "Read")) < events.index(("enter", "Write")) + + +class TestParallelSafeFlags: + def test_read_only_builtins_are_parallel_safe(self) -> None: + from pythinker_code.tools.file.glob import Glob + from pythinker_code.tools.file.grep_local import Grep, SmartSearch + from pythinker_code.tools.file.read import ReadFile + from pythinker_code.tools.mcp_resource import ListMcpResources, ReadMcpResource + from pythinker_code.tools.think import Think + + for tool_cls in ( + Glob, + Grep, + SmartSearch, + ReadFile, + ListMcpResources, + ReadMcpResource, + Think, + ): + assert getattr(tool_cls, "supports_parallel", False), tool_cls.__name__ + + def test_mutating_builtins_stay_exclusive(self) -> None: + from pythinker_code.tools.file.replace import StrReplaceFile + from pythinker_code.tools.file.write import WriteFile + from pythinker_code.tools.shell import Shell + + for tool_cls in (WriteFile, StrReplaceFile, Shell): + assert not getattr(tool_cls, "supports_parallel", False), tool_cls.__name__ From 1615cfbd5407a5e3bcc82359e49f5206a23eb8a3 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:46:32 -0400 Subject: [PATCH 14/49] feat(soul): reactive context-overflow recovery (compact-and-retry) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A provider context-length 400 was telemetry-classified but treated as fatal: the step raised and the turn died, even though the proactive prune/compact thresholds run on heuristic counts that can undercount. Two recovery layers, both bounded: - Agent loop: on a context_overflow-classified step error, prune (best-effort), force a full compaction, and retry the step — once per turn; telemetry records recovered vs failed. - SimpleCompaction: the compaction request itself carries the whole to-compact slice and can overflow too; on a context-length rejection it drops the oldest half and retries, terminally falling back to the preserved tail plus an explicit dropped-context note. classify_api_error moves to soul/api_errors.py (re-exported from pythinkersoul) so compaction can classify without a circular import. Plan items: core-loop/reactive-overflow-recovery + context-mgmt/context-overflow-recovery (Tier 1). --- src/pythinker_code/soul/api_errors.py | 61 ++++++++++++++ src/pythinker_code/soul/compaction.py | 72 +++++++++++++--- src/pythinker_code/soul/pythinkersoul.py | 84 ++++++++++--------- tests/core/test_compaction_overflow.py | 100 +++++++++++++++++++++++ tests/core/test_overflow_recovery.py | 68 +++++++++++++++ 5 files changed, 335 insertions(+), 50 deletions(-) create mode 100644 src/pythinker_code/soul/api_errors.py create mode 100644 tests/core/test_compaction_overflow.py create mode 100644 tests/core/test_overflow_recovery.py diff --git a/src/pythinker_code/soul/api_errors.py b/src/pythinker_code/soul/api_errors.py new file mode 100644 index 00000000..5ffb0eb1 --- /dev/null +++ b/src/pythinker_code/soul/api_errors.py @@ -0,0 +1,61 @@ +"""Provider API error classification shared by the soul loop and compaction. + +Lives in its own module so compaction can classify context-overflow +rejections without importing the soul (which imports compaction). +""" + +from __future__ import annotations + +from pythinker_core.chat_provider import ( + APIConnectionError, + APIEmptyResponseError, + APIStatusError, + APITimeoutError, +) + +_CONTEXT_OVERFLOW_MARKERS = ( + "context length", + "context_length", + "max tokens", + "maximum context", + "too many tokens", +) + + +def classify_api_error(e: Exception) -> tuple[str, int | None]: + """Classify an LLM API exception into (error_type, status_code). + + Exposed at module level so telemetry tests can import the real function + instead of duplicating the classification table. + + Returns: + (error_type, status_code) where status_code is None for non-HTTP errors. + """ + status_code: int | None = None + if isinstance(e, APIStatusError): + status = getattr(e, "status_code", getattr(e, "status", 0)) + status_code = int(status) if status else None + if status == 429: + return "rate_limit", status_code + if status in (401, 403): + return "auth", status_code + if status >= 500: + return "5xx_server", status_code + if 400 <= status < 500: + msg_lower = str(e).lower() + if any(marker in msg_lower for marker in _CONTEXT_OVERFLOW_MARKERS): + return "context_overflow", status_code + return "4xx_client", status_code + return "api", status_code + if isinstance(e, APIConnectionError): + return "network", None + if isinstance(e, (APITimeoutError, TimeoutError)): + return "timeout", None + if isinstance(e, APIEmptyResponseError): + return "empty_response", None + return "other", None + + +def is_context_overflow_error(e: Exception) -> bool: + """Whether *e* is a provider rejection for exceeding the context window.""" + return classify_api_error(e)[0] == "context_overflow" diff --git a/src/pythinker_code/soul/compaction.py b/src/pythinker_code/soul/compaction.py index b5a386e8..db4f8284 100644 --- a/src/pythinker_code/soul/compaction.py +++ b/src/pythinker_code/soul/compaction.py @@ -10,6 +10,7 @@ import pythinker_code.prompts as prompts from pythinker_code.llm import LLM +from pythinker_code.soul.api_errors import is_context_overflow_error from pythinker_code.soul.message import system from pythinker_code.utils.logging import logger from pythinker_code.wire.types import ContentPart, TextPart, ThinkPart @@ -152,9 +153,10 @@ def __init__(self, max_preserved_messages: int = 2, base_prompt: str | None = No async def compact( self, messages: Sequence[Message], llm: LLM, *, custom_instruction: str = "" ) -> CompactionResult: - compact_message, to_preserve = self.prepare(messages, custom_instruction=custom_instruction) + prepared = self.prepare(messages, custom_instruction=custom_instruction) + compact_message, to_preserve = prepared.compact_message, prepared.to_preserve if compact_message is None: - return CompactionResult(messages=to_preserve, usage=None) + return CompactionResult(messages=list(to_preserve), usage=None) # Call pythinker_core.step to get the compacted context. # NOTE: the summary length is bounded by the chat provider's construction-time @@ -163,12 +165,50 @@ async def compact( # ``ChatProvider.generate`` (and ``pythinker_core.step``), which neither exposes # today; adding one ripples across every provider backend, so it is out of scope here. logger.debug("Compacting context...") - result = await pythinker_core.step( - chat_provider=llm.chat_provider, - system_prompt="You are a helpful assistant that compacts conversation context.", - toolset=EmptyToolset(), - history=[compact_message], - ) + to_compact = list(prepared.to_compact) + while True: + try: + result = await pythinker_core.step( + chat_provider=llm.chat_provider, + system_prompt="You are a helpful assistant that compacts conversation context.", + toolset=EmptyToolset(), + history=[compact_message], + ) + break + except Exception as e: + # The compaction request itself can exceed the context window + # (it carries the whole to-compact slice). Drop the oldest half + # and retry; when nothing summarizable fits, preserve only the + # tail with an explicit dropped-context note instead of failing. + if not is_context_overflow_error(e): + raise + if len(to_compact) <= 1: + logger.warning( + "Compaction request still exceeds the context window with a " + "single message; dropping unsummarized older context" + ) + note = Message( + role="user", + content=[ + system( + "Previous context exceeded the model's window and was " + "dropped without summarization. Re-read files or re-run " + "commands if earlier results are needed." + ) + ], + ) + return CompactionResult(messages=[note, *to_preserve], usage=None) + dropped = len(to_compact) // 2 + to_compact = to_compact[dropped:] + logger.warning( + "Compaction request exceeded the context window; retrying with the " + "newest {kept} of the slice ({dropped} oldest dropped)", + kept=len(to_compact), + dropped=dropped, + ) + compact_message = self._build_compact_message( + to_compact, custom_instruction=custom_instruction + ) if result.usage: logger.debug( "Compaction used {input} input tokens and {output} output tokens", @@ -190,6 +230,7 @@ async def compact( class PrepareResult(NamedTuple): compact_message: Message | None to_preserve: Sequence[Message] + to_compact: Sequence[Message] = () def prepare( self, messages: Sequence[Message], *, custom_instruction: str = "" @@ -217,7 +258,18 @@ def prepare( # Let's hope this won't exceed the context size limit return self.PrepareResult(compact_message=None, to_preserve=to_preserve) - # Create input message for compaction + compact_message = self._build_compact_message( + to_compact, custom_instruction=custom_instruction + ) + return self.PrepareResult( + compact_message=compact_message, + to_preserve=to_preserve, + to_compact=to_compact, + ) + + def _build_compact_message( + self, to_compact: Sequence[Message], *, custom_instruction: str = "" + ) -> Message: compact_message = Message(role="user", content=[]) for i, msg in enumerate(to_compact): compact_message.content.append( @@ -235,4 +287,4 @@ def prepare( f"{custom_instruction}" ) compact_message.content.append(TextPart(text=prompt_text)) - return self.PrepareResult(compact_message=compact_message, to_preserve=to_preserve) + return compact_message diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index edebb9f8..908d1a15 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -52,6 +52,10 @@ wire_send, ) from pythinker_code.soul.agent import Agent, Runtime + +# classify_api_error is re-exported so telemetry tests and existing imports +# keep resolving against this module. +from pythinker_code.soul.api_errors import classify_api_error as classify_api_error from pythinker_code.soul.approval import deliberation_scope from pythinker_code.soul.compaction import ( CompactionResult, @@ -163,46 +167,6 @@ def classify_llm_system(chat_provider: object | None) -> str: return "unknown" -def classify_api_error(e: Exception) -> tuple[str, int | None]: - """Classify an LLM API exception into (error_type, status_code). - - Exposed at module level so telemetry tests can import the real function - instead of duplicating the classification table. - - Returns: - (error_type, status_code) where status_code is None for non-HTTP errors. - """ - status_code: int | None = None - if isinstance(e, APIStatusError): - status = getattr(e, "status_code", getattr(e, "status", 0)) - status_code = int(status) if status else None - if status == 429: - return "rate_limit", status_code - if status in (401, 403): - return "auth", status_code - if status >= 500: - return "5xx_server", status_code - if 400 <= status < 500: - msg_lower = str(e).lower() - if ( - "context length" in msg_lower - or "context_length" in msg_lower - or "max tokens" in msg_lower - or "maximum context" in msg_lower - or "too many tokens" in msg_lower - ): - return "context_overflow", status_code - return "4xx_client", status_code - return "api", status_code - if isinstance(e, APIConnectionError): - return "network", None - if isinstance(e, (APITimeoutError, TimeoutError)): - return "timeout", None - if isinstance(e, APIEmptyResponseError): - return "empty_response", None - return "other", None - - def _is_hard_usage_limit(exception: BaseException) -> bool: """Whether a 429 is a subscription usage cap (resets in hours) rather than a transient RPM/TPM burst (clears in seconds). @@ -1419,6 +1383,9 @@ async def _agent_loop(self) -> TurnOutcome: self._intent_nudge_used = False # Reset the degenerate-loop failure tracker at the start of each turn. self._consecutive_failures = 0 + # One-shot per turn: reactive compact-and-retry after a provider + # context-length rejection (proactive thresholds can undercount). + overflow_recovery_used = False while True: step_no += 1 if step_no > self._loop_control.max_steps_per_turn: @@ -1507,6 +1474,10 @@ async def _agent_loop(self) -> TurnOutcome: if status_code is not None: api_error_props["status_code"] = status_code track("api_error", **api_error_props) + if error_type == "context_overflow" and not overflow_recovery_used: + overflow_recovery_used = True + if await self._recover_from_context_overflow(step_no): + continue # --- StopFailure hook --- from pythinker_code.hooks import events as _hook_events @@ -1988,6 +1959,39 @@ async def _grow_context(self, result: StepResult, tool_results: list[ToolResult] await self._context.append_message(tool_messages) # token count of tool results are not available yet + async def _recover_from_context_overflow(self, step_no: int) -> bool: + """Reactive shrink-and-retry after a provider context-length rejection. + + The proactive prune/compact thresholds run on heuristic token counts + and can undercount (e.g. large pending tool output), so the provider + may still reject a step. Prune (best-effort), force a full + compaction, and let the loop retry the step once. Returns False when + compaction itself fails — the original error then propagates. + """ + from pythinker_code.telemetry import track + + logger.warning( + "Provider rejected step {step_no} for context length; compacting and retrying once", + step_no=step_no, + ) + try: + with contextlib.suppress(Exception): + await self.prune_context() + await self.compact_context() + except Exception as compact_err: + from pythinker_code.telemetry.errors import report_handled_error + + report_handled_error(compact_err, site="soul.context.overflow_recovery") + logger.error( + "Context-overflow recovery compaction failed: {error_type}: {error}", + error_type=type(compact_err).__name__, + error=compact_err, + ) + track("context_overflow_recovery", outcome="failed") + return False + track("context_overflow_recovery", outcome="recovered") + return True + async def prune_context(self) -> bool: """Cheap, fidelity-preserving compaction tier: replace large stale tool-result bodies in deep history with placeholders, then rewrite the diff --git a/tests/core/test_compaction_overflow.py b/tests/core/test_compaction_overflow.py new file mode 100644 index 00000000..e2c9bdad --- /dev/null +++ b/tests/core/test_compaction_overflow.py @@ -0,0 +1,100 @@ +"""SimpleCompaction shrink-on-overflow fallback. + +Compaction sends the whole to-compact slice to the provider, so a turn +that already overflowed the context window can overflow the compaction +request too. On a context-length rejection the compactor drops the +oldest half of the slice and retries; when nothing summarizable fits it +falls back to preserving only the tail with an explicit dropped-context +note instead of failing the turn. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import cast + +import pytest +import pythinker_core +from pythinker_core.chat_provider import APIStatusError +from pythinker_core.message import Message + +from pythinker_code.llm import LLM +from pythinker_code.soul.compaction import SimpleCompaction +from pythinker_code.wire.types import TextPart + + +def _history(n_pairs: int = 4) -> list[Message]: + messages: list[Message] = [] + for index in range(n_pairs): + messages.append(Message(role="user", content=[TextPart(text=f"request {index}")])) + messages.append(Message(role="assistant", content=[TextPart(text=f"reply {index}")])) + return messages + + +def _fake_llm() -> LLM: + return cast(LLM, SimpleNamespace(chat_provider=None)) + + +def _overflow_error() -> APIStatusError: + return APIStatusError(400, "This model's maximum context length is exceeded") + + +def _summary_result() -> SimpleNamespace: + return SimpleNamespace( + message=Message(role="assistant", content=[TextPart(text="the summary")]), + usage=None, + ) + + +class _FakeStep: + def __init__(self, failures_before_success: int) -> None: + self.failures_before_success = failures_before_success + self.histories: list[list[Message]] = [] + + async def __call__(self, *, chat_provider, system_prompt, toolset, history): + self.histories.append(list(history)) + if len(self.histories) <= self.failures_before_success: + raise _overflow_error() + return _summary_result() + + +def _section_count(message: Message) -> int: + return message.extract_text(" ").count("## Message") + + +@pytest.mark.asyncio +async def test_overflow_retries_with_smaller_slice(monkeypatch) -> None: + fake_step = _FakeStep(failures_before_success=1) + monkeypatch.setattr(pythinker_core, "step", fake_step) + + result = await SimpleCompaction(max_preserved_messages=2).compact(_history(), llm=_fake_llm()) + + assert len(fake_step.histories) == 2 + first, second = (h[0] for h in fake_step.histories) + assert _section_count(second) < _section_count(first) + assert "the summary" in result.messages[0].extract_text(" ") + + +@pytest.mark.asyncio +async def test_exhausted_retries_fall_back_to_tail_with_note(monkeypatch) -> None: + fake_step = _FakeStep(failures_before_success=99) + monkeypatch.setattr(pythinker_core, "step", fake_step) + + history = _history() + result = await SimpleCompaction(max_preserved_messages=2).compact(history, llm=_fake_llm()) + + joined = " ".join(m.extract_text(" ") for m in result.messages) + assert "dropped" in joined.lower() + # The preserved tail survives. + assert "reply 3" in joined + + +@pytest.mark.asyncio +async def test_non_overflow_error_propagates(monkeypatch) -> None: + async def _step_raises(**kwargs): + raise APIStatusError(400, "invalid request: bad tool schema") + + monkeypatch.setattr(pythinker_core, "step", _step_raises) + + with pytest.raises(APIStatusError): + await SimpleCompaction(max_preserved_messages=2).compact(_history(), llm=_fake_llm()) diff --git a/tests/core/test_overflow_recovery.py b/tests/core/test_overflow_recovery.py new file mode 100644 index 00000000..2d470e6f --- /dev/null +++ b/tests/core/test_overflow_recovery.py @@ -0,0 +1,68 @@ +"""Reactive context-overflow recovery in the agent loop. + +The proactive prune/compact thresholds run on heuristic token counts and +can undercount (e.g. large pending tool output), so the provider may +still reject a step with a context-length 400. The loop recovers once +per turn — prune, force-compact, retry the step — instead of killing +the turn. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest +from pythinker_core.tooling.simple import SimpleToolset + +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul + + +def _make_soul(runtime: Runtime, tmp_path) -> PythinkerSoul: + agent = Agent(name="Overflow", system_prompt="sys", toolset=SimpleToolset(), runtime=runtime) + context = Context(file_backend=tmp_path / "history.jsonl") + return PythinkerSoul(agent, context=context) + + +class TestClassifierLocation: + def test_classifier_importable_from_api_errors_and_soul(self) -> None: + from pythinker_code.soul.api_errors import classify_api_error as from_module + from pythinker_code.soul.pythinkersoul import classify_api_error as from_soul + + assert from_module is from_soul + + +class TestRecoverFromContextOverflow: + @pytest.mark.asyncio + async def test_prunes_compacts_and_reports_recovered(self, runtime, tmp_path) -> None: + soul = _make_soul(runtime, tmp_path) + soul.prune_context = AsyncMock(return_value=True) # type: ignore[method-assign] + soul.compact_context = AsyncMock() # type: ignore[method-assign] + + recovered = await soul._recover_from_context_overflow(step_no=3) + + assert recovered is True + soul.prune_context.assert_awaited_once() + soul.compact_context.assert_awaited_once() + + @pytest.mark.asyncio + async def test_prune_failure_does_not_block_compaction(self, runtime, tmp_path) -> None: + soul = _make_soul(runtime, tmp_path) + soul.prune_context = AsyncMock(side_effect=RuntimeError("prune broke")) # type: ignore[method-assign] + soul.compact_context = AsyncMock() # type: ignore[method-assign] + + recovered = await soul._recover_from_context_overflow(step_no=3) + + assert recovered is True + soul.compact_context.assert_awaited_once() + + @pytest.mark.asyncio + async def test_compaction_failure_reports_not_recovered(self, runtime, tmp_path) -> None: + soul = _make_soul(runtime, tmp_path) + soul.prune_context = AsyncMock(return_value=False) # type: ignore[method-assign] + soul.compact_context = AsyncMock(side_effect=RuntimeError("compact broke")) # type: ignore[method-assign] + + recovered = await soul._recover_from_context_overflow(step_no=3) + + assert recovered is False From dfa21ff41137a8492490f6c322216c3c93489420 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:47:10 -0400 Subject: [PATCH 15/49] docs(tasks): record Tier-1 adoption progress and next M-item queue --- tasks/todo.md | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tasks/todo.md b/tasks/todo.md index 7539c922..53757b4c 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -2,14 +2,26 @@ ## Active -- [ ] Agent-harness adoption arc (`feat/agent-harness-enhancements`): port the - reference harness's remaining coding-agent design into pythinker, - generically framed. Gap map DONE: 124 verified items (3 refuted) ranked in - `tasks/agent-harness-adoption-plan.md` (tiers 1-4, execution discipline - inside). Done so far: checkpoint 0 `047a0b29` (orchestration injection - provider + name-scrub), checkpoint 1 `b40cdb71` (ACP hides - AskUserQuestion). Now executing Tier 1 checkpoints: TDD per item, - clean-code-guard per checkpoint, make check + pytest green per commit. +- [ ] Agent-harness adoption arc (`feat/agent-harness-enhancements`): executing + `tasks/agent-harness-adoption-plan.md` (124 verified items, tiers 1-4). + DONE: all 5 Tier-1 high/S + first high/M — `047a0b29` orchestration + provider+scrub, `b40cdb71` ACP question-tool hide, `36cedafd` plan, + `e722278c` restore-time history invariant repair, `388da2d3` + decision-complete plan mode, `f5b9b06a` print channel discipline, + `c8d82d38` review git-context+merge-base, `e2e74b70` parallel-tool + concurrency policy, `1615cfbd` reactive overflow recovery (loop + + SimpleCompaction halving; classify_api_error → soul/api_errors.py). + NEXT (Tier-1 high/M, plan order): per-project trust gating of + config/hooks; unknown-config-key diagnostics; model-switch context + continuity; known-safe command auto-approval; MCP startup + timeout+diagnostics; MCP per-server tool filtering; subagent context + fork; workspace isolation for parallel writers; turn rollup analytics; + feedback diagnostics; fuzzy edit ladder; permissions-state + instructions; model escalation w/ justification; JSONL lifecycle + stream; schema-constrained final output; /review command; hook trust + gating; PostToolUse feedback to model; deferred tool loading; foreign + schema sanitization. Discipline: TDD + clean-code-guard + make check + per checkpoint; single writer now. - [ ] Windows shell hardening (researched, not yet implemented): bash-first shell policy (Git Bash probe → pwsh → powershell, never cmd), Windows tool-description guidance (`;` not `&&` on PS 5.1, `$env:`, quoting), From 0f39a3b1c6f356bdef0412b3de1d786c970479ab Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:55:36 -0400 Subject: [PATCH 16/49] feat(config): gate project-scope hooks behind durable per-project trust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cloned repository's .pythinker/config.toml merged unconditionally, so its [[hooks]] shell commands auto-executed at session start — arbitrary code execution from cloning a repo. Project/local-scope hooks now load only after the user records trust: - New project_trust store (user-scope trusted_projects.json, atomic writes, fail-closed on corruption) keyed by the resolved repo root, so the repo itself can never grant its own trust. - _load_scoped strips hooks from untrusted project/local scopes with a warning naming /trust as the fix; broken TOML in an untrusted project degrades to an empty scope instead of blocking startup (trusted projects keep the loud error). - /trust on|off persists the per-project decision alongside the session flags and points at /reload for hook activation. - find_project_root promoted to public API (the /trust path needs it). Plan item: config-features/per-project-trust-gating (Tier 1). Out of scope (own plan item): sanitize-and-warn for scope-locked keys in untrusted scopes — they keep the existing loud ConfigError. --- CHANGELOG.md | 1 + src/pythinker_code/config.py | 34 ++++++- src/pythinker_code/project_trust.py | 81 ++++++++++++++++ src/pythinker_code/ui/shell/slash.py | 23 +++++ tests/core/test_config.py | 19 ++-- tests/core/test_project_trust.py | 136 +++++++++++++++++++++++++++ 6 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 src/pythinker_code/project_trust.py create mode 100644 tests/core/test_project_trust.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 462cd47d..f5edce99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Project-scope hooks now require trusting the project.** A cloned repository's `.pythinker/config.toml` could previously auto-execute its shell hooks at session start. Hooks from project and local scopes now load only after `/trust` records a durable per-project decision (stored user-side in `trusted_projects.json`, keyed by the resolved repo root); until then they are stripped with a warning naming the fix. Broken TOML in an untrusted project no longer blocks startup — the scope is treated as empty with a warning, while trusted projects keep the loud error. - **Agent orchestration guidance is sharper for substantial tasks.** The default prompt now sharpens work-shaping guidance, and a new root-only runtime reminder nudges substantial normal-mode tasks toward the lightest effective path — direct tools, `SetTodoList`, foreground `RunAgents`, or verification — while backing off for plan mode, `/goal`, auto mode, and subagents. - **The in-app updater recovers from Homebrew's untrusted-tap refusal.** Homebrew 5.0 (`HOMEBREW_REQUIRE_TAP_TRUST`) refuses to load formulas from third-party taps until `brew trust <tap>` is run once, which made the in-app `brew upgrade` fail with only a generic "run manually" hint. The updater now detects both the hard `Refusing to load … from untrusted tap` refusal and the soft `Skipping … not trusted` warning, offers to run `brew trust pythoughts-labs/pythinker` and retry the upgrade once on an interactive terminal, and otherwise prints the exact remediation. It also catches the silent no-op where an untrusted tap is skipped during `brew update` and `brew upgrade` exits 0 without advancing the version. - **Session exports redact secrets surfaced by tool output.** A tool result (e.g. `grep`/`cat` over a `.env`) could write a secret value into an exported transcript in plaintext. `/export` now redacts the value of secret-named keys (`password`, `token`, `api_key`, `secret`, …) to `[REDACTED]` in both the markdown and YAML formats, while leaving non-secret keys such as `token_count`, usernames, and ports intact. diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index e74b7769..73b6ae6a 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -31,7 +31,7 @@ from pythinker_code.utils.logging import logger -def _find_project_root(cwd: Path) -> Path | None: +def find_project_root(cwd: Path) -> Path | None: """Walk up from cwd to find the nearest directory containing .git/. Returns None when no .git marker is found before reaching the filesystem @@ -261,10 +261,36 @@ def _read_toml(path: Path) -> dict[str, Any]: local_dict: dict[str, Any] = {} if project_root is not None: + from pythinker_code.project_trust import is_project_trusted + + project_trusted = is_project_trusted(project_root) project_file = project_root / ".pythinker" / "config.toml" local_file = project_root / ".pythinker" / "config.local.toml" - project_dict = _read_toml(project_file) - local_dict = _read_toml(local_file) + if project_trusted: + project_dict = _read_toml(project_file) + local_dict = _read_toml(local_file) + else: + # An untrusted (e.g. freshly cloned) project must not brick startup + # with broken TOML, and must not auto-execute anything: hooks are + # shell commands run on lifecycle events, so they load only after + # the user records trust (/trust). + try: + project_dict = _read_toml(project_file) + local_dict = _read_toml(local_file) + except ConfigError as exc: + logger.warning( + "Ignoring unreadable project config in untrusted project: {error}", + error=exc, + ) + project_dict = {} + local_dict = {} + for scope_dict, scope_file in ((project_dict, project_file), (local_dict, local_file)): + if scope_dict.pop("hooks", None) is not None: + logger.warning( + "Project hooks in {file} are disabled until the project is " + "trusted; run /trust to enable them", + file=scope_file, + ) # ── GUARD ───────────────────────────────────────────────────────────── if project_file is not None: @@ -1039,7 +1065,7 @@ def load_config(config_file: Path | None = None) -> Config: behaviour used by tests and the CLI --config flag. """ if config_file is None: - project_root = _find_project_root(Path.cwd()) + project_root = find_project_root(Path.cwd()) return _load_scoped(project_root) # ── Explicit path: legacy single-file load (unchanged) ──────────────── diff --git a/src/pythinker_code/project_trust.py b/src/pythinker_code/project_trust.py new file mode 100644 index 00000000..c48c0ae6 --- /dev/null +++ b/src/pythinker_code/project_trust.py @@ -0,0 +1,81 @@ +"""Persistent per-project trust decisions. + +A cloned repository's project-scope config (``.pythinker/config.toml``) +carries auto-executed surfaces — shell hooks above all — so those load +only after the user trusts the project root. The decision persists +across sessions in a user-scope file keyed by the normalized root path; +it is never stored inside the project, where the repo could edit it. +""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from typing import cast + +from pythinker_code.share import get_share_dir +from pythinker_code.utils.logging import logger + +_TRUST_FILE_NAME = "trusted_projects.json" + + +def _trust_file() -> Path: + return get_share_dir() / _TRUST_FILE_NAME + + +def _normalize(project_root: Path) -> str: + return str(project_root.expanduser().resolve(strict=False)) + + +def _read_trusted_roots() -> set[str]: + path = _trust_file() + if not path.exists(): + return set() + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.warning( + "Unreadable project trust file {path}; treating all projects as untrusted: {error}", + path=path, + error=exc, + ) + return set() + if not isinstance(data, dict): + return set() + roots: object = cast("dict[str, object]", data).get("trusted_roots") + if not isinstance(roots, list): + return set() + return {root for root in cast("list[object]", roots) if isinstance(root, str)} + + +def _write_trusted_roots(roots: set[str]) -> None: + path = _trust_file() + path.parent.mkdir(parents=True, exist_ok=True) + payload = json.dumps({"trusted_roots": sorted(roots)}, indent=2) + "\n" + # Atomic replace so a crash mid-write cannot corrupt the trust store. + fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=path.name, suffix=".tmp") + tmp_path = Path(tmp_name) + try: + with open(fd, "w", encoding="utf-8") as tmp_file: + tmp_file.write(payload) + tmp_path.replace(path) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + +def is_project_trusted(project_root: Path) -> bool: + """Whether the user has durably trusted *project_root*.""" + return _normalize(project_root) in _read_trusted_roots() + + +def set_project_trusted(project_root: Path, trusted: bool) -> None: + """Durably record (or revoke) trust for *project_root*.""" + roots = _read_trusted_roots() + normalized = _normalize(project_root) + if trusted: + roots.add(normalized) + else: + roots.discard(normalized) + _write_trusted_roots(roots) diff --git a/src/pythinker_code/ui/shell/slash.py b/src/pythinker_code/ui/shell/slash.py index 17783965..f3a10028 100644 --- a/src/pythinker_code/ui/shell/slash.py +++ b/src/pythinker_code/ui/shell/slash.py @@ -3,6 +3,7 @@ import asyncio import re from collections.abc import Awaitable, Callable +from pathlib import Path from typing import TYPE_CHECKING, Any, NoReturn, cast from prompt_toolkit.shortcuts.choice_input import ChoiceInput @@ -1734,6 +1735,11 @@ def trust(app: Shell, args: str) -> None: console.print( f"[{_t_trust.success}]Workspace trusted. Safe mode disabled for this session.[/]" ) + if _persist_project_trust(soul, trusted=True): + console.print( + f"[{_t_trust.info}]Project config trust recorded — project hooks load " + "on /reload or next start.[/]" + ) return if mode in {"off", "no", "untrust", "safe"}: state.trusted = False @@ -1743,6 +1749,7 @@ def trust(app: Shell, args: str) -> None: soul.runtime.approval.set_auto(False) soul.runtime.session.state.approval.auto_approve_actions.clear() soul.runtime.session.save_state() + _persist_project_trust(soul, trusted=False) console.print( f"[{_t_trust.warning}]Workspace untrusted. Safe mode enabled; " "auto-approval is disabled.[/]" @@ -1757,6 +1764,22 @@ def trust(app: Shell, args: str) -> None: console.print(f"Workspace trust: [bold]{status}[/bold] safe mode: [bold]{safe}[/bold]") +def _persist_project_trust(soul: PythinkerSoul, *, trusted: bool) -> bool: + """Record the durable per-project trust decision for the session's repo. + + Returns True when a project root was found and recorded. Sessions outside + a git project have no project config scope to gate, so nothing persists. + """ + from pythinker_code.config import find_project_root + from pythinker_code.project_trust import set_project_trusted + + root = find_project_root(Path(str(soul.runtime.session.work_dir))) + if root is None: + return False + set_project_trusted(root, trusted) + return True + + @registry.command @shell_mode_registry.command async def worklog(app: Shell, args: str) -> None: diff --git a/tests/core/test_config.py b/tests/core/test_config.py index cae6ebff..419ff589 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -10,11 +10,11 @@ Config, _apply_env_vars, _check_scope_locks, - _find_project_root, _load_scoped, _lookup_provenance, _set_nested, _type_based_merge, + find_project_root, get_default_config, load_config, load_config_from_string, @@ -319,22 +319,22 @@ def test_auto_deliberate_is_a_valid_policy() -> None: assert c.ask_user_question_policy == "auto_deliberate" -def test_find_project_root_finds_git_root(tmp_path): +def testfind_project_root_finds_git_root(tmp_path): git_dir = tmp_path / ".git" git_dir.mkdir() subdir = tmp_path / "src" / "pkg" subdir.mkdir(parents=True) - assert _find_project_root(subdir) == tmp_path + assert find_project_root(subdir) == tmp_path -def test_find_project_root_returns_none_outside_git(tmp_path): +def testfind_project_root_returns_none_outside_git(tmp_path): # tmp_path itself has no .git ancestor in practice - assert _find_project_root(tmp_path) is None + assert find_project_root(tmp_path) is None -def test_find_project_root_finds_root_in_cwd(tmp_path): +def testfind_project_root_finds_root_in_cwd(tmp_path): (tmp_path / ".git").mkdir() - assert _find_project_root(tmp_path) == tmp_path + assert find_project_root(tmp_path) == tmp_path def test_set_nested_flat(): @@ -616,6 +616,8 @@ def test_load_scoped_local_overrides_project(tmp_path, monkeypatch): def test_load_scoped_hooks_concatenate(tmp_path, monkeypatch): + from pythinker_code.project_trust import set_project_trusted + monkeypatch.setenv("PYTHINKER_SHARE_DIR", str(tmp_path)) _write_toml(tmp_path / "config.toml", {"hooks": [{"event": "Stop", "command": "user-hook"}]}) project_root = tmp_path / "myproject" @@ -623,6 +625,9 @@ def test_load_scoped_hooks_concatenate(tmp_path, monkeypatch): project_root / ".pythinker" / "config.toml", {"hooks": [{"event": "Stop", "command": "project-hook"}]}, ) + # Project hooks auto-execute, so they only merge once the project is + # trusted (see test_project_trust.py for the untrusted paths). + set_project_trusted(project_root, True) config = _load_scoped(project_root=project_root) commands = [h.command for h in config.hooks] assert "user-hook" in commands diff --git a/tests/core/test_project_trust.py b/tests/core/test_project_trust.py new file mode 100644 index 00000000..6a291ff5 --- /dev/null +++ b/tests/core/test_project_trust.py @@ -0,0 +1,136 @@ +"""Per-project trust gating of auto-executed project config. + +A cloned repository's .pythinker/config.toml can define shell hooks that +run automatically at session start. Project-scope hooks must therefore +load only after the user trusts the project root; the decision persists +across sessions in a user-scope trust file. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pythinker_code.project_trust import is_project_trusted, set_project_trusted + + +@pytest.fixture(autouse=True) +def _isolated_share_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHINKER_SHARE_DIR", str(tmp_path / "share")) + + +def _project_with_hooks(tmp_path: Path, body: str | None = None) -> Path: + root = tmp_path / "repo" + (root / ".git").mkdir(parents=True) + config_dir = root / ".pythinker" + config_dir.mkdir() + (config_dir / "config.toml").write_text( + body + if body is not None + else '[[hooks]]\nevent = "SessionStart"\ncommand = "touch /tmp/pwned"\n', + encoding="utf-8", + ) + return root + + +class TestTrustStore: + def test_unknown_project_is_untrusted(self, tmp_path: Path) -> None: + assert is_project_trusted(tmp_path / "nowhere") is False + + def test_set_and_revoke_roundtrip(self, tmp_path: Path) -> None: + root = tmp_path / "repo" + root.mkdir() + + set_project_trusted(root, True) + assert is_project_trusted(root) is True + + set_project_trusted(root, False) + assert is_project_trusted(root) is False + + def test_paths_are_normalized(self, tmp_path: Path) -> None: + root = tmp_path / "repo" + (root / "sub").mkdir(parents=True) + + set_project_trusted(root / "sub" / "..", True) + + assert is_project_trusted(root) is True + + def test_corrupt_trust_file_is_tolerated(self, tmp_path: Path) -> None: + root = tmp_path / "repo" + root.mkdir() + set_project_trusted(root, True) + trust_files = list((tmp_path / "share").glob("trusted_projects.json")) + assert trust_files + trust_files[0].write_text("{not json", encoding="utf-8") + + assert is_project_trusted(root) is False + + +class TestUntrustedProjectConfigGating: + def test_untrusted_project_hooks_are_stripped(self, tmp_path: Path) -> None: + from pythinker_code.config import _load_scoped + + root = _project_with_hooks(tmp_path) + + config = _load_scoped(root) + + assert config.hooks == [] + + def test_trusted_project_hooks_load(self, tmp_path: Path) -> None: + from pythinker_code.config import _load_scoped + + root = _project_with_hooks(tmp_path) + set_project_trusted(root, True) + + config = _load_scoped(root) + + assert len(config.hooks) == 1 + assert config.hooks[0].event == "SessionStart" + + def test_user_scope_hooks_unaffected_by_project_trust( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from pythinker_code.config import _load_scoped, get_config_file + + root = _project_with_hooks(tmp_path, body="") + user_file = get_config_file() + user_file.parent.mkdir(parents=True, exist_ok=True) + user_file.write_text( + '[[hooks]]\nevent = "SessionStart"\ncommand = "echo mine"\n', encoding="utf-8" + ) + + config = _load_scoped(root) + + assert len(config.hooks) == 1 + assert config.hooks[0].command == "echo mine" + + def test_local_scope_hooks_also_gated(self, tmp_path: Path) -> None: + from pythinker_code.config import _load_scoped + + root = _project_with_hooks(tmp_path, body="") + (root / ".pythinker" / "config.local.toml").write_text( + '[[hooks]]\nevent = "SessionStart"\ncommand = "echo local"\n', encoding="utf-8" + ) + + config = _load_scoped(root) + + assert config.hooks == [] + + def test_invalid_toml_in_untrusted_project_is_empty_scope(self, tmp_path: Path) -> None: + from pythinker_code.config import _load_scoped + + root = _project_with_hooks(tmp_path, body="this = [is not toml") + + config = _load_scoped(root) # must not raise + + assert config.hooks == [] + + def test_invalid_toml_in_trusted_project_still_raises(self, tmp_path: Path) -> None: + from pythinker_code.config import ConfigError, _load_scoped + + root = _project_with_hooks(tmp_path, body="this = [is not toml") + set_project_trusted(root, True) + + with pytest.raises(ConfigError): + _load_scoped(root) From 9d1178f4be6bc6098c2e6f522999c423139ca899 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:59:03 -0400 Subject: [PATCH 17/49] feat(config): warn on unknown config keys with source-located diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Config models ignore extra keys, so a typo'd key silently vanished and changed behavior with no signal. After merge, the raw dict is now diffed against the model field tree (aliases and AliasChoices honored; recursion follows provable shapes only — nested models, dict-of-model maps, lists of models — so unmodellable values can never false- positive). Each finding warns with the dotted path and the scope file it came from via the existing provenance map; PYTHINKER_STRICT_CONFIG=1 escalates to ConfigError for CI use. Plan item: config-features/unknown-config-key-detection (Tier 1). --- CHANGELOG.md | 1 + src/pythinker_code/config.py | 110 ++++++++++++++++++++++++- tests/core/test_config_unknown_keys.py | 97 ++++++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 tests/core/test_config_unknown_keys.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f5edce99..2dab1f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Typo'd config keys are no longer silently ignored.** Loading now diffs the merged config against the model schema and warns with the dotted path and originating scope file for every unrecognized key (`defaut_yolo = true` names itself and its file instead of silently changing nothing); `PYTHINKER_STRICT_CONFIG=1` escalates the findings to a startup error for CI. - **Project-scope hooks now require trusting the project.** A cloned repository's `.pythinker/config.toml` could previously auto-execute its shell hooks at session start. Hooks from project and local scopes now load only after `/trust` records a durable per-project decision (stored user-side in `trusted_projects.json`, keyed by the resolved repo root); until then they are stripped with a warning naming the fix. Broken TOML in an untrusted project no longer blocks startup — the scope is treated as empty with a warning, while trusted projects keep the loud error. - **Agent orchestration guidance is sharper for substantial tasks.** The default prompt now sharpens work-shaping guidance, and a new root-only runtime reminder nudges substantial normal-mode tasks toward the lightest effective path — direct tools, `SetTodoList`, foreground `RunAgents`, or verification — while backing off for plan mode, `/goal`, auto mode, and subagents. - **The in-app updater recovers from Homebrew's untrusted-tap refusal.** Homebrew 5.0 (`HOMEBREW_REQUIRE_TAP_TRUST`) refuses to load formulas from third-party taps until `brew trust <tap>` is run once, which made the in-app `brew upgrade` fail with only a generic "run manually" hint. The updater now detects both the hard `Refusing to load … from untrusted tap` refusal and the soft `Skipping … not trusted` warning, offers to run `brew trust pythoughts-labs/pythinker` and retry the upgrade once on an interactive terminal, and otherwise prints the exact remediation. It also catches the silent no-op where an untrusted tap is skipped during `brew update` and `brew upgrade` exits 0 without advancing the version. diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index 73b6ae6a..6c776968 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -4,7 +4,8 @@ import json import os from pathlib import Path -from typing import Any, Literal, Self, cast +from types import UnionType +from typing import Any, Literal, Self, Union, cast, get_args, get_origin import tomlkit from pydantic import ( @@ -95,6 +96,110 @@ def find_project_root(cwd: Path) -> Path | None: # --------------------------------------------------------------------------- +def unknown_config_key_paths( + model_cls: type[BaseModel], data: dict[str, Any], _prefix: tuple[str, ...] = () +) -> list[tuple[str, ...]]: + """Dotted paths in *data* that no model field will consume. + + Pydantic's extra='ignore' makes a typo'd key silently vanish and change + behavior with no signal. Conservative by design: recursion only follows + shapes it can prove (nested models, dict-of-model maps, lists of models), + so an unmodellable value is never reported — a false positive here would + teach users to ignore the diagnostics. + """ + if model_cls.model_config.get("extra") == "allow": + return [] + known: dict[str, Any] = {} + for name, field in model_cls.model_fields.items(): + known[name] = field + if field.alias: + known[field.alias] = field + if isinstance(field.validation_alias, str): + known[field.validation_alias] = field + elif isinstance(field.validation_alias, AliasChoices): + for choice in field.validation_alias.choices: + if isinstance(choice, str): + known[choice] = field + + unknown: list[tuple[str, ...]] = [] + for key, value in data.items(): + path = (*_prefix, key) + field = known.get(key) + if field is None: + unknown.append(path) + continue + unknown.extend(_nested_unknown_paths(field.annotation, value, path)) + return unknown + + +def _nested_unknown_paths( + annotation: Any, value: Any, path: tuple[str, ...] +) -> list[tuple[str, ...]]: + annotation = _sole_non_none_type(annotation) + origin = get_origin(annotation) + if origin is dict: + args = get_args(annotation) + if len(args) == 2 and isinstance(value, dict): + item_type = _sole_non_none_type(args[1]) + if isinstance(item_type, type) and issubclass(item_type, BaseModel): + nested: list[tuple[str, ...]] = [] + for map_key, item in cast(dict[str, Any], value).items(): + if isinstance(item, dict): + nested.extend( + unknown_config_key_paths( + item_type, cast(dict[str, Any], item), (*path, str(map_key)) + ) + ) + return nested + return [] + if origin in (list, tuple, set): + args = get_args(annotation) + if args and isinstance(value, list): + item_type = _sole_non_none_type(args[0]) + if isinstance(item_type, type) and issubclass(item_type, BaseModel): + nested = [] + for item in cast(list[Any], value): + if isinstance(item, dict): + # Item paths omit the index: the field name plus the + # offending key is what a user greps their TOML for. + nested.extend( + unknown_config_key_paths(item_type, cast(dict[str, Any], item), path) + ) + return nested + return [] + if ( + isinstance(annotation, type) + and issubclass(annotation, BaseModel) + and isinstance(value, dict) + ): + return unknown_config_key_paths(annotation, cast(dict[str, Any], value), path) + return [] + + +def _sole_non_none_type(annotation: Any) -> Any: + """Unwrap ``X | None`` to ``X``; ambiguous unions return unchanged.""" + if get_origin(annotation) in (Union, UnionType): + non_none = [arg for arg in get_args(annotation) if arg is not type(None)] + if len(non_none) == 1: + return non_none[0] + return annotation + + +def _report_unknown_config_keys(merged: dict[str, Any], provenance: dict[str, Any]) -> None: + """Warn (or raise under PYTHINKER_STRICT_CONFIG) for unconsumed keys.""" + unknown_paths = unknown_config_key_paths(Config, merged) + if not unknown_paths: + return + findings = [ + f"{'.'.join(path)} (from {_lookup_provenance(provenance, path)})" + for path in sorted(unknown_paths) + ] + if os.environ.get("PYTHINKER_STRICT_CONFIG"): + raise ConfigError("Unknown configuration keys:\n " + "\n ".join(findings)) + for finding in findings: + logger.warning("Unknown config key ignored: {finding}", finding=finding) + + def _set_nested(d: dict[str, Any], path: tuple[str, ...], value: object) -> None: """Walk *path* into *d*, creating intermediate dicts, then set the leaf.""" node = d @@ -309,6 +414,9 @@ def _read_toml(path: Path) -> dict[str, Any]: # ── ENV OVERLAY ─────────────────────────────────────────────────────── _apply_env_vars(merged, provenance) + # ── DIAGNOSE ────────────────────────────────────────────────────────── + _report_unknown_config_keys(merged, provenance) + # ── VALIDATE ────────────────────────────────────────────────────────── try: config = Config.model_validate(merged) diff --git a/tests/core/test_config_unknown_keys.py b/tests/core/test_config_unknown_keys.py new file mode 100644 index 00000000..3d1c14d8 --- /dev/null +++ b/tests/core/test_config_unknown_keys.py @@ -0,0 +1,97 @@ +"""Unknown-config-key detection with source-located diagnostics. + +Config models ignore extra keys, so a typo'd key ('defaut_yolo') silently +vanishes and changes behavior with no signal. Loading now diffs the raw +merged dict against the model field tree and warns with the dotted path +and originating scope file; PYTHINKER_STRICT_CONFIG escalates to an error. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pythinker_code.config import Config, ConfigError, _load_scoped, unknown_config_key_paths + + +def _write(path: Path, body: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + +@pytest.fixture(autouse=True) +def _isolated_share_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHINKER_SHARE_DIR", str(tmp_path / "share")) + monkeypatch.delenv("PYTHINKER_STRICT_CONFIG", raising=False) + + +class TestUnknownKeyPaths: + def test_top_level_typo_detected(self) -> None: + unknown = unknown_config_key_paths(Config, {"defaut_yolo": True}) + + assert ("defaut_yolo",) in unknown + + def test_nested_typo_detected(self) -> None: + unknown = unknown_config_key_paths(Config, {"tui": {"statuslin": {}}}) + + assert ("tui", "statuslin") in unknown + + def test_valid_keys_produce_no_findings(self) -> None: + data = { + "default_model": "m", + "tui": {"statusline": {"enabled": True}}, + "loop_control": {"max_steps_per_run": 5}, # validation alias + } + + assert unknown_config_key_paths(Config, data) == [] + + def test_map_fields_allow_arbitrary_keys_but_check_values(self) -> None: + data = { + "providers": { + "mine": {"type": "openai", "base_url": "x", "api_key": "k", "tpyo": 1} + } + } + + unknown = unknown_config_key_paths(Config, data) + + assert ("providers", "mine", "tpyo") in unknown + assert all(path[:2] != ("providers", "mine") or len(path) == 3 for path in unknown) + + def test_list_of_models_checks_items(self) -> None: + data = {"hooks": [{"event": "Stop", "command": "x", "matchr": ".*"}]} + + unknown = unknown_config_key_paths(Config, data) + + assert ("hooks", "matchr") in unknown + + +class TestLoadTimeDiagnostics: + def test_unknown_key_warned_with_scope(self, tmp_path: Path, monkeypatch) -> None: + from unittest.mock import patch + + _write(tmp_path / "share" / "config.toml", "defaut_yolo = true\n") + + with patch("pythinker_code.config.logger") as mock_logger: + _load_scoped(None) + + joined = " ".join(str(call) for call in mock_logger.warning.call_args_list) + assert "defaut_yolo" in joined + assert "config.toml" in joined + + def test_strict_mode_escalates_to_error(self, tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("PYTHINKER_STRICT_CONFIG", "1") + _write(tmp_path / "share" / "config.toml", "defaut_yolo = true\n") + + with pytest.raises(ConfigError, match="defaut_yolo"): + _load_scoped(None) + + def test_clean_config_loads_silently_in_strict_mode( + self, tmp_path: Path, monkeypatch + ) -> None: + monkeypatch.setenv("PYTHINKER_STRICT_CONFIG", "1") + _write(tmp_path / "share" / "config.toml", "session_retention_days = 30\n") + + config = _load_scoped(None) + + assert config.session_retention_days == 30 From 8afc647ac6ddd28c2b6d50a678797dda617bd778 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 05:59:34 -0400 Subject: [PATCH 18/49] docs(tasks): record trust-gating and unknown-key checkpoints --- tasks/todo.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tasks/todo.md b/tasks/todo.md index 53757b4c..027caec1 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -10,9 +10,12 @@ decision-complete plan mode, `f5b9b06a` print channel discipline, `c8d82d38` review git-context+merge-base, `e2e74b70` parallel-tool concurrency policy, `1615cfbd` reactive overflow recovery (loop + - SimpleCompaction halving; classify_api_error → soul/api_errors.py). - NEXT (Tier-1 high/M, plan order): per-project trust gating of - config/hooks; unknown-config-key diagnostics; model-switch context + SimpleCompaction halving; classify_api_error → soul/api_errors.py), + `0f39a3b1` per-project trust gating of project hooks (project_trust.py + store + /trust persistence + untrusted-TOML tolerance), `9d1178f4` + unknown-config-key diagnostics (unknown_config_key_paths + + PYTHINKER_STRICT_CONFIG). + NEXT (Tier-1 high/M, plan order): model-switch context continuity; known-safe command auto-approval; MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From bf549c28afca5e65ccd194612db4eca2a6148c57 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:06:43 -0400 Subject: [PATCH 19/49] feat(shell): carry a conversation summary across /model switches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /model discarded the entire conversation by starting a fresh session. The switch now summarizes the outgoing session with the OUTGOING model — only plain text crosses the model boundary, so the incoming provider never sees foreign thinking blocks or tool-call schemas — and seeds the new session's context with it before Reload. Best-effort with a start-fresh fallback on empty history, summarization failure, or model_switch_carryover=false. SimpleCompaction gains summarize_all() (no preserved tail) atop the extracted overflow-halving summarizer. Plan item: core-loop/model-switch-context-continuity (Tier 1). --- CHANGELOG.md | 1 + src/pythinker_code/config.py | 8 + src/pythinker_code/soul/compaction.py | 120 ++++++++---- src/pythinker_code/ui/shell/slash.py | 49 ++++- tests/core/test_config.py | 1 + tests/core/test_config_unknown_keys.py | 8 +- tests/core/test_model_switch_carryover.py | 172 ++++++++++++++++++ .../ui_and_conv/test_shell_slash_commands.py | 3 + 8 files changed, 315 insertions(+), 47 deletions(-) create mode 100644 tests/core/test_model_switch_carryover.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dab1f1e..e5c27170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Switching models keeps your conversation.** `/model` used to start a fresh session, discarding all context. The switch now seeds the new session with a plain-text summary written by the outgoing model (so provider-specific message formats never cross the boundary), falling back to the old fresh start if summarization fails; disable with `model_switch_carryover = false`. - **Typo'd config keys are no longer silently ignored.** Loading now diffs the merged config against the model schema and warns with the dotted path and originating scope file for every unrecognized key (`defaut_yolo = true` names itself and its file instead of silently changing nothing); `PYTHINKER_STRICT_CONFIG=1` escalates the findings to a startup error for CI. - **Project-scope hooks now require trusting the project.** A cloned repository's `.pythinker/config.toml` could previously auto-execute its shell hooks at session start. Hooks from project and local scopes now load only after `/trust` records a durable per-project decision (stored user-side in `trusted_projects.json`, keyed by the resolved repo root); until then they are stripped with a warning naming the fix. Broken TOML in an untrusted project no longer blocks startup — the scope is treated as empty with a warning, while trusted projects keep the loud error. - **Agent orchestration guidance is sharper for substantial tasks.** The default prompt now sharpens work-shaping guidance, and a new root-only runtime reminder nudges substantial normal-mode tasks toward the lightest effective path — direct tools, `SetTodoList`, foreground `RunAgents`, or verification — while backing off for plan mode, `/goal`, auto mode, and subagents. diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index 6c776968..ba6443e4 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -1060,6 +1060,14 @@ class Config(BaseModel): "still appended on top." ), ) + model_switch_carryover: bool = Field( + default=True, + description=( + "On /model switches, seed the new session with a plain-text summary " + "of the conversation produced by the outgoing model. False keeps the " + "previous start-fresh behavior." + ), + ) background: BackgroundConfig = Field( default_factory=BackgroundConfig, description="Background task configuration" ) diff --git a/src/pythinker_code/soul/compaction.py b/src/pythinker_code/soul/compaction.py index db4f8284..8933f2f5 100644 --- a/src/pythinker_code/soul/compaction.py +++ b/src/pythinker_code/soul/compaction.py @@ -158,14 +158,85 @@ async def compact( if compact_message is None: return CompactionResult(messages=list(to_preserve), usage=None) - # Call pythinker_core.step to get the compacted context. - # NOTE: the summary length is bounded by the chat provider's construction-time - # max output tokens (LLM default_max_tokens, or PYTHINKER_MODEL_MAX_TOKENS). A - # tighter *per-call* cap would require a max-tokens parameter on - # ``ChatProvider.generate`` (and ``pythinker_core.step``), which neither exposes - # today; adding one ripples across every provider backend, so it is out of scope here. logger.debug("Compacting context...") - to_compact = list(prepared.to_compact) + summary_message, usage = await self._summarize_to_message( + list(prepared.to_compact), compact_message, llm, custom_instruction + ) + if summary_message is None: + note = Message( + role="user", + content=[ + system( + "Previous context exceeded the model's window and was " + "dropped without summarization. Re-read files or re-run " + "commands if earlier results are needed." + ) + ], + ) + return CompactionResult(messages=[note, *to_preserve], usage=None) + if usage: + logger.debug( + "Compaction used {input} input tokens and {output} output tokens", + input=usage.input, + output=usage.output, + ) + + content: list[ContentPart] = [ + system("Previous context has been compacted. Here is the compaction output:") + ] + # drop thinking parts if any + content.extend(part for part in summary_message.content if not isinstance(part, ThinkPart)) + compacted_messages: list[Message] = [Message(role="user", content=content)] + compacted_messages.extend(to_preserve) + return CompactionResult(messages=compacted_messages, usage=usage) + + async def summarize_all( + self, messages: Sequence[Message], llm: LLM, *, custom_instruction: str = "" + ) -> str | None: + """Summarize *messages* to plain text with no preserved tail. + + For boundaries where raw history must not cross — e.g. carrying a + conversation to a different model, whose provider may reject the + outgoing model's thinking blocks or tool-call schemas — only text + survives. Returns ``None`` when there is nothing to summarize or + nothing fits the context window. + """ + to_compact = list(messages) + if not to_compact: + return None + compact_message = self._build_compact_message( + to_compact, custom_instruction=custom_instruction + ) + summary_message, _usage = await self._summarize_to_message( + to_compact, compact_message, llm, custom_instruction + ) + if summary_message is None: + return None + text = "\n".join( + part.text for part in summary_message.content if isinstance(part, TextPart) + ).strip() + return text or None + + async def _summarize_to_message( + self, + to_compact: list[Message], + compact_message: Message, + llm: LLM, + custom_instruction: str, + ) -> tuple[Message | None, TokenUsage | None]: + """Run the summarization request, halving the slice on context overflow. + + The request carries the whole to-compact slice, so it can itself + exceed the context window. On a context-length rejection the oldest + half is dropped and the request retried; ``(None, None)`` means even + a single message did not fit. + + NOTE: the summary length is bounded by the chat provider's + construction-time max output tokens (LLM default_max_tokens, or + PYTHINKER_MODEL_MAX_TOKENS). A tighter per-call cap would require a + max-tokens parameter on ``ChatProvider.generate`` (and + ``pythinker_core.step``), which neither exposes today. + """ while True: try: result = await pythinker_core.step( @@ -174,12 +245,8 @@ async def compact( toolset=EmptyToolset(), history=[compact_message], ) - break + return result.message, result.usage except Exception as e: - # The compaction request itself can exceed the context window - # (it carries the whole to-compact slice). Drop the oldest half - # and retry; when nothing summarizable fits, preserve only the - # tail with an explicit dropped-context note instead of failing. if not is_context_overflow_error(e): raise if len(to_compact) <= 1: @@ -187,17 +254,7 @@ async def compact( "Compaction request still exceeds the context window with a " "single message; dropping unsummarized older context" ) - note = Message( - role="user", - content=[ - system( - "Previous context exceeded the model's window and was " - "dropped without summarization. Re-read files or re-run " - "commands if earlier results are needed." - ) - ], - ) - return CompactionResult(messages=[note, *to_preserve], usage=None) + return None, None dropped = len(to_compact) // 2 to_compact = to_compact[dropped:] logger.warning( @@ -209,23 +266,6 @@ async def compact( compact_message = self._build_compact_message( to_compact, custom_instruction=custom_instruction ) - if result.usage: - logger.debug( - "Compaction used {input} input tokens and {output} output tokens", - input=result.usage.input, - output=result.usage.output, - ) - - content: list[ContentPart] = [ - system("Previous context has been compacted. Here is the compaction output:") - ] - compacted_msg = result.message - - # drop thinking parts if any - content.extend(part for part in compacted_msg.content if not isinstance(part, ThinkPart)) - compacted_messages: list[Message] = [Message(role="user", content=content)] - compacted_messages.extend(to_preserve) - return CompactionResult(messages=compacted_messages, usage=result.usage) class PrepareResult(NamedTuple): compact_message: Message | None diff --git a/src/pythinker_code/ui/shell/slash.py b/src/pythinker_code/ui/shell/slash.py index f3a10028..a2fb08b7 100644 --- a/src/pythinker_code/ui/shell/slash.py +++ b/src/pythinker_code/ui/shell/slash.py @@ -363,12 +363,59 @@ async def model(app: Shell, args: str): session.state.additional_dirs = list(current_session.state.additional_dirs) if session.state.additional_dirs: await asyncio.to_thread(session.save_state) - console.print(f"[{_t.success}]Starting fresh session for the new model...[/]") + carried = False + if config.model_switch_carryover: + carried = await _carry_context_to_session(soul, session) + if carried: + console.print(f"[{_t.success}]Carrying a conversation summary to the new model...[/]") + else: + console.print(f"[{_t.success}]Starting fresh session for the new model...[/]") raise Reload(session_id=session.id) raise Reload(session_id=soul.runtime.session.id) +async def _carry_context_to_session(soul: PythinkerSoul, new_session: Any) -> bool: + """Seed the new session with a summary written by the outgoing model. + + Summarizing before the switch means only plain text crosses the model + boundary — no provider-specific thinking blocks or tool-call schemas the + incoming provider might reject. Best-effort: any failure leaves the new + session untouched so the switch degrades to the start-fresh behavior. + """ + from pythinker_code.soul.compaction import SimpleCompaction + from pythinker_code.soul.context import Context + from pythinker_code.soul.message import system + from pythinker_code.utils.logging import logger + + history = list(soul.context.history) + llm = soul.runtime.llm + if not history or llm is None: + return False + compaction = SimpleCompaction(base_prompt=soul.runtime.config.compact_prompt) + try: + summary = await compaction.summarize_all(history, llm) + except Exception: + logger.warning("Model-switch carry-over summarization failed", exc_info=True) + return False + if not summary: + return False + from pythinker_core.message import Message + + seed = Message( + role="user", + content=[ + system( + "Summary of the conversation so far, carried over from the previous " + f"model session:\n{summary}\nContinue from this state. Details were " + "summarized away; re-read files before editing them." + ) + ], + ) + await Context(file_backend=new_session.context_file).append_message(seed) + return True + + _PROVIDER_LABEL_OVERRIDES = { "managed:minimax-anthropic": "MiniMax", "managed:opencode-go-openai": "OpenCode Go (OpenAI)", diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 419ff589..0fd051f8 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -73,6 +73,7 @@ def test_default_config_dump(): }, "goal": {"auto_continue": False, "max_continuations": 3}, "compact_prompt": None, + "model_switch_carryover": True, "notifications": { "claim_stale_after_ms": 15000, }, diff --git a/tests/core/test_config_unknown_keys.py b/tests/core/test_config_unknown_keys.py index 3d1c14d8..386af197 100644 --- a/tests/core/test_config_unknown_keys.py +++ b/tests/core/test_config_unknown_keys.py @@ -48,9 +48,7 @@ def test_valid_keys_produce_no_findings(self) -> None: def test_map_fields_allow_arbitrary_keys_but_check_values(self) -> None: data = { - "providers": { - "mine": {"type": "openai", "base_url": "x", "api_key": "k", "tpyo": 1} - } + "providers": {"mine": {"type": "openai", "base_url": "x", "api_key": "k", "tpyo": 1}} } unknown = unknown_config_key_paths(Config, data) @@ -86,9 +84,7 @@ def test_strict_mode_escalates_to_error(self, tmp_path: Path, monkeypatch) -> No with pytest.raises(ConfigError, match="defaut_yolo"): _load_scoped(None) - def test_clean_config_loads_silently_in_strict_mode( - self, tmp_path: Path, monkeypatch - ) -> None: + def test_clean_config_loads_silently_in_strict_mode(self, tmp_path: Path, monkeypatch) -> None: monkeypatch.setenv("PYTHINKER_STRICT_CONFIG", "1") _write(tmp_path / "share" / "config.toml", "session_retention_days = 30\n") diff --git a/tests/core/test_model_switch_carryover.py b/tests/core/test_model_switch_carryover.py new file mode 100644 index 00000000..12d75cb7 --- /dev/null +++ b/tests/core/test_model_switch_carryover.py @@ -0,0 +1,172 @@ +"""Model-switch context continuity. + +/model used to discard the whole conversation ("Starting fresh session"). +The switch now seeds the new session with a summary produced by the +OUTGOING model — only plain text crosses the model boundary, sidestepping +provider-specific message formats — and falls back to a fresh session on +any failure. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import cast + +import pytest +import pythinker_core +from pythinker_core.chat_provider import APIStatusError +from pythinker_core.message import Message + +from pythinker_code.config import Config +from pythinker_code.llm import LLM +from pythinker_code.soul.compaction import SimpleCompaction +from pythinker_code.wire.types import TextPart + + +def _fake_llm() -> LLM: + return cast(LLM, SimpleNamespace(chat_provider=None)) + + +def _history(n_pairs: int = 3) -> list[Message]: + messages: list[Message] = [] + for index in range(n_pairs): + messages.append(Message(role="user", content=[TextPart(text=f"request {index}")])) + messages.append(Message(role="assistant", content=[TextPart(text=f"reply {index}")])) + return messages + + +def _summary_step(summary: str = "carried summary"): + async def _step(**kwargs): + return SimpleNamespace( + message=Message(role="assistant", content=[TextPart(text=summary)]), + usage=None, + ) + + return _step + + +class TestSummarizeAll: + @pytest.mark.asyncio + async def test_returns_plain_text_summary(self, monkeypatch) -> None: + monkeypatch.setattr(pythinker_core, "step", _summary_step()) + + summary = await SimpleCompaction().summarize_all(_history(), llm=_fake_llm()) + + assert summary == "carried summary" + + @pytest.mark.asyncio + async def test_empty_history_returns_none(self, monkeypatch) -> None: + monkeypatch.setattr(pythinker_core, "step", _summary_step()) + + assert await SimpleCompaction().summarize_all([], llm=_fake_llm()) is None + + @pytest.mark.asyncio + async def test_overflow_halving_applies(self, monkeypatch) -> None: + calls: list[int] = [] + + async def _step(**kwargs): + calls.append(1) + if len(calls) == 1: + raise APIStatusError(400, "maximum context length exceeded") + return SimpleNamespace( + message=Message(role="assistant", content=[TextPart(text="short summary")]), + usage=None, + ) + + monkeypatch.setattr(pythinker_core, "step", _step) + + summary = await SimpleCompaction().summarize_all(_history(8), llm=_fake_llm()) + + assert summary == "short summary" + assert len(calls) == 2 + + @pytest.mark.asyncio + async def test_exhausted_overflow_returns_none(self, monkeypatch) -> None: + async def _step(**kwargs): + raise APIStatusError(400, "maximum context length exceeded") + + monkeypatch.setattr(pythinker_core, "step", _step) + + assert await SimpleCompaction().summarize_all(_history(), llm=_fake_llm()) is None + + +class TestCarryoverConfig: + def test_carryover_defaults_on(self) -> None: + assert Config(default_model="", models={}, providers={}).model_switch_carryover is True + + +class TestCarryContextToSession: + @pytest.mark.asyncio + async def test_seeds_new_session_context(self, runtime, tmp_path, monkeypatch) -> None: + from unittest.mock import AsyncMock + + from pythinker_core.tooling.simple import SimpleToolset + + from pythinker_code.soul.agent import Agent + from pythinker_code.soul.context import Context + from pythinker_code.soul.pythinkersoul import PythinkerSoul + from pythinker_code.ui.shell.slash import _carry_context_to_session + + agent = Agent(name="Carry", system_prompt="sys", toolset=SimpleToolset(), runtime=runtime) + context = Context(file_backend=tmp_path / "old.jsonl") + await context.append_message(_history()) + soul = PythinkerSoul(agent, context=context) + monkeypatch.setattr( + SimpleCompaction, "summarize_all", AsyncMock(return_value="THE SUMMARY") + ) + new_session = SimpleNamespace(context_file=tmp_path / "new" / "context.jsonl") + new_session.context_file.parent.mkdir(parents=True) + + carried = await _carry_context_to_session(soul, new_session) + + assert carried is True + seeded = Context(file_backend=new_session.context_file) + assert await seeded.restore() + assert "THE SUMMARY" in seeded.history[0].extract_text(" ") + + @pytest.mark.asyncio + async def test_empty_history_carries_nothing(self, runtime, tmp_path) -> None: + from pythinker_core.tooling.simple import SimpleToolset + + from pythinker_code.soul.agent import Agent + from pythinker_code.soul.context import Context + from pythinker_code.soul.pythinkersoul import PythinkerSoul + from pythinker_code.ui.shell.slash import _carry_context_to_session + + agent = Agent(name="Carry", system_prompt="sys", toolset=SimpleToolset(), runtime=runtime) + soul = PythinkerSoul(agent, context=Context(file_backend=tmp_path / "old.jsonl")) + new_session = SimpleNamespace(context_file=tmp_path / "new" / "context.jsonl") + new_session.context_file.parent.mkdir(parents=True) + + carried = await _carry_context_to_session(soul, new_session) + + assert carried is False + assert not new_session.context_file.exists() + + @pytest.mark.asyncio + async def test_summarization_failure_falls_back_to_fresh( + self, runtime, tmp_path, monkeypatch + ) -> None: + from unittest.mock import AsyncMock + + from pythinker_core.tooling.simple import SimpleToolset + + from pythinker_code.soul.agent import Agent + from pythinker_code.soul.context import Context + from pythinker_code.soul.pythinkersoul import PythinkerSoul + from pythinker_code.ui.shell.slash import _carry_context_to_session + + agent = Agent(name="Carry", system_prompt="sys", toolset=SimpleToolset(), runtime=runtime) + context = Context(file_backend=tmp_path / "old.jsonl") + await context.append_message(_history()) + soul = PythinkerSoul(agent, context=context) + monkeypatch.setattr( + SimpleCompaction, "summarize_all", AsyncMock(side_effect=RuntimeError("boom")) + ) + new_session = SimpleNamespace(context_file=tmp_path / "new" / "context.jsonl") + new_session.context_file.parent.mkdir(parents=True) + + carried = await _carry_context_to_session(soul, new_session) + + assert carried is False + assert not new_session.context_file.exists() diff --git a/tests/ui_and_conv/test_shell_slash_commands.py b/tests/ui_and_conv/test_shell_slash_commands.py index a90b984c..ddc15bff 100644 --- a/tests/ui_and_conv/test_shell_slash_commands.py +++ b/tests/ui_and_conv/test_shell_slash_commands.py @@ -101,6 +101,9 @@ async def test_model_switch_starts_fresh_session(monkeypatch: pytest.MonkeyPatch config = Config( is_from_default_location=True, default_model="model-a", + # Pin the start-fresh path; carry-over has its own coverage in + # tests/core/test_model_switch_carryover.py. + model_switch_carryover=False, providers={ "test-provider": LLMProvider( type="pythinker", From 0ccad00516b9ba0b2f76a4d194388a96df57c4ab Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:06:53 -0400 Subject: [PATCH 20/49] docs(tasks): record model-switch carry-over checkpoint --- tasks/todo.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/todo.md b/tasks/todo.md index 027caec1..bafcea6c 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -15,8 +15,9 @@ store + /trust persistence + untrusted-TOML tolerance), `9d1178f4` unknown-config-key diagnostics (unknown_config_key_paths + PYTHINKER_STRICT_CONFIG). - NEXT (Tier-1 high/M, plan order): model-switch context - continuity; known-safe command auto-approval; MCP startup + `bf549c28` model-switch carry-over (summarize_all with the outgoing + model seeds the new session; model_switch_carryover flag). + NEXT (Tier-1 high/M, plan order): known-safe command auto-approval; MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; feedback diagnostics; fuzzy edit ladder; permissions-state From 5dc87aaff0fd28aac9dee48de7a7952ac28eb3cd Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:16:45 -0400 Subject: [PATCH 21/49] feat(shell): elide approval prompts for provably read-only commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first ls or git status of a session always interrupted the user with an approval dialog. soul/permission.py gains is_known_safe_command(): a positive allowlist, fail closed — the mutation guard's hidden-command/substitution/newline, write-redirection, and network/mutation rejections run first, then every ;/&&/||/| segment must start with an allowlisted read-only binary or read-only git subcommand (--output rejected). Wrappers (sudo/env/time) are never unwrapped, and absolute command paths must live in a system bin dir so a workspace-local fake git cannot ride its basename onto the allowlist. Shell consults it only in the root agent (subagent approval requests stay — they are part of the unattended-denial defense surface) and only after the deny gate, so elision can never override a deny. Elisions are tracked in telemetry; the started event fires at the elision point. Plan item: exec-safety/known-safe-command-auto-approval (Tier 1). --- CHANGELOG.md | 1 + src/pythinker_code/soul/permission.py | 131 +++++++++++++++++++++ src/pythinker_code/tools/shell/__init__.py | 45 ++++--- tests/core/test_permission_profiles.py | 13 +- tests/core/test_safe_command_elision.py | 102 ++++++++++++++++ 5 files changed, 274 insertions(+), 18 deletions(-) create mode 100644 tests/core/test_safe_command_elision.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e5c27170..479c09ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Provably read-only commands no longer prompt for approval.** The first `ls` or `git status` of a session used to interrupt with an approval dialog. A tight positive allowlist (read-only binaries and git subcommands, with hidden-command, write-redirection, wrapper, and fake-path rejections, fail closed) now elides the prompt in the root agent; subagents keep requesting approval as their unattended defense surface, and deny-profile decisions are never overridden. - **Switching models keeps your conversation.** `/model` used to start a fresh session, discarding all context. The switch now seeds the new session with a plain-text summary written by the outgoing model (so provider-specific message formats never cross the boundary), falling back to the old fresh start if summarization fails; disable with `model_switch_carryover = false`. - **Typo'd config keys are no longer silently ignored.** Loading now diffs the merged config against the model schema and warns with the dotted path and originating scope file for every unrecognized key (`defaut_yolo = true` names itself and its file instead of silently changing nothing); `PYTHINKER_STRICT_CONFIG=1` escalates the findings to a startup error for CI. - **Project-scope hooks now require trusting the project.** A cloned repository's `.pythinker/config.toml` could previously auto-execute its shell hooks at session start. Hooks from project and local scopes now load only after `/trust` records a durable per-project decision (stored user-side in `trusted_projects.json`, keyed by the resolved repo root); until then they are stripped with a warning naming the fix. Broken TOML in an untrusted project no longer blocks startup — the scope is treated as empty with a warning, while trusted projects keep the loud error. diff --git a/src/pythinker_code/soul/permission.py b/src/pythinker_code/soul/permission.py index 73660b09..476db2e1 100644 --- a/src/pythinker_code/soul/permission.py +++ b/src/pythinker_code/soul/permission.py @@ -483,6 +483,137 @@ def _shell_hidden_command_reason(command: str) -> str | None: return None +# Positive allowlist for approval elision. Deliberately tight: every entry is +# read-only regardless of arguments once write redirection, hidden commands, +# and network/mutation segments are rejected. Deliberately EXCLUDED: +# env/printenv (env executes commands and prints secrets), find (-exec/ +# -delete), xargs (executes), awk/sed (program execution / in-place), rg +# (--pre executes), sort/uniq/tee (write to file args), less/more (shell +# escapes). +_SAFE_READONLY_COMMANDS = frozenset( + { + "ls", + "pwd", + "cat", + "head", + "tail", + "wc", + "stat", + "file", + "which", + "whoami", + "id", + "date", + "uname", + "basename", + "dirname", + "realpath", + "readlink", + "du", + "df", + "tr", + "cut", + "nl", + "column", + "true", + "false", + "echo", + "printf", + "grep", + "diff", + "cmp", + "md5sum", + "sha1sum", + "sha256sum", + "shasum", + "ps", + "uptime", + "hostname", + "arch", + "tty", + } +) +# Read-only git subcommands. `branch` is excluded (positional arg creates one); +# `--output*` is rejected separately because log/diff/show can write files. +_SAFE_GIT_SUBCOMMANDS = frozenset( + { + "status", + "log", + "diff", + "show", + "rev-parse", + "describe", + "blame", + "ls-files", + "shortlog", + } +) +# Absolute command paths must come from here; a workspace-local fake `git` +# (e.g. ./git or /tmp/x/git) must never ride the allowlist via its basename. +_SYSTEM_BIN_DIRS = frozenset( + {"/bin", "/usr/bin", "/usr/local/bin", "/sbin", "/usr/sbin", "/opt/homebrew/bin"} +) + + +def is_known_safe_command(command: str) -> bool: + """Whether *command* is provably read-only, qualifying for prompt elision. + + Positive allowlist, fail closed. ``shell_mutation_reason`` rejects hidden + sub-commands (substitution, glued operators, unquoted newlines), write + redirections, and mutating/network segments first; then every + ``;``/``&&``/``||``/``|`` segment must start with an allowlisted + read-only binary or read-only git subcommand. Wrapper commands + (sudo/env/time/nohup/...) are never unwrapped here — they disqualify — + and absolute command paths must live in a system bin dir. + """ + if shell_mutation_reason(command) is not None: + return False + try: + tokens = shlex.split(command, posix=True) + except ValueError: + return False + if not tokens: + return False + + saw_segment = False + segment: list[str] = [] + for token in [*tokens, ";"]: + if token in _SHELL_SEGMENT_SEPARATORS: + if segment: + saw_segment = True + if not _is_safe_readonly_segment(segment): + return False + segment = [] + else: + segment.append(token) + return saw_segment + + +def _is_safe_readonly_segment(tokens: list[str]) -> bool: + rest = list(tokens) + # Allow pure KEY=VALUE env-assignment prefixes (FOO=1 grep x); anything + # else that precedes the command (wrappers) disqualifies below. + while rest and "=" in rest[0] and not rest[0].startswith("=") and rest[0].split("=", 1)[0]: + rest.pop(0) + if not rest: + return False + command, args = rest[0], rest[1:] + if "/" in command: + directory, _, base = command.rpartition("/") + if directory not in _SYSTEM_BIN_DIRS: + return False + else: + base = command + base = base.lower() + if base == "git": + subcommand = _git_subcommand(args) + if subcommand not in _SAFE_GIT_SUBCOMMANDS: + return False + # log/diff/show accept --output=<file>, which writes. + return not any(arg.startswith("--output") for arg in args) + return base in _SAFE_READONLY_COMMANDS + + def shell_mutation_reason(command: str) -> str | None: """Best-effort guard for obviously mutating or network-accessing shell commands. diff --git a/src/pythinker_code/tools/shell/__init__.py b/src/pythinker_code/tools/shell/__init__.py index adae1a44..f35c6dd6 100644 --- a/src/pythinker_code/tools/shell/__init__.py +++ b/src/pythinker_code/tools/shell/__init__.py @@ -15,8 +15,12 @@ from pythinker_code.soul.permission import ( active_permission_profile, check_shell_command_allowed, + is_known_safe_command, +) +from pythinker_code.soul.toolset import ( + emit_current_tool_execution_started, + get_current_tool_call_or_none, ) -from pythinker_code.soul.toolset import get_current_tool_call_or_none from pythinker_code.tools.display import BackgroundTaskDisplayBlock, ShellDisplayBlock from pythinker_code.tools.utils import ToolResultBuilder, ToolResultStatus, load_desc from pythinker_code.utils.environment import Environment @@ -152,19 +156,32 @@ async def __call__(self, params: Params) -> ToolReturnValue: if params.run_in_background: return await self._run_in_background(params, scrub_secrets=restricted_profile) - result = await self._approval.request( - self.name, - "run command", - f"Run command `{params.command}`", - display=[ - ShellDisplayBlock( - language="powershell" if self._is_powershell else "bash", - command=params.command, - ) - ], - ) - if not result: - return result.rejection_error() + if self._runtime.role == "root" and is_known_safe_command(params.command): + # Provably read-only — elide the approval prompt for the root + # agent, where prompt fatigue hits the human. Subagents keep the + # request: their approval path is part of the unattended-denial + # defense surface (mutation parsing is best-effort there). The + # deny-path gate (check_shell_command_allowed) already ran above, + # so this only ever replaces a would-be ask, never a deny. The + # started event normally fires when approval resolves; emit it. + from pythinker_code.telemetry import track + + track("shell_safe_command_elision") + emit_current_tool_execution_started() + else: + result = await self._approval.request( + self.name, + "run command", + f"Run command `{params.command}`", + display=[ + ShellDisplayBlock( + language="powershell" if self._is_powershell else "bash", + command=params.command, + ) + ], + ) + if not result: + return result.rejection_error() tool_call = get_current_tool_call_or_none() diff --git a/tests/core/test_permission_profiles.py b/tests/core/test_permission_profiles.py index beebd05a..c81e8df3 100644 --- a/tests/core/test_permission_profiles.py +++ b/tests/core/test_permission_profiles.py @@ -922,11 +922,12 @@ async def request(self, sender, action, description, display=None): # type: ign @pytest.mark.skipif( platform.system() == "Windows", reason="Shell mutation guard examples use POSIX" ) -async def test_read_only_shell_in_root_agent_still_requests_approval( +async def test_root_agent_elides_approval_for_known_safe_commands( runtime: Runtime, environment: Environment, ) -> None: - """In the root (non-subagent) context, even read-only commands still go through approval.""" + """Root context: provably read-only commands skip the prompt; anything + outside the positive allowlist still goes through approval.""" approval_requested: list[str] = [] class TrackingApproval(Approval): @@ -935,14 +936,18 @@ async def request(self, sender, action, description, display=None): # type: ign return await super().request(sender, action, description, display) runtime.role = "root" - tracking = TrackingApproval(yolo=True) # yolo so the approval auto-passes + tracking = TrackingApproval(yolo=True) # yolo so any request auto-passes with tool_call_context("Shell"): shell = Shell(tracking, environment, runtime) result = await shell(ShellParams(command="echo hello")) + unlisted = await shell(ShellParams(command="true && python3 -c 'pass'")) assert not result.is_error - assert "run command" in approval_requested, "root agent should still request approval" + assert approval_requested == ["run command"], ( + "safe command must elide; the unlisted one must still request" + ) + assert not unlisted.is_error @pytest.mark.skipif( diff --git a/tests/core/test_safe_command_elision.py b/tests/core/test_safe_command_elision.py new file mode 100644 index 00000000..64f6b8dc --- /dev/null +++ b/tests/core/test_safe_command_elision.py @@ -0,0 +1,102 @@ +"""Known-safe read-only command auto-approval (prompt elision). + +A positive allowlist, fail closed: every ;/&&/||/| segment must start with +an allowlisted read-only binary (or read-only git subcommand), with the +mutation guard's hidden-command/redirection/network rejections applied +first. Wrappers (sudo/env/time) are never unwrapped — they disqualify. +Absolute command paths must live in a system bin dir so a workspace-local +fake `git` cannot ride the allowlist. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from pythinker_code.soul.permission import is_known_safe_command + +SAFE = [ + "ls -la", + "pwd", + "git status", + "git log --oneline -5", + "git diff HEAD~1", + "cat README.md", + "grep -rn pattern src", + "wc -l file.txt", + "ls | head -3", + "pwd && git status", + "FOO=1 grep x file", + "/usr/bin/git status", + "echo done", +] + +UNSAFE = [ + "rm -rf /tmp/x", + "git push", + "git commit -m x", + "git branch new-branch", + "git status --output=/tmp/f", + "git log --output=/tmp/f --oneline", + "git -c core.pager=rm status", + "ls > /tmp/out", + "echo hi >> /tmp/out", + "ls $(rm -rf /tmp/x)", + "ls `rm -rf /tmp/x`", + "ls; rm -rf /tmp/x", + "ls;rm -rf /tmp/x", + "ls && rm -rf /tmp/x", + "git status\nrm -rf /tmp/x", + "/tmp/fake/git status", + "./git status", + "sudo ls", + "env ls", + "nohup ls", + "find . -exec rm {} ;", + "python -c 'print(1)'", + "curl http://example.com", + "ls | tee /tmp/out", + "sort -o /tmp/out input", + "", + " ", +] + + +class TestIsKnownSafeCommand: + @pytest.mark.parametrize("command", SAFE) + def test_safe_commands_qualify(self, command: str) -> None: + assert is_known_safe_command(command) is True, command + + @pytest.mark.parametrize("command", UNSAFE) + def test_unsafe_commands_never_qualify(self, command: str) -> None: + assert is_known_safe_command(command) is False, repr(command) + + +class TestShellPromptElision: + @pytest.mark.asyncio + async def test_safe_command_skips_approval(self, shell_tool) -> None: + from pythinker_code.tools.shell import Params + + spy = AsyncMock() + shell_tool._approval.request = spy # type: ignore[method-assign] + + result = await shell_tool(Params(command="echo elision-proof")) + + spy.assert_not_awaited() + assert "elision-proof" in result.output + + @pytest.mark.asyncio + async def test_unlisted_command_still_requests_approval(self, shell_tool) -> None: + from pythinker_code.tools.shell import Params + + class _ApprovalReached(Exception): + pass + + async def _raise(*args: object, **kwargs: object) -> object: + raise _ApprovalReached + + shell_tool._approval.request = _raise # type: ignore[method-assign] + + with pytest.raises(_ApprovalReached): + await shell_tool(Params(command="touch /tmp/should-not-run")) From c216386d180b999f1c7645d7533357fc72b14f79 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:16:55 -0400 Subject: [PATCH 22/49] docs(tasks): record safe-command elision checkpoint --- tasks/todo.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks/todo.md b/tasks/todo.md index bafcea6c..7c5666ac 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -17,7 +17,9 @@ PYTHINKER_STRICT_CONFIG). `bf549c28` model-switch carry-over (summarize_all with the outgoing model seeds the new session; model_switch_carryover flag). - NEXT (Tier-1 high/M, plan order): known-safe command auto-approval; MCP startup + `5dc87aaf` known-safe command auto-approval (is_known_safe_command + positive allowlist; root-only elision, deny-gate preserved). + NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; feedback diagnostics; fuzzy edit ladder; permissions-state From 59deceff2130b0d06793349a7540f8f079b06144 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:19:14 -0400 Subject: [PATCH 23/49] fix(security): allowlist inline env prefixes on the elision path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The safe-command elision accepted any KEY=VALUE prefix, so PATH=/tmp/evil ls would resolve ls from the attacker directory — defeating the system-bin pinning — and LD_PRELOAD/DYLD_*/GIT_PAGER prefixes could inject code into otherwise read-only commands. Only harmless locale/timezone assignments (LANG/LC_*/TZ) may now prefix an elidable command; every other assignment fails closed to the normal approval prompt. Flagged by automated security review. --- src/pythinker_code/soul/permission.py | 16 +++++++++++++--- tests/core/test_safe_command_elision.py | 7 ++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/pythinker_code/soul/permission.py b/src/pythinker_code/soul/permission.py index 476db2e1..65f7b1bd 100644 --- a/src/pythinker_code/soul/permission.py +++ b/src/pythinker_code/soul/permission.py @@ -553,6 +553,10 @@ def _shell_hidden_command_reason(command: str) -> str | None: _SYSTEM_BIN_DIRS = frozenset( {"/bin", "/usr/bin", "/usr/local/bin", "/sbin", "/usr/sbin", "/opt/homebrew/bin"} ) +# The only env assignments allowed to prefix an elidable command. Anything +# else fails closed — PATH/LD_*/DYLD_*/GIT_* prefixes can redirect or inject +# code into an otherwise read-only command. +_SAFE_INLINE_ENV_VARS = frozenset({"LANG", "LC_ALL", "LC_COLLATE", "LC_CTYPE", "LC_MESSAGES", "TZ"}) def is_known_safe_command(command: str) -> bool: @@ -591,9 +595,15 @@ def is_known_safe_command(command: str) -> bool: def _is_safe_readonly_segment(tokens: list[str]) -> bool: rest = list(tokens) - # Allow pure KEY=VALUE env-assignment prefixes (FOO=1 grep x); anything - # else that precedes the command (wrappers) disqualifies below. - while rest and "=" in rest[0] and not rest[0].startswith("=") and rest[0].split("=", 1)[0]: + # Env-assignment prefixes are allowlisted, not generically skipped: an + # arbitrary KEY=VALUE prefix is an injection vector (PATH=/tmp/evil ls + # resolves ls from the attacker dir; LD_PRELOAD/DYLD_*/GIT_PAGER inject + # code into otherwise read-only commands). Only harmless locale/timezone + # assignments may prefix an elidable command. + while rest and "=" in rest[0] and not rest[0].startswith("="): + key = rest[0].split("=", 1)[0] + if key not in _SAFE_INLINE_ENV_VARS: + return False rest.pop(0) if not rest: return False diff --git a/tests/core/test_safe_command_elision.py b/tests/core/test_safe_command_elision.py index 64f6b8dc..47aeee41 100644 --- a/tests/core/test_safe_command_elision.py +++ b/tests/core/test_safe_command_elision.py @@ -27,7 +27,7 @@ "wc -l file.txt", "ls | head -3", "pwd && git status", - "FOO=1 grep x file", + "LC_ALL=C grep x file", "/usr/bin/git status", "echo done", ] @@ -58,6 +58,11 @@ "curl http://example.com", "ls | tee /tmp/out", "sort -o /tmp/out input", + "FOO=1 grep x file", + "PATH=/tmp/evil ls", + "LD_PRELOAD=/tmp/evil.so cat f", + "DYLD_INSERT_LIBRARIES=/tmp/e.dylib pwd", + "GIT_PAGER=rm git log", "", " ", ] From 5dadb1c6dffedfe93cbb4baa23f71bf4c152b8f2 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:29:56 -0400 Subject: [PATCH 24/49] test(e2e): pin shell approval round-trip with non-elidable commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The approval-protocol e2e tests drove Shell with 'echo ok', which the new known-safe elision now runs without a prompt — the round-trip these tests exist to pin never started. 'env echo ok' keeps stdout identical while the wrapper prefix disqualifies elision, so the approval exchange still exercises request/approve/reject. Fallout from 5dc87aaf (caught by the full tests_e2e scope). --- tests_e2e/test_wire_approvals_tools.py | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests_e2e/test_wire_approvals_tools.py b/tests_e2e/test_wire_approvals_tools.py index d8748ba4..44c02c2f 100644 --- a/tests_e2e/test_wire_approvals_tools.py +++ b/tests_e2e/test_wire_approvals_tools.py @@ -58,7 +58,7 @@ def test_shell_approval_approve(tmp_path) -> None: "\n".join( [ "text: step1", - build_shell_tool_call("tc-1", "echo ok"), + build_shell_tool_call("tc-1", "env echo ok"), ] ), "text: done", @@ -105,7 +105,7 @@ def test_shell_approval_approve(tmp_path) -> None: "payload": { "type": "function", "id": "tc-1", - "function": {"name": "Shell", "arguments": '{"command": "echo ok"}'}, + "function": {"name": "Shell", "arguments": '{"command": "env echo ok"}'}, "extras": None, }, }, @@ -132,13 +132,13 @@ def test_shell_approval_approve(tmp_path) -> None: "tool_call_id": "tc-1", "sender": "Shell", "action": "run command", - "description": "Run command `echo ok`", + "description": "Run command `env echo ok`", "source_kind": "foreground_turn", "source_id": "<uuid>", "agent_id": None, "subagent_type": None, "source_description": None, - "display": [{"type": "shell", "language": "bash", "command": "echo ok"}], + "display": [{"type": "shell", "language": "bash", "command": "env echo ok"}], }, }, { @@ -208,7 +208,7 @@ def test_shell_approval_reject(tmp_path) -> None: "\n".join( [ "text: step1", - build_shell_tool_call("tc-1", "echo ok"), + build_shell_tool_call("tc-1", "env echo ok"), ] ), "text: done", @@ -255,7 +255,7 @@ def test_shell_approval_reject(tmp_path) -> None: "payload": { "type": "function", "id": "tc-1", - "function": {"name": "Shell", "arguments": '{"command": "echo ok"}'}, + "function": {"name": "Shell", "arguments": '{"command": "env echo ok"}'}, "extras": None, }, }, @@ -282,13 +282,13 @@ def test_shell_approval_reject(tmp_path) -> None: "tool_call_id": "tc-1", "sender": "Shell", "action": "run command", - "description": "Run command `echo ok`", + "description": "Run command `env echo ok`", "source_kind": "foreground_turn", "source_id": "<uuid>", "agent_id": None, "subagent_type": None, "source_description": None, - "display": [{"type": "shell", "language": "bash", "command": "echo ok"}], + "display": [{"type": "shell", "language": "bash", "command": "env echo ok"}], }, }, { @@ -322,14 +322,14 @@ def test_approve_for_session(tmp_path) -> None: "\n".join( [ "text: step1", - build_shell_tool_call("tc-1", "echo first"), + build_shell_tool_call("tc-1", "env echo first"), ] ), "text: done", "\n".join( [ "text: step1", - build_shell_tool_call("tc-2", "echo second"), + build_shell_tool_call("tc-2", "env echo second"), ] ), "text: done", @@ -388,7 +388,7 @@ def test_approve_for_session(tmp_path) -> None: "payload": { "type": "function", "id": "tc-1", - "function": {"name": "Shell", "arguments": '{"command": "echo first"}'}, + "function": {"name": "Shell", "arguments": '{"command": "env echo first"}'}, "extras": None, }, }, @@ -415,13 +415,13 @@ def test_approve_for_session(tmp_path) -> None: "tool_call_id": "tc-1", "sender": "Shell", "action": "run command", - "description": "Run command `echo first`", + "description": "Run command `env echo first`", "source_kind": "foreground_turn", "source_id": "<uuid>", "agent_id": None, "subagent_type": None, "source_description": None, - "display": [{"type": "shell", "language": "bash", "command": "echo first"}], + "display": [{"type": "shell", "language": "bash", "command": "env echo first"}], }, }, { @@ -501,7 +501,7 @@ def test_approve_for_session(tmp_path) -> None: "payload": { "type": "function", "id": "tc-2", - "function": {"name": "Shell", "arguments": '{"command": "echo second"}'}, + "function": {"name": "Shell", "arguments": '{"command": "env echo second"}'}, "extras": None, }, }, @@ -582,7 +582,7 @@ def test_yolo_skips_approval(tmp_path) -> None: "\n".join( [ "text: step1", - build_shell_tool_call("tc-1", "echo ok"), + build_shell_tool_call("tc-1", "env echo ok"), ] ), "text: done", @@ -626,7 +626,7 @@ def test_yolo_skips_approval(tmp_path) -> None: "payload": { "type": "function", "id": "tc-1", - "function": {"name": "Shell", "arguments": '{"command": "echo ok"}'}, + "function": {"name": "Shell", "arguments": '{"command": "env echo ok"}'}, "extras": None, }, }, @@ -707,7 +707,7 @@ def test_display_block_shell(tmp_path) -> None: "\n".join( [ "text: step1", - build_shell_tool_call("tc-1", "echo ok"), + build_shell_tool_call("tc-1", "env echo ok"), ] ), "text: done", @@ -747,13 +747,13 @@ def test_display_block_shell(tmp_path) -> None: "tool_call_id": "tc-1", "sender": "Shell", "action": "run command", - "description": "Run command `echo ok`", + "description": "Run command `env echo ok`", "source_kind": "foreground_turn", "source_id": "<uuid>", "agent_id": None, "subagent_type": None, "source_description": None, - "display": [{"type": "shell", "language": "bash", "command": "echo ok"}], + "display": [{"type": "shell", "language": "bash", "command": "env echo ok"}], } ) finally: From 05f8642848b5ae8e8df21a8e1f65937beec8a426 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:31:06 -0400 Subject: [PATCH 25/49] feat(mcp): per-server startup timeout with actionable failure diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hung MCP connect left the server in 'connecting' forever and blocked every agent turn (the loop awaits MCP loading with no bound). Connect + inventory is now wrapped in asyncio.wait_for governed by a new mcp.client.startup_timeout_ms (default 30s), and every connect failure is classified into one short actionable line — timeout names the config knob, 401/unauthorized names the exact 'pythinker mcp auth' command, ENOENT names the missing binary — carried on MCPServerInfo and MCPServerSnapshot and rendered by /mcp instead of a bare 'failed'. Plan item: mcp/per-server-startup-timeout-diagnostics (Tier 1). --- CHANGELOG.md | 1 + src/pythinker_code/config.py | 4 + src/pythinker_code/soul/toolset.py | 38 +++++++- src/pythinker_code/ui/shell/mcp_status.py | 7 ++ src/pythinker_code/wire/types.py | 2 + tests/core/test_config.py | 2 +- tests/tools/test_mcp_startup_timeout.py | 108 ++++++++++++++++++++++ tests_e2e/test_wire_skills_mcp.py | 4 +- 8 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 tests/tools/test_mcp_startup_timeout.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 479c09ff..fd5f0e85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **A hung MCP server can no longer stall the whole session.** Server connects are bounded by a new `mcp.client.startup_timeout_ms` (default 30s) — previously a hung connect blocked every agent turn. `/mcp` now shows one actionable line per failed server (timeout → the config knob, 401 → the exact auth command, missing binary → the command path) instead of a bare "failed". - **Provably read-only commands no longer prompt for approval.** The first `ls` or `git status` of a session used to interrupt with an approval dialog. A tight positive allowlist (read-only binaries and git subcommands, with hidden-command, write-redirection, wrapper, and fake-path rejections, fail closed) now elides the prompt in the root agent; subagents keep requesting approval as their unattended defense surface, and deny-profile decisions are never overridden. - **Switching models keeps your conversation.** `/model` used to start a fresh session, discarding all context. The switch now seeds the new session with a plain-text summary written by the outgoing model (so provider-specific message formats never cross the boundary), falling back to the old fresh start if summarization fails; disable with `model_switch_carryover = false`. - **Typo'd config keys are no longer silently ignored.** Loading now diffs the merged config against the model schema and warns with the dotted path and originating scope file for every unrecognized key (`defaut_yolo = true` names itself and its file instead of silently changing nothing); `PYTHINKER_STRICT_CONFIG=1` escalates the findings to a startup error for CI. diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index ba6443e4..aab327bc 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -776,6 +776,10 @@ class MCPClientConfig(BaseModel): tool_call_timeout_ms: int = 60000 """Timeout for tool calls in milliseconds.""" + startup_timeout_ms: int = 30000 + """Per-server connect/inventory timeout. A hung connect would otherwise + block every agent turn, since the loop awaits MCP loading.""" + STATUSLINE_SEGMENT_IDS: tuple[str, ...] = ( "spinner", diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index eff7a8b8..46400233 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -205,6 +205,29 @@ def _set_transport_log_file(transport: Any) -> None: _set_transport_log_file(getattr(client, "transport", None)) +def _classify_mcp_connect_error(error: BaseException, server_name: str) -> str: + """One short actionable line for /mcp explaining a connect failure. + + A bare 'failed' tells the user nothing; each common failure shape names + its fix (config knob, auth command, command path) so recovery does not + require reading logs. + """ + if isinstance(error, TimeoutError): + return ( + "startup timed out — raise mcp.client.startup_timeout_ms if the server is slow to start" + ) + if isinstance(error, FileNotFoundError): + missing = error.filename or str(error) + return f"command not found: {missing} — check the server command/path" + if isinstance(error, ConnectionError): + return "connection failed — is the server running and the URL reachable?" + text = str(error) or type(error).__name__ + lowered = text.lower() + if "401" in text or "unauthorized" in lowered or "authentication" in lowered: + return f"authentication failed — run: pythinker mcp auth {server_name}" + return text.splitlines()[0][:200] + + type ToolType = CallableTool | CallableTool2[Any] type ToolCallKey = tuple[str, str] @@ -857,6 +880,7 @@ def mcp_status_snapshot(self) -> MCPStatusSnapshot | None: name=name, status=info.status, tools=tuple(tool.name for tool in info.tools), + error=info.error, ) for name, info in self._mcp_servers.items() ) @@ -1006,7 +1030,8 @@ async def _connect_server( return server_name, None server_info.status = "connecting" - try: + + async def _open_and_inventory() -> None: async with server_info.client as client: for tool in await client.list_tools(): server_info.tools.append( @@ -1024,6 +1049,14 @@ async def _connect_server( server_name, "prompts", client.list_prompts ) + try: + # Bound connect+inventory: a hung server would otherwise block + # every agent turn (the loop awaits MCP loading). + await asyncio.wait_for( + _open_and_inventory(), + timeout=runtime.config.mcp.client.startup_timeout_ms / 1000, + ) + self._register_mcp_tools(server_name, server_info.tools) for tool in server_info.tools: runtime.mcp_tools[f"mcp__{server_name}__{tool.name}"] = tool @@ -1041,6 +1074,7 @@ async def _connect_server( error=e, ) server_info.status = "failed" + server_info.error = _classify_mcp_connect_error(e, server_name) return server_name, e async def _connect(): @@ -1141,6 +1175,8 @@ class MCPServerInfo: # (mcpext-1). Empty for servers that expose none or do not support them. resources: list[mcp.Resource] prompts: list[mcp.types.Prompt] + # One short actionable line explaining a failed connect, surfaced by /mcp. + error: str | None = None class MCPTool[T: ClientTransport](CallableTool): diff --git a/src/pythinker_code/ui/shell/mcp_status.py b/src/pythinker_code/ui/shell/mcp_status.py index 2270a400..3abcfd1d 100644 --- a/src/pythinker_code/ui/shell/mcp_status.py +++ b/src/pythinker_code/ui/shell/mcp_status.py @@ -125,6 +125,13 @@ def _server_inventory_lines(server: MCPServerSnapshot) -> list[RenderableType]: lines: list[RenderableType] = [Text.assemble(f" {LIST_BULLET} ", (server_name, status_style))] lines.append(Text.assemble(f" {LIST_BULLET} Status: ", (status, status_style))) + if server.status == "failed" and server.error: + lines.append( + Text( + f" {LIST_BULLET} Error: {_safe_text(server.error)}", + style=tui_rich_style("muted"), + ) + ) if server.status == "unauthorized": lines.append( Text( diff --git a/src/pythinker_code/wire/types.py b/src/pythinker_code/wire/types.py index 5e364368..28353fd6 100644 --- a/src/pythinker_code/wire/types.py +++ b/src/pythinker_code/wire/types.py @@ -202,6 +202,8 @@ class MCPServerSnapshot(BaseModel): name: str status: Literal["pending", "connecting", "connected", "failed", "unauthorized"] tools: tuple[str, ...] = () + error: str | None = None + """One short actionable line explaining a failed connect.""" class MCPStatusSnapshot(BaseModel): diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 0fd051f8..fbcaa27e 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -78,7 +78,7 @@ def test_default_config_dump(): "claim_stale_after_ms": 15000, }, "services": {"pythinker_ai_search": None, "pythinker_ai_fetch": None}, - "mcp": {"client": {"tool_call_timeout_ms": 60000}}, + "mcp": {"client": {"tool_call_timeout_ms": 60000, "startup_timeout_ms": 30000}}, "memory": { "lexical_recall": True, "injection_bus": True, diff --git a/tests/tools/test_mcp_startup_timeout.py b/tests/tools/test_mcp_startup_timeout.py new file mode 100644 index 00000000..7479e6db --- /dev/null +++ b/tests/tools/test_mcp_startup_timeout.py @@ -0,0 +1,108 @@ +"""MCP per-server startup timeout and actionable failure diagnostics. + +A hung connect used to leave a server in 'connecting' forever — and the +agent loop awaits MCP loading, so it blocked every turn. Connects are now +bounded by mcp.client.startup_timeout_ms, and failures carry one short +actionable line surfaced by /mcp instead of a bare 'failed'. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, cast + +import pytest + +from pythinker_code.exception import MCPRuntimeError +from pythinker_code.soul.toolset import ( + MCPServerInfo, + PythinkerToolset, + _classify_mcp_connect_error, +) + + +def _hanging_client() -> Any: + return cast(Any, _HangingClient()) + + +class _HangingClient: + async def __aenter__(self) -> _HangingClient: + await asyncio.sleep(60) + return self + + async def __aexit__(self, *exc: object) -> None: + return None + + +class TestStartupTimeout: + @pytest.mark.asyncio + async def test_hung_connect_fails_with_actionable_error(self, runtime) -> None: + runtime.config.mcp.client.startup_timeout_ms = 100 + toolset = PythinkerToolset() + toolset._mcp_servers["slow"] = MCPServerInfo( + status="pending", client=_hanging_client(), tools=[], resources=[], prompts=[] + ) + + with pytest.raises(MCPRuntimeError): + await toolset.load_mcp_tools([], runtime, in_background=False) + + info = toolset._mcp_servers["slow"] + assert info.status == "failed" + assert info.error is not None + assert "timed out" in info.error + assert "startup_timeout_ms" in info.error + + +class TestErrorClassification: + def test_timeout_points_at_the_config_knob(self) -> None: + message = _classify_mcp_connect_error(TimeoutError(), "db") + assert "timed out" in message + assert "startup_timeout_ms" in message + + def test_unauthorized_points_at_auth_command(self) -> None: + message = _classify_mcp_connect_error(Exception("HTTP 401 Unauthorized"), "db") + assert "pythinker mcp auth db" in message + + def test_missing_command_named(self) -> None: + error = FileNotFoundError(2, "No such file or directory") + error.filename = "npxx" + message = _classify_mcp_connect_error(error, "db") + assert "command not found" in message + assert "npxx" in message + + def test_connection_refused(self) -> None: + message = _classify_mcp_connect_error(ConnectionRefusedError("refused"), "db") + assert "running" in message + + def test_generic_error_is_first_line_only(self) -> None: + message = _classify_mcp_connect_error(Exception("first line\nsecond line"), "db") + assert message == "first line" + + +class TestDiagnosticsSurface: + def test_snapshot_carries_error(self) -> None: + toolset = PythinkerToolset() + toolset._mcp_servers["db"] = MCPServerInfo( + status="failed", + client=_hanging_client(), + tools=[], + resources=[], + prompts=[], + error="connection refused — is the server running?", + ) + + snapshot = toolset.mcp_status_snapshot() + + assert snapshot is not None + assert snapshot.servers[0].error == "connection refused — is the server running?" + + def test_failed_server_render_includes_error(self) -> None: + from pythinker_code.ui.shell.mcp_status import _server_inventory_lines + from pythinker_code.wire.types import MCPServerSnapshot + + lines = _server_inventory_lines( + MCPServerSnapshot(name="db", status="failed", error="startup timed out") + ) + + joined = " ".join(str(line) for line in lines) + assert "startup timed out" in joined diff --git a/tests_e2e/test_wire_skills_mcp.py b/tests_e2e/test_wire_skills_mcp.py index 0ea92be7..a081ccf9 100644 --- a/tests_e2e/test_wire_skills_mcp.py +++ b/tests_e2e/test_wire_skills_mcp.py @@ -317,7 +317,7 @@ def ping(text: str) -> str: "connected": 0, "total": 1, "tools": 0, - "servers": [{"name": "test", "status": "connecting", "tools": []}], + "servers": [{"name": "test", "status": "connecting", "tools": [], "error": None}], }, }, }, @@ -339,7 +339,7 @@ def ping(text: str) -> str: "connected": 1, "total": 1, "tools": 1, - "servers": [{"name": "test", "status": "connected", "tools": ["ping"]}], + "servers": [{"name": "test", "status": "connected", "tools": ["ping"], "error": None}], }, }, }, From df4801f54035061ea4b8fe90af301bb03672946b Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:33:37 -0400 Subject: [PATCH 26/49] test: refresh wire snapshots for the MCP server error field The serde and e2e snapshots pin wire-model dumps; the new optional MCPServerSnapshot.error field appears as null in them. Applied via --inline-snapshot=fix (deliberate, follows 05f86428). --- tests/core/test_wire_message.py | 6 +----- tests_e2e/test_wire_approvals_tools.py | 17 +++++++++++++---- tests_e2e/test_wire_skills_mcp.py | 13 +++++++++++-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/core/test_wire_message.py b/tests/core/test_wire_message.py index 5583e914..e9d39e3c 100644 --- a/tests/core/test_wire_message.py +++ b/tests/core/test_wire_message.py @@ -182,11 +182,7 @@ async def test_wire_message_serde(): "total": 1, "tools": 0, "servers": [ - { - "name": "context7", - "status": "connecting", - "tools": [], - } + {"name": "context7", "status": "connecting", "tools": [], "error": None} ], }, }, diff --git a/tests_e2e/test_wire_approvals_tools.py b/tests_e2e/test_wire_approvals_tools.py index 44c02c2f..00466bc2 100644 --- a/tests_e2e/test_wire_approvals_tools.py +++ b/tests_e2e/test_wire_approvals_tools.py @@ -138,7 +138,9 @@ def test_shell_approval_approve(tmp_path) -> None: "agent_id": None, "subagent_type": None, "source_description": None, - "display": [{"type": "shell", "language": "bash", "command": "env echo ok"}], + "display": [ + {"type": "shell", "language": "bash", "command": "env echo ok"} + ], }, }, { @@ -288,7 +290,9 @@ def test_shell_approval_reject(tmp_path) -> None: "agent_id": None, "subagent_type": None, "source_description": None, - "display": [{"type": "shell", "language": "bash", "command": "env echo ok"}], + "display": [ + {"type": "shell", "language": "bash", "command": "env echo ok"} + ], }, }, { @@ -421,7 +425,9 @@ def test_approve_for_session(tmp_path) -> None: "agent_id": None, "subagent_type": None, "source_description": None, - "display": [{"type": "shell", "language": "bash", "command": "env echo first"}], + "display": [ + {"type": "shell", "language": "bash", "command": "env echo first"} + ], }, }, { @@ -501,7 +507,10 @@ def test_approve_for_session(tmp_path) -> None: "payload": { "type": "function", "id": "tc-2", - "function": {"name": "Shell", "arguments": '{"command": "env echo second"}'}, + "function": { + "name": "Shell", + "arguments": '{"command": "env echo second"}', + }, "extras": None, }, }, diff --git a/tests_e2e/test_wire_skills_mcp.py b/tests_e2e/test_wire_skills_mcp.py index a081ccf9..3048aa3c 100644 --- a/tests_e2e/test_wire_skills_mcp.py +++ b/tests_e2e/test_wire_skills_mcp.py @@ -317,7 +317,9 @@ def ping(text: str) -> str: "connected": 0, "total": 1, "tools": 0, - "servers": [{"name": "test", "status": "connecting", "tools": [], "error": None}], + "servers": [ + {"name": "test", "status": "connecting", "tools": [], "error": None} + ], }, }, }, @@ -339,7 +341,14 @@ def ping(text: str) -> str: "connected": 1, "total": 1, "tools": 1, - "servers": [{"name": "test", "status": "connected", "tools": ["ping"], "error": None}], + "servers": [ + { + "name": "test", + "status": "connected", + "tools": ["ping"], + "error": None, + } + ], }, }, }, From 1487afdfb305e75d36b7de29952df44291dfb0b6 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:33:53 -0400 Subject: [PATCH 27/49] docs(tasks): record elision and MCP-timeout checkpoints --- tasks/todo.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tasks/todo.md b/tasks/todo.md index 7c5666ac..9ecf8fe9 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -19,6 +19,10 @@ model seeds the new session; model_switch_carryover flag). `5dc87aaf` known-safe command auto-approval (is_known_safe_command positive allowlist; root-only elision, deny-gate preserved). + `5dc87aaf`+`59deceff` known-safe command elision (+env-prefix + allowlist security fix; e2e approval pins moved to wrapper commands + `5dadb1c6`), `05f86428`+`df4801f5` MCP startup timeout + actionable + failure diagnostics (/mcp shows classified error lines). NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From 60fc8b16e82353a9f620269ce315f4d7f56a0288 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:41:00 -0400 Subject: [PATCH 28/49] feat(mcp): per-server tool allow/deny filtering (enabledTools/disabledTools) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A server listing 30 tools floods the model tool list with all of them. mcp.json server entries now accept optional enabledTools (exclusive allowlist) and disabledTools (denylist, wins on conflict): filtered tools are skipped at connect time — never registered in the toolset or runtime.mcp_tools — and MCPTool re-checks membership at call time as defense in depth for tool maps shared with subagents and future live tool-list updates. No filter fields keeps today's permissive behavior. Plan item: mcp/per-server-tool-allow-deny-filtering (Tier 1). --- CHANGELOG.md | 1 + src/pythinker_code/soul/toolset.py | 78 ++++++++++++++++- tests/tools/test_mcp_tool_filter.py | 127 ++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 tests/tools/test_mcp_tool_filter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fd5f0e85..bd778ee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **MCP servers can be scoped to specific tools.** Optional `enabledTools` (exclusive allowlist) and `disabledTools` (denylist, wins on conflict) arrays per server in `mcp.json` keep a noisy server from flooding the model's tool list — filtered tools are never registered, and a call-time re-check guards shared tool maps. - **A hung MCP server can no longer stall the whole session.** Server connects are bounded by a new `mcp.client.startup_timeout_ms` (default 30s) — previously a hung connect blocked every agent turn. `/mcp` now shows one actionable line per failed server (timeout → the config knob, 401 → the exact auth command, missing binary → the command path) instead of a bare "failed". - **Provably read-only commands no longer prompt for approval.** The first `ls` or `git status` of a session used to interrupt with an approval dialog. A tight positive allowlist (read-only binaries and git subcommands, with hidden-command, write-redirection, wrapper, and fake-path rejections, fail closed) now elides the prompt in the root agent; subagents keep requesting approval as their unattended defense surface, and deny-profile decisions are never overridden. - **Switching models keeps your conversation.** `/model` used to start a fresh session, discarding all context. The switch now seeds the new session with a plain-text summary written by the outgoing model (so provider-specific message formats never cross the boundary), falling back to the old fresh start if summarization fails; disable with `model_switch_carryover = false`. diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index 46400233..a0947fb5 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -228,6 +228,40 @@ def _classify_mcp_connect_error(error: BaseException, server_name: str) -> str: return text.splitlines()[0][:200] +@dataclass(frozen=True, slots=True) +class McpToolFilter: + """Optional per-server tool scoping from mcp.json. + + ``enabledTools`` (exclusive allowlist) and ``disabledTools`` (denylist, + wins on conflict) keep noisy servers from flooding the model tool list + and double as a safety scoping knob. No filter fields → permissive. + """ + + enabled: frozenset[str] | None = None + deny: frozenset[str] = frozenset() + + @classmethod + def from_server_config(cls, server_config: Any) -> McpToolFilter: + enabled: object = getattr(server_config, "enabledTools", None) + disabled: object = getattr(server_config, "disabledTools", None) or () + enabled_names: frozenset[str] | None = None + if isinstance(enabled, list): + enabled_names = frozenset( + name for name in cast(list[Any], enabled) if isinstance(name, str) + ) + deny_names: frozenset[str] = frozenset() + if isinstance(disabled, list): + deny_names = frozenset( + name for name in cast(list[Any], disabled) if isinstance(name, str) + ) + return cls(enabled=enabled_names, deny=deny_names) + + def allows(self, tool_name: str) -> bool: + if tool_name in self.deny: + return False + return self.enabled is None or tool_name in self.enabled + + type ToolType = CallableTool | CallableTool2[Any] type ToolCallKey = tuple[str, str] @@ -1033,9 +1067,29 @@ async def _connect_server( async def _open_and_inventory() -> None: async with server_info.client as client: + skipped: list[str] = [] for tool in await client.list_tools(): + if server_info.tool_filter and not server_info.tool_filter.allows( + tool.name + ): + skipped.append(tool.name) + continue server_info.tools.append( - MCPTool(server_name, tool, client, runtime=runtime) + MCPTool( + server_name, + tool, + client, + runtime=runtime, + tool_filter=server_info.tool_filter, + ) + ) + if skipped: + logger.info( + "MCP server {server_name}: {n} tools filtered out by " + "mcp.json enabledTools/disabledTools: {names}", + server_name=server_name, + n=len(skipped), + names=", ".join(sorted(skipped)), ) # Resources/prompts are optional MCP capabilities; a server # that exposes none (or does not support the request) must @@ -1121,7 +1175,12 @@ async def _connect(): client = fastmcp.Client(MCPConfig(mcpServers={server_name: server_config})) _configure_mcp_client_stderr_log(client, runtime, server_name) self._mcp_servers[server_name] = MCPServerInfo( - status="pending", client=client, tools=[], resources=[], prompts=[] + status="pending", + client=client, + tools=[], + resources=[], + prompts=[], + tool_filter=McpToolFilter.from_server_config(server_config), ) if not any(server_info.status == "pending" for server_info in self._mcp_servers.values()): @@ -1177,6 +1236,8 @@ class MCPServerInfo: prompts: list[mcp.types.Prompt] # One short actionable line explaining a failed connect, surfaced by /mcp. error: str | None = None + # Optional mcp.json enabledTools/disabledTools scoping for this server. + tool_filter: McpToolFilter | None = None class MCPTool[T: ClientTransport](CallableTool): @@ -1207,6 +1268,7 @@ def __init__( client: fastmcp.Client[T], *, runtime: Runtime, + tool_filter: McpToolFilter | None = None, **kwargs: Any, ): super().__init__( @@ -1227,6 +1289,7 @@ def __init__( self._runtime = runtime self._timeout = timedelta(milliseconds=runtime.config.mcp.client.tool_call_timeout_ms) self._action_name = f"mcp:{mcp_tool.name}" + self._tool_filter = tool_filter @property def mcp_server_name(self) -> str: @@ -1234,6 +1297,17 @@ def mcp_server_name(self) -> str: return self._mcp_server_name async def __call__(self, *args: Any, **kwargs: Any) -> ToolReturnValue: + # Call-time re-check of the list-time filter: defense in depth for + # tool maps shared across agents (e.g. runtime.mcp_tools handed to + # subagent specs) and future live tools/list_changed updates. + if self._tool_filter is not None and not self._tool_filter.allows(self._mcp_tool.name): + return ToolError( + message=( + f"MCP tool '{self._mcp_tool.name}' is disabled for server " + f"'{self._mcp_server_name}' by mcp.json tool filtering." + ), + brief="Tool disabled", + ) description = f"Call MCP tool `{self._mcp_tool.name}`." result = await self._runtime.approval.request(self.name, self._action_name, description) if not result: diff --git a/tests/tools/test_mcp_tool_filter.py b/tests/tools/test_mcp_tool_filter.py new file mode 100644 index 00000000..77ed1ee8 --- /dev/null +++ b/tests/tools/test_mcp_tool_filter.py @@ -0,0 +1,127 @@ +"""Per-server MCP tool allow/deny filtering (mcp.json enabledTools/disabledTools). + +Noisy servers flood the model tool list with every tool they expose; +filtering scopes them at list time (never registered) and re-checks at +call time as defense in depth for shared tool maps. +""" + +from __future__ import annotations + +from typing import Any, cast + +import mcp.types +import pytest +from fastmcp.mcp_config import MCPConfig + +from pythinker_code.soul.toolset import ( + MCPServerInfo, + MCPTool, + McpToolFilter, + PythinkerToolset, +) + + +class _ListingClient: + def __init__(self, tool_names: list[str], *, fail_on_call: bool = False) -> None: + self._tool_names = tool_names + self.calls: list[str] = [] + self._fail_on_call = fail_on_call + + async def __aenter__(self) -> _ListingClient: + return self + + async def __aexit__(self, *exc: object) -> None: + return None + + async def list_tools(self) -> list[mcp.types.Tool]: + return [mcp.types.Tool(name=name, inputSchema={}) for name in self._tool_names] + + async def list_resources(self) -> list[object]: + return [] + + async def list_prompts(self) -> list[object]: + return [] + + async def call_tool(self, *args: Any, **kwargs: Any) -> Any: + raise AssertionError("call_tool must not be reached for a filtered tool") + + +class TestFilterSemantics: + def test_no_filter_allows_everything(self) -> None: + assert McpToolFilter().allows("anything") is True + + def test_enabled_list_is_exclusive(self) -> None: + flt = McpToolFilter(enabled=frozenset({"a"})) + assert flt.allows("a") is True + assert flt.allows("b") is False + + def test_disabled_blocks(self) -> None: + flt = McpToolFilter(deny=frozenset({"b"})) + assert flt.allows("a") is True + assert flt.allows("b") is False + + def test_deny_wins_over_enabled(self) -> None: + flt = McpToolFilter(enabled=frozenset({"a"}), deny=frozenset({"a"})) + assert flt.allows("a") is False + + def test_from_server_config_reads_extras(self) -> None: + config = MCPConfig.model_validate( + { + "mcpServers": { + "x": { + "command": "echo", + "args": [], + "enabledTools": ["a", "b"], + "disabledTools": ["b"], + } + } + } + ) + + flt = McpToolFilter.from_server_config(config.mcpServers["x"]) + + assert flt.allows("a") is True + assert flt.allows("b") is False + assert flt.allows("c") is False + + def test_from_server_config_defaults_permissive(self) -> None: + config = MCPConfig.model_validate({"mcpServers": {"x": {"command": "echo", "args": []}}}) + + assert McpToolFilter.from_server_config(config.mcpServers["x"]).allows("any") is True + + +class TestListTimeFiltering: + @pytest.mark.asyncio + async def test_disallowed_tools_never_register(self, runtime) -> None: + toolset = PythinkerToolset() + toolset._mcp_servers["srv"] = MCPServerInfo( + status="pending", + client=cast(Any, _ListingClient(["read_db", "drop_db"])), + tools=[], + resources=[], + prompts=[], + tool_filter=McpToolFilter(enabled=frozenset({"read_db"})), + ) + + await toolset.load_mcp_tools([], runtime, in_background=False) + + assert "read_db" in toolset._tool_dict + assert "drop_db" not in toolset._tool_dict + assert "mcp__srv__drop_db" not in runtime.mcp_tools + + +class TestCallTimeGate: + @pytest.mark.asyncio + async def test_denied_tool_errors_without_reaching_server(self, runtime) -> None: + tool = MCPTool( + "srv", + mcp.types.Tool(name="drop_db", inputSchema={}), + cast(Any, _ListingClient([], fail_on_call=True)), + runtime=runtime, + tool_filter=McpToolFilter(deny=frozenset({"drop_db"})), + ) + + result = await tool.call({}) + + assert result.is_error + assert "disabled" in (result.message or "").lower() From 143de0b571efff1b11f53f54d1dbca141250c3ba Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:41:08 -0400 Subject: [PATCH 29/49] docs(tasks): record MCP tool-filtering checkpoint --- tasks/todo.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/todo.md b/tasks/todo.md index 9ecf8fe9..b0a06bba 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -23,6 +23,8 @@ allowlist security fix; e2e approval pins moved to wrapper commands `5dadb1c6`), `05f86428`+`df4801f5` MCP startup timeout + actionable failure diagnostics (/mcp shows classified error lines). + `60fc8b16` MCP per-server tool filtering (enabledTools/disabledTools, + list-time + call-time). NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From a01e5940b4b25b6f5b5d389a95ded80256b28616 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:50:02 -0400 Subject: [PATCH 30/49] feat(subagents): spawn-time context fork for foreground agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New children started blank, relying on the orchestrator hand-writing a context packet into every prompt. Agent(fork_context=true) now seeds a new foreground child with the parent's conversational spine — user requests and assistant text, with tool traffic (whose call/result pairing would dangle), thinking parts, injected reminders/notifications, and checkpoint markers all filtered out. The fork reads the persisted parent context (inheriting the restore-time pairing repair) and seeds the child's own context file, so resume keeps working unchanged. Invalid with resume or run_in_background (background fork is a tracked follow-up); read failures degrade to a blank child rather than failing the spawn. Tool/agent schema snapshots refreshed. Plan item: multi-agent/spawn-time-context-fork (Tier 1, foreground slice). --- src/pythinker_code/subagents/core.py | 57 ++++++++- src/pythinker_code/subagents/runner.py | 26 ++++ src/pythinker_code/tools/agent/__init__.py | 19 +++ tests/core/test_default_agent.py | 5 + tests/subagents/test_context_fork.py | 133 +++++++++++++++++++++ tests/tools/test_tool_schemas.py | 5 + 6 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 tests/subagents/test_context_fork.py diff --git a/src/pythinker_code/subagents/core.py b/src/pythinker_code/subagents/core.py index e7f5fc98..d5a57520 100644 --- a/src/pythinker_code/subagents/core.py +++ b/src/pythinker_code/subagents/core.py @@ -8,15 +8,21 @@ from __future__ import annotations -from collections.abc import Callable +import re +from collections.abc import Callable, Sequence from dataclasses import dataclass, replace from typing import TYPE_CHECKING +from pythinker_core.message import Message + +from pythinker_code.notifications import is_notification_message from pythinker_code.soul.context import Context +from pythinker_code.soul.message import is_system_reminder_message from pythinker_code.soul.pythinkersoul import PythinkerSoul from pythinker_code.subagents.builder import SubagentBuilder from pythinker_code.subagents.models import AgentLaunchSpec, AgentTypeDefinition from pythinker_code.subagents.store import SubagentStore +from pythinker_code.wire.types import TextPart, ThinkPart GIT_CONTEXT_AGENT_TYPES = frozenset({"explore", "review", "code_reviewer", "security_reviewer"}) """Read-oriented agent types whose first prompt gets a git-context prefix. @@ -49,6 +55,54 @@ class SubagentRunSpec: launch_spec: AgentLaunchSpec prompt: str resumed: bool + # Filtered parent transcript to seed a NEW child with (spawn-time context + # fork). Ignored on resume; see filter_history_for_fork. + fork_history: Sequence[Message] | None = None + + +_CHECKPOINT_MARKER_RE = re.compile(r"^CHECKPOINT \d+$") + + +def filter_history_for_fork(history: Sequence[Message]) -> list[Message]: + """Filter a parent transcript down to its conversational spine. + + Keeps user requests and assistant text; drops tool traffic (whose + call/result pairing would dangle out of context), thinking parts, + injected reminder/notification wrappers, and checkpoint markers — the + child should inherit intent and conclusions, not raw activity. + """ + forked: list[Message] = [] + for message in history: + if message.role == "user": + if is_notification_message(message) or is_system_reminder_message(message): + continue + text = message.extract_text(" ").strip() + if not text or _CHECKPOINT_MARKER_RE.match(text): + continue + forked.append(Message(role="user", content=[TextPart(text=text)])) + elif message.role == "assistant": + text = " ".join( + part.text + for part in message.content + if isinstance(part, TextPart) and not isinstance(part, ThinkPart) + ).strip() + if not text: + continue + forked.append(Message(role="assistant", content=[TextPart(text=text)])) + return forked + + +async def seed_forked_history( + context: Context, fork_history: Sequence[Message] | None, *, resumed: bool +) -> None: + """Seed a NEW child's context with the forked parent transcript. + + Persisting through the child's own context file keeps resume working + unchanged. No-op on resume or when the child already has history. + """ + if not fork_history or resumed or context.history: + return + await context.append_message(list(fork_history)) def _prepend_output_language_instruction(prompt: str) -> str: @@ -82,6 +136,7 @@ async def prepare_soul( # 2. Restore conversation context context = Context(store.context_path(spec.agent_id)) await context.restore() + await seed_forked_history(context, spec.fork_history, resumed=spec.resumed) if on_stage: on_stage("context_restored") diff --git a/src/pythinker_code/subagents/runner.py b/src/pythinker_code/subagents/runner.py index 6b8acf81..30062dec 100644 --- a/src/pythinker_code/subagents/runner.py +++ b/src/pythinker_code/subagents/runner.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING from pythinker_core.chat_provider import APIStatusError, ChatProviderError +from pythinker_core.message import Message from pythinker_core.tooling import ToolError, ToolOk, ToolReturnValue from pythinker_code.approval_runtime import ( @@ -259,6 +260,7 @@ class ForegroundRunRequest: requested_type: str model: str | None resume: str | None + fork_context: bool = False @dataclass(frozen=True, slots=True, kw_only=True) @@ -275,6 +277,26 @@ def __init__(self, runtime: Runtime): self._store: SubagentStore = runtime.subagent_store self._builder = SubagentBuilder(runtime) + async def _load_fork_history(self) -> list[Message] | None: + """Read and filter the parent transcript for a context-forked child. + + Reads the persisted parent context file rather than live soul state, + so the fork inherits exactly what would survive a parent restore + (including the restore-time pairing repair). Best-effort: a read + failure degrades to a blank child rather than failing the spawn. + """ + from pythinker_code.soul.context import Context + from pythinker_code.subagents.core import filter_history_for_fork + + try: + parent_context = Context(file_backend=self._runtime.session.context_file) + await parent_context.restore() + except Exception: + logger.warning("Context fork: failed to read parent history", exc_info=True) + return None + forked = filter_history_for_fork(parent_context.history) + return forked or None + async def run(self, req: ForegroundRunRequest) -> ToolReturnValue: prepared = await self._prepare_instance(req) agent_id = prepared.record.agent_id @@ -293,12 +315,16 @@ async def run(self, req: ForegroundRunRequest) -> ToolReturnValue: output_writer = SubagentOutputWriter(self._store.output_path(agent_id)) output_writer.stage("runner_started") + fork_history = None + if req.fork_context and not resumed: + fork_history = await self._load_fork_history() spec = SubagentRunSpec( agent_id=agent_id, type_def=type_def, launch_spec=launch_spec, prompt=req.prompt, resumed=resumed, + fork_history=fork_history, ) self._store.update_instance( agent_id, diff --git a/src/pythinker_code/tools/agent/__init__.py b/src/pythinker_code/tools/agent/__init__.py index c9aaba6b..1e54e245 100644 --- a/src/pythinker_code/tools/agent/__init__.py +++ b/src/pythinker_code/tools/agent/__init__.py @@ -51,6 +51,16 @@ class Params(BaseModel): default=None, description="Optional agent ID to resume instead of creating a new instance.", ) + fork_context: bool = Field( + default=False, + description=( + "Seed the new agent with a filtered transcript of this conversation (user " + "requests and assistant replies; tool traffic and thinking are dropped). Use " + "when the child needs the discussion so far without a hand-written context " + "packet. New foreground instances only — invalid with resume or " + "run_in_background." + ), + ) run_in_background: bool = Field( default=False, description=( @@ -272,6 +282,14 @@ async def __call__(self, params: Params) -> ToolReturnValue: requested_type = params.subagent_type or "coder" if err := self.check_execution_policy(requested_type): return err + if params.fork_context and (params.resume is not None or params.run_in_background): + return ToolError( + message=( + "fork_context seeds a NEW foreground agent; it cannot be combined " + "with resume or run_in_background." + ), + brief="Invalid fork_context", + ) if params.run_in_background: return await self._run_in_background(params) if params.isolation != "none": @@ -290,6 +308,7 @@ async def __call__(self, params: Params) -> ToolReturnValue: requested_type=params.subagent_type or "coder", model=params.model, resume=params.resume, + fork_context=params.fork_context, ) if timeout is not None: return await asyncio.wait_for(runner.run(req), timeout=timeout) diff --git a/tests/core/test_default_agent.py b/tests/core/test_default_agent.py index 562979ab..497f4c96 100644 --- a/tests/core/test_default_agent.py +++ b/tests/core/test_default_agent.py @@ -463,6 +463,11 @@ async def test_default_agent_background_bash_guardrails(runtime: Runtime): "default": None, "description": "Optional agent ID to resume instead of creating a new instance.", }, + "fork_context": { + "default": False, + "description": "Seed the new agent with a filtered transcript of this conversation (user requests and assistant replies; tool traffic and thinking are dropped). Use when the child needs the discussion so far without a hand-written context packet. New foreground instances only — invalid with resume or run_in_background.", + "type": "boolean", + }, "run_in_background": { "default": False, "description": "Whether to run the agent in the background. Prefer false unless the task can continue independently and there is a clear benefit to returning control before the result is needed.", diff --git a/tests/subagents/test_context_fork.py b/tests/subagents/test_context_fork.py new file mode 100644 index 00000000..7c5db8ad --- /dev/null +++ b/tests/subagents/test_context_fork.py @@ -0,0 +1,133 @@ +"""Spawn-time context fork: child inherits a filtered parent transcript. + +New children start blank and rely on the orchestrator hand-writing context +packets. fork_context seeds the child with the conversational spine — user +requests and assistant text — and drops tool traffic (whose call/result +pairing would dangle out of context), thinking, and injected wrappers. +""" + +from __future__ import annotations + +import pytest +from pythinker_core.message import Message, ToolCall + +from pythinker_code.soul.context import Context +from pythinker_code.subagents.core import filter_history_for_fork, seed_forked_history +from pythinker_code.wire.types import TextPart, ThinkPart + + +def _user(text: str) -> Message: + return Message(role="user", content=[TextPart(text=text)]) + + +def _assistant(text: str) -> Message: + return Message(role="assistant", content=[TextPart(text=text)]) + + +def _tool_call(call_id: str) -> ToolCall: + return ToolCall.model_validate( + {"type": "function", "id": call_id, "function": {"name": "Shell", "arguments": "{}"}} + ) + + +class TestFilterHistoryForFork: + def test_keeps_conversational_spine_in_order(self) -> None: + history = [_user("do the thing"), _assistant("done, summary follows")] + + forked = filter_history_for_fork(history) + + assert [(m.role, m.extract_text(" ")) for m in forked] == [ + ("user", "do the thing"), + ("assistant", "done, summary follows"), + ] + + def test_drops_tool_messages_and_strips_tool_calls(self) -> None: + assistant = Message( + role="assistant", + content=[TextPart(text="running a check")], + tool_calls=[_tool_call("c1")], + ) + history = [ + _user("task"), + assistant, + Message(role="tool", content=[TextPart(text="raw output")], tool_call_id="c1"), + ] + + forked = filter_history_for_fork(history) + + assert [m.role for m in forked] == ["user", "assistant"] + assert forked[1].tool_calls is None + assert "raw output" not in " ".join(m.extract_text(" ") for m in forked) + + def test_drops_thinking_parts(self) -> None: + history = [ + Message( + role="assistant", + content=[ThinkPart(think="private reasoning"), TextPart(text="the answer")], + ) + ] + + forked = filter_history_for_fork(history) + + assert forked[0].extract_text(" ") == "the answer" + + def test_drops_reminders_notifications_and_checkpoints(self) -> None: + history = [ + _user("<system-reminder>\ninjected guidance\n</system-reminder>"), + _user( + '<notification id="n1" category="task" type="complete" ' + 'source_kind="background_task" source_id="t1">done</notification>' + ), + _user("CHECKPOINT 3"), + _user("real request"), + ] + + forked = filter_history_for_fork(history) + + assert [m.extract_text(" ") for m in forked] == ["real request"] + + def test_drops_empty_assistant_messages(self) -> None: + history = [ + Message(role="assistant", content=[], tool_calls=[_tool_call("c1")]), + ] + + assert filter_history_for_fork(history) == [] + + +class TestSeedForkedHistory: + @pytest.mark.asyncio + async def test_seeds_new_child_and_persists(self, tmp_path) -> None: + context = Context(file_backend=tmp_path / "child.jsonl") + forked = [_user("inherited request"), _assistant("inherited summary")] + + await seed_forked_history(context, forked, resumed=False) + + assert [m.role for m in context.history] == ["user", "assistant"] + reloaded = Context(file_backend=tmp_path / "child.jsonl") + assert await reloaded.restore() + assert reloaded.history[0].extract_text(" ") == "inherited request" + + @pytest.mark.asyncio + async def test_noop_on_resume(self, tmp_path) -> None: + context = Context(file_backend=tmp_path / "child.jsonl") + + await seed_forked_history(context, [_user("x")], resumed=True) + + assert list(context.history) == [] + + @pytest.mark.asyncio + async def test_noop_when_child_already_has_history(self, tmp_path) -> None: + context = Context(file_backend=tmp_path / "child.jsonl") + await context.append_message(_user("existing")) + + await seed_forked_history(context, [_user("forked")], resumed=False) + + assert len(context.history) == 1 + + @pytest.mark.asyncio + async def test_noop_without_fork_history(self, tmp_path) -> None: + context = Context(file_backend=tmp_path / "child.jsonl") + + await seed_forked_history(context, None, resumed=False) + + assert list(context.history) == [] diff --git a/tests/tools/test_tool_schemas.py b/tests/tools/test_tool_schemas.py index 298371ad..a0caf26a 100644 --- a/tests/tools/test_tool_schemas.py +++ b/tests/tools/test_tool_schemas.py @@ -48,6 +48,11 @@ def test_agent_params_schema(agent_tool: AgentTool): "default": None, "description": "Optional agent ID to resume instead of creating a new instance.", }, + "fork_context": { + "default": False, + "description": "Seed the new agent with a filtered transcript of this conversation (user requests and assistant replies; tool traffic and thinking are dropped). Use when the child needs the discussion so far without a hand-written context packet. New foreground instances only — invalid with resume or run_in_background.", + "type": "boolean", + }, "run_in_background": { "default": False, "description": "Whether to run the agent in the background. Prefer false unless the task can continue independently and there is a clear benefit to returning control before the result is needed.", From 763f78ae14e2244d38a9f12566d764d2b40a3a5b Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:50:13 -0400 Subject: [PATCH 31/49] docs(tasks): record context-fork checkpoint --- tasks/todo.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tasks/todo.md b/tasks/todo.md index b0a06bba..166d2a78 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -25,6 +25,9 @@ failure diagnostics (/mcp shows classified error lines). `60fc8b16` MCP per-server tool filtering (enabledTools/disabledTools, list-time + call-time). + `a01e5940` spawn-time context fork (Agent fork_context=true seeds + foreground children with the filtered conversational spine; background + fork is a tracked follow-up). NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From 5c29c06ff96059b9bfd33926f8297487fd1a3336 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 06:53:47 -0400 Subject: [PATCH 32/49] =?UTF-8?q?docs(tasks):=20worktree-isolation=20desig?= =?UTF-8?q?n=20note=20(re-sized=20M=E2=86=92L,=20phased=20seam=20plan)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found ~94 work-dir consumer sites and shared session/ builtin_args across child runtimes; honoring isolation=worktree without a single work-dir seam first would yield false isolation. Phases: P1 mechanical Runtime.work_dir seam, P2 worktree lifecycle in the background runner (create/redirect/report/cleanup, non-git rejection), P3 RunAgents batch reuse. --- tasks/todo.md | 4 +++ tasks/worktree-isolation-design.md | 44 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tasks/worktree-isolation-design.md diff --git a/tasks/todo.md b/tasks/todo.md index 166d2a78..7816a2eb 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -28,6 +28,10 @@ `a01e5940` spawn-time context fork (Agent fork_context=true seeds foreground children with the filtered conversational spine; background fork is a tracked follow-up). + Workspace isolation re-sized to L: design note at + tasks/worktree-isolation-design.md (94 work_dir consumer sites; P1 + seam migration → P2 worktree lifecycle → P3 RunAgents). Execute P1 + next iteration. NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; diff --git a/tasks/worktree-isolation-design.md b/tasks/worktree-isolation-design.md new file mode 100644 index 00000000..821508eb --- /dev/null +++ b/tasks/worktree-isolation-design.md @@ -0,0 +1,44 @@ +# Worktree isolation for write-capable children — design note + +**Status:** approved design, not yet implemented. Plan item: +`multi-agent/enforced-workspace-isolation` (Tier 1). Sized here as L: the +audit found ~94 `session.work_dir` / `PYTHINKER_WORK_DIR` consumer sites +across tools, soul, and permission layers, and child runtimes share the +parent's `session` and `builtin_args` objects — so honoring +`isolation="worktree"` requires a single work-dir seam first. A partial +redirect would produce *false* isolation (child believes it is isolated +while some tools still write the parent tree), which is worse than the +current honest intent-only metadata. + +## Phases + +1. **P1 — work-dir seam (mechanical, behavior-preserving).** + `Runtime` gains `work_dir_override: HostPath | None = None` and a + `work_dir` property returning `work_dir_override or session.work_dir`. + Migrate consumers from `runtime.session.work_dir` to `runtime.work_dir` + (sed-able; session object itself stays shared for persistence paths — + ONLY operational cwd/path-resolution sites migrate; session-file paths + like context/wire stores intentionally keep `session.*`). + `copy_for_subagent(work_dir_override=...)` re-renders `builtin_args` + (`PYTHINKER_WORK_DIR`, `PYTHINKER_WORK_DIR_LS`) for the child. + Verify: full suite green, zero behavior change without an override. + +2. **P2 — worktree lifecycle in the background runner.** + When `isolation="worktree"` and the child type has a write profile: + - Reject with an actionable error when the work dir is not a git repo. + - `git worktree add <session_dir>/worktrees/<agent_id> HEAD` before + launch; build the child runtime with `work_dir_override` pointing at + it. + - On completion, append to the final report: the worktree path and + `git -C <wt> diff --stat` so the orchestrator merges deliberately. + - Cleanup: remove the worktree when the child finished clean with no + changes; retain it (and say so in the report) when it has changes or + failed, matching existing recovery rules. + +3. **P3 — RunAgents batch support** reusing P2 per child. + +## Verification + +- Unit: override property; builtin_args re-render; non-git rejection. +- Integration: two parallel background coders editing the same file land + in distinct worktrees with no cross-clobber; reports name both paths. From 56d6fa53e8af36e85d348623ef20cfa18ff0351d Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:06:42 -0400 Subject: [PATCH 33/49] refactor(soul): Runtime.work_dir seam for worktree isolation (P1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operational cwd/path-resolution sites (26 across tools, soul, permission, app, UI) now read runtime.work_dir — work_dir_override or the session's — instead of reaching through runtime.session.work_dir. copy_for_subagent accepts work_dir_override (re-rendering the child's PYTHINKER_WORK_DIR/_LS prompt args) and propagates it to grandchildren; the shared session keeps owning persistence paths. Behavior-preserving with no override set; full suite green (5381 local + e2e, the two TimeoutError wire tests verified pre-existing/machine-local on the stashed tree). Phase P1 of tasks/worktree-isolation-design.md; P2 wires the worktree lifecycle into the background runner. --- src/pythinker_code/app.py | 4 +- src/pythinker_code/soul/agent.py | 35 ++++++++++++--- src/pythinker_code/soul/permission.py | 4 +- src/pythinker_code/soul/pythinkersoul.py | 16 +++---- src/pythinker_code/soul/slash.py | 2 +- src/pythinker_code/tools/agent/__init__.py | 4 +- src/pythinker_code/tools/memory/__init__.py | 2 +- src/pythinker_code/tools/recall/__init__.py | 4 +- src/pythinker_code/tools/shell/__init__.py | 2 +- src/pythinker_code/tools/todo/__init__.py | 2 +- src/pythinker_code/ui/shell/__init__.py | 2 +- src/pythinker_code/ui/shell/slash.py | 6 +-- tests/core/test_work_dir_seam.py | 49 +++++++++++++++++++++ tests/tools/test_memory_tool.py | 8 +++- tests/ui_and_conv/test_memory_slash.py | 1 + 15 files changed, 110 insertions(+), 31 deletions(-) create mode 100644 tests/core/test_work_dir_seam.py diff --git a/src/pythinker_code/app.py b/src/pythinker_code/app.py index 75bde90f..a5f9774d 100644 --- a/src/pythinker_code/app.py +++ b/src/pythinker_code/app.py @@ -603,7 +603,7 @@ async def await_bg_tasks_shutdown(self, timeout: float = 2.0) -> None: async def _env(self) -> AsyncGenerator[None]: async with _CWD_LOCK: original_cwd = HostPath.cwd() - await pythinker_host.chdir(self._runtime.session.work_dir) + await pythinker_host.chdir(self._runtime.work_dir) try: # to ignore possible warnings from dateparser warnings.filterwarnings("ignore", category=DeprecationWarning) @@ -782,7 +782,7 @@ async def run_shell( """Run the Pythinker CLI instance with shell UI.""" from pythinker_code.ui.shell import Shell, WelcomeInfoItem - work_dir = self._runtime.session.work_dir + work_dir = self._runtime.work_dir welcome_info = [ WelcomeInfoItem(name="Directory", value=str(shorten_home(work_dir))), ] diff --git a/src/pythinker_code/soul/agent.py b/src/pythinker_code/soul/agent.py index b64154c2..a6ceb851 100644 --- a/src/pythinker_code/soul/agent.py +++ b/src/pythinker_code/soul/agent.py @@ -3,7 +3,7 @@ import asyncio import re from collections.abc import Callable -from dataclasses import asdict, dataclass, field +from dataclasses import asdict, dataclass, field, replace from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any, Literal @@ -213,6 +213,16 @@ class Runtime: """HookEngine instance, set by PythinkerCLI after soul creation.""" rearm_injection: Callable[[str], None] | None = None """Callback set by PythinkerSoul so tools can refresh dynamic injections.""" + work_dir_override: HostPath | None = None + """Operational working directory override (e.g. a per-child git worktree). + + The session object stays shared for persistence paths; only the + operational cwd/path-resolution surface reads ``work_dir``.""" + + @property + def work_dir(self) -> HostPath: + """The operational working directory (override, else the session's).""" + return self.work_dir_override or self.session.work_dir def __post_init__(self) -> None: if self.subagent_store is None: @@ -384,14 +394,28 @@ def copy_for_subagent( agent_id: str, subagent_type: str, llm_override: LLM | None = None, + work_dir_override: HostPath | None = None, + work_dir_ls: str | None = None, ) -> Runtime: - """Clone runtime for a subagent.""" + """Clone runtime for a subagent. + + ``work_dir_override`` points the child's operational surface (and its + system-prompt work-dir args) at another directory, e.g. an isolation + worktree; the shared session keeps owning persistence paths. + """ + builtin_args = self.builtin_args + if work_dir_override is not None: + builtin_args = replace( + builtin_args, + PYTHINKER_WORK_DIR=work_dir_override, + PYTHINKER_WORK_DIR_LS=work_dir_ls or "", + ) return Runtime( config=self.config, oauth=self.oauth, llm=llm_override if llm_override is not None else self.llm, session=self.session, - builtin_args=self.builtin_args, + builtin_args=builtin_args, denwa_renji=DenwaRenji(), # subagent must have its own DenwaRenji approval=self.approval.share(), labor_market=self.labor_market, @@ -411,6 +435,7 @@ def copy_for_subagent( subagent_id=agent_id, subagent_type=subagent_type, role="subagent", + work_dir_override=work_dir_override or self.work_dir_override, ) @@ -483,9 +508,7 @@ async def load_agent( ) ) - external_agents = await discover_markdown_agents( - await resolve_agent_roots(runtime.session.work_dir) - ) + external_agents = await discover_markdown_agents(await resolve_agent_roots(runtime.work_dir)) for type_def in materialize_markdown_agent_specs( external_agents, output_dir=runtime.session.dir / "external_agents", diff --git a/src/pythinker_code/soul/permission.py b/src/pythinker_code/soul/permission.py index 65f7b1bd..75a785ae 100644 --- a/src/pythinker_code/soul/permission.py +++ b/src/pythinker_code/soul/permission.py @@ -354,14 +354,14 @@ def check_shell_command_allowed(runtime: Runtime, command: str) -> ToolError | N ) if reason := shell_workspace_escape_reason( command, - work_dir=runtime.session.work_dir, + work_dir=runtime.work_dir, additional_dirs=runtime.additional_dirs, ): return ToolError( message=( f"The active {profile.description} permission profile blocks this shell command " f"because {reason}. Use the Glob/Grep/ReadFile tools or restrict path arguments " - f"to the workspace root ({runtime.session.work_dir}) and approved additional " + f"to the workspace root ({runtime.work_dir}) and approved additional " "directories." ), brief="Permission profile restriction", diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index 908d1a15..b09799a1 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -963,7 +963,7 @@ async def run( matcher_value=text_input_for_hook, input_data=events.user_prompt_submit( session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.session.work_dir)), + cwd=_safe_cwd(str(self._runtime.work_dir)), prompt=text_input_for_hook, ), ) @@ -1006,7 +1006,7 @@ async def run( "Stop", input_data=events.stop( session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.session.work_dir)), + cwd=_safe_cwd(str(self._runtime.work_dir)), stop_hook_active=False, ), ) @@ -1486,7 +1486,7 @@ async def _agent_loop(self) -> TurnOutcome: matcher_value=type(e).__name__, input_data=_hook_events.stop_failure( session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.session.work_dir)), + cwd=_safe_cwd(str(self._runtime.work_dir)), error_type=type(e).__name__, error_message=str(e), ), @@ -1552,7 +1552,7 @@ async def _append_notification(view: NotificationView) -> None: matcher_value=view.event.type, input_data=events.notification( session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.session.work_dir)), + cwd=_safe_cwd(str(self._runtime.work_dir)), sink="llm", notification_type=view.event.type, title=view.event.title, @@ -2097,7 +2097,7 @@ async def _compact_with_retry() -> CompactionResult: matcher_value=trigger_reason, input_data=events.pre_compact( session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.session.work_dir)), + cwd=_safe_cwd(str(self._runtime.work_dir)), trigger=trigger_reason, token_count=before_tokens, custom_instructions=custom_instruction, @@ -2168,7 +2168,7 @@ async def _compact_with_retry() -> CompactionResult: matcher_value=trigger_reason, input_data=events.post_compact( session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.session.work_dir)), + cwd=_safe_cwd(str(self._runtime.work_dir)), trigger=trigger_reason, estimated_token_count=estimated_token_count, compact_summary=summary_text, @@ -2179,7 +2179,7 @@ async def _compact_with_retry() -> CompactionResult: matcher_value="compact", input_data=events.session_start( session_id=self._runtime.session.id, - cwd=_safe_cwd(str(self._runtime.session.work_dir)), + cwd=_safe_cwd(str(self._runtime.work_dir)), source="compact", ), ) @@ -2263,7 +2263,7 @@ async def _harvest_before_compaction( for note in notes: try: await append_scratch_note( - self._runtime.session.work_dir, + self._runtime.work_dir, kind=note.kind, content=note.content, session_id=self._runtime.session.id, diff --git a/src/pythinker_code/soul/slash.py b/src/pythinker_code/soul/slash.py index 41002dd0..4a768f89 100644 --- a/src/pythinker_code/soul/slash.py +++ b/src/pythinker_code/soul/slash.py @@ -68,7 +68,7 @@ async def recap(soul: PythinkerSoul, args: str) -> None: from pythinker_code.session_recap import build_pythinker_recap try: - text = await build_pythinker_recap(soul.runtime.session.work_dir, args) + text = await build_pythinker_recap(soul.runtime.work_dir, args) except ValueError as exc: wire_send(TextPart(text=str(exc))) return diff --git a/src/pythinker_code/tools/agent/__init__.py b/src/pythinker_code/tools/agent/__init__.py index 1e54e245..d1627cb4 100644 --- a/src/pythinker_code/tools/agent/__init__.py +++ b/src/pythinker_code/tools/agent/__init__.py @@ -236,7 +236,7 @@ async def _journal_foreground_agent_start(self, params: Params, actual_type: str if params.resume: details.append(f"resume: {params.resume}") await append_scratch_event( - self._runtime.session.work_dir, + self._runtime.work_dir, session_id=self._runtime.session.id, session_title=self._runtime.session.title or self._runtime.session.state.custom_title, labels=["kind:agent", f"agent-type:{actual_type}"], @@ -737,7 +737,7 @@ async def __call__(self, params: RunAgentsParams) -> ToolReturnValue: from pythinker_code.scratchpad import append_scratch_event scratchpad_result = await append_scratch_event( - self._runtime.session.work_dir, + self._runtime.work_dir, session_id=self._runtime.session.id, session_title=self._runtime.session.title or self._runtime.session.state.custom_title, labels=["kind:agent-batch"], diff --git a/src/pythinker_code/tools/memory/__init__.py b/src/pythinker_code/tools/memory/__init__.py index 40c19bc0..e1a03667 100644 --- a/src/pythinker_code/tools/memory/__init__.py +++ b/src/pythinker_code/tools/memory/__init__.py @@ -26,7 +26,7 @@ class Memory(CallableTool2[Params]): def __init__(self, runtime: Runtime) -> None: super().__init__() self._runtime = runtime - self._store = ProjectMemoryStore(runtime.session.work_dir) + self._store = ProjectMemoryStore(runtime.work_dir) @override async def __call__(self, params: Params) -> ToolReturnValue: diff --git a/src/pythinker_code/tools/recall/__init__.py b/src/pythinker_code/tools/recall/__init__.py index 2f8db506..316cd497 100644 --- a/src/pythinker_code/tools/recall/__init__.py +++ b/src/pythinker_code/tools/recall/__init__.py @@ -137,7 +137,7 @@ async def __call__(self, params: Params) -> ToolReturnValue: return await self._read(session_id) async def _search(self, query: str) -> ToolReturnValue: - work_dir = self._runtime.session.work_dir + work_dir = self._runtime.work_dir try: sessions = await Session.list(work_dir) except Exception as exc: @@ -166,7 +166,7 @@ async def _search(self, query: str) -> ToolReturnValue: async def _read(self, session_id: str) -> ToolReturnValue: if session_id == self._runtime.session.id: return ToolError(message="Cannot recall the current session.", brief="Current session") - work_dir = self._runtime.session.work_dir + work_dir = self._runtime.work_dir try: session = await Session.find(work_dir, session_id) except Exception as exc: diff --git a/src/pythinker_code/tools/shell/__init__.py b/src/pythinker_code/tools/shell/__init__.py index f35c6dd6..3e0e3e1e 100644 --- a/src/pythinker_code/tools/shell/__init__.py +++ b/src/pythinker_code/tools/shell/__init__.py @@ -303,7 +303,7 @@ async def _run_in_background( tool_call_id=tool_call.id, shell_name="Windows PowerShell" if self._is_powershell else "bash", shell_path=str(self._shell_path), - cwd=str(self._runtime.session.work_dir), + cwd=str(self._runtime.work_dir), scrub_secrets=scrub_secrets, ) except Exception as exc: diff --git a/src/pythinker_code/tools/todo/__init__.py b/src/pythinker_code/tools/todo/__init__.py index 7b46cef9..b608dbd7 100644 --- a/src/pythinker_code/tools/todo/__init__.py +++ b/src/pythinker_code/tools/todo/__init__.py @@ -149,7 +149,7 @@ async def _journal_todo_update(self, todos: list[Todo]) -> None: if not todos: details.append("cleared: true") await append_scratch_event( - self._runtime.session.work_dir, + self._runtime.work_dir, session_id=self._runtime.session.id, session_title=self._runtime.session.title or self._runtime.session.state.custom_title, labels=["kind:todo"], diff --git a/src/pythinker_code/ui/shell/__init__.py b/src/pythinker_code/ui/shell/__init__.py index cb913898..4bc2267b 100644 --- a/src/pythinker_code/ui/shell/__init__.py +++ b/src/pythinker_code/ui/shell/__init__.py @@ -601,7 +601,7 @@ def _print_cwd_lost_crash(self) -> None: """Print a crash report when the working directory is no longer accessible.""" runtime = self.soul.runtime if isinstance(self.soul, PythinkerSoul) else None session_id = runtime.session.id if runtime else "unknown" - work_dir = str(runtime.session.work_dir) if runtime else "unknown" + work_dir = str(runtime.work_dir) if runtime else "unknown" info = Table.grid(padding=(0, 1)) info.add_row("Session:", session_id) diff --git a/src/pythinker_code/ui/shell/slash.py b/src/pythinker_code/ui/shell/slash.py index a2fb08b7..a8c4c8ef 100644 --- a/src/pythinker_code/ui/shell/slash.py +++ b/src/pythinker_code/ui/shell/slash.py @@ -1820,7 +1820,7 @@ def _persist_project_trust(soul: PythinkerSoul, *, trusted: bool) -> bool: from pythinker_code.config import find_project_root from pythinker_code.project_trust import set_project_trusted - root = find_project_root(Path(str(soul.runtime.session.work_dir))) + root = find_project_root(Path(str(soul.runtime.work_dir))) if root is None: return False set_project_trusted(root, trusted) @@ -2018,7 +2018,7 @@ async def show_memory(app: Shell, args: str): return from pythinker_code.project_memory import ProjectMemoryStore - store = ProjectMemoryStore(soul.runtime.session.work_dir) + store = ProjectMemoryStore(soul.runtime.work_dir) parts = args.split() if parts and parts[0] == "inbox": memory_config = getattr(soul.runtime.config, "memory", None) @@ -2037,7 +2037,7 @@ async def show_memory(app: Shell, args: str): action = parts[1] if len(parts) > 1 else "list" if action == "scan": - created = await generate_inbox_candidates(store, soul.runtime.session.work_dir) + created = await generate_inbox_candidates(store, soul.runtime.work_dir) console.print(f"Staged {len(created)} memory inbox candidate(s).") return if action == "approve" and len(parts) > 2: diff --git a/tests/core/test_work_dir_seam.py b/tests/core/test_work_dir_seam.py new file mode 100644 index 00000000..3028d842 --- /dev/null +++ b/tests/core/test_work_dir_seam.py @@ -0,0 +1,49 @@ +"""Runtime.work_dir seam (worktree-isolation P1). + +Operational cwd/path-resolution reads go through runtime.work_dir so a +child runtime can be pointed at an isolation worktree without touching +the shared session, which keeps owning persistence paths. +""" + +from __future__ import annotations + +from pythinker_host.path import HostPath + + +class TestWorkDirSeam: + def test_defaults_to_session_work_dir(self, runtime) -> None: + assert runtime.work_dir == runtime.session.work_dir + + def test_subagent_clone_inherits_by_default(self, runtime) -> None: + child = runtime.copy_for_subagent(agent_id="a1", subagent_type="coder") + + assert child.work_dir == runtime.session.work_dir + assert child.builtin_args is runtime.builtin_args + + def test_override_redirects_child_only(self, runtime, tmp_path) -> None: + worktree = HostPath.unsafe_from_local_path(tmp_path / "wt") + + child = runtime.copy_for_subagent( + agent_id="a1", + subagent_type="coder", + work_dir_override=worktree, + work_dir_ls="wt-listing", + ) + + assert child.work_dir == worktree + assert worktree == child.builtin_args.PYTHINKER_WORK_DIR + assert child.builtin_args.PYTHINKER_WORK_DIR_LS == "wt-listing" + # Parent surfaces untouched; session stays shared for persistence. + assert runtime.work_dir == runtime.session.work_dir + assert runtime.session.work_dir == runtime.builtin_args.PYTHINKER_WORK_DIR + assert child.session is runtime.session + + def test_grandchild_inherits_override(self, runtime, tmp_path) -> None: + worktree = HostPath.unsafe_from_local_path(tmp_path / "wt") + child = runtime.copy_for_subagent( + agent_id="a1", subagent_type="coder", work_dir_override=worktree + ) + + grandchild = child.copy_for_subagent(agent_id="a2", subagent_type="explore") + + assert grandchild.work_dir == worktree diff --git a/tests/tools/test_memory_tool.py b/tests/tools/test_memory_tool.py index 9b1359b1..a290391d 100644 --- a/tests/tools/test_memory_tool.py +++ b/tests/tools/test_memory_tool.py @@ -26,7 +26,13 @@ async def __call__(self, argv): def _runtime(tmp_path, role="root"): session = SimpleNamespace(id="sess1", title="t", work_dir=_hp(tmp_path / "repo")) - return SimpleNamespace(role=role, session=session, rearmed=[], rearm_injection=lambda key: None) + return SimpleNamespace( + role=role, + session=session, + work_dir=session.work_dir, + rearmed=[], + rearm_injection=lambda key: None, + ) def _make_tool(tmp_path, monkeypatch, role="root"): diff --git a/tests/ui_and_conv/test_memory_slash.py b/tests/ui_and_conv/test_memory_slash.py index d832c3ae..aa4272e0 100644 --- a/tests/ui_and_conv/test_memory_slash.py +++ b/tests/ui_and_conv/test_memory_slash.py @@ -19,6 +19,7 @@ def _fake_soul(tmp_path, *, consolidation: bool) -> SimpleNamespace: runtime=SimpleNamespace( config=SimpleNamespace(memory=SimpleNamespace(consolidation=consolidation)), session=SimpleNamespace(work_dir=HostPath.unsafe_from_local_path(tmp_path)), + work_dir=HostPath.unsafe_from_local_path(tmp_path), rearm_injection=None, ) ) From d058db0e4e4141254acb283d696b57ebb1e5e26f Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:06:53 -0400 Subject: [PATCH 34/49] docs(tasks): record work-dir seam (P1) checkpoint --- tasks/todo.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/todo.md b/tasks/todo.md index 7816a2eb..6205f1e3 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -28,10 +28,10 @@ `a01e5940` spawn-time context fork (Agent fork_context=true seeds foreground children with the filtered conversational spine; background fork is a tracked follow-up). - Workspace isolation re-sized to L: design note at - tasks/worktree-isolation-design.md (94 work_dir consumer sites; P1 - seam migration → P2 worktree lifecycle → P3 RunAgents). Execute P1 - next iteration. + Workspace isolation: design note at tasks/worktree-isolation-design.md; + P1 work_dir seam DONE `56d6fa53` (Runtime.work_dir property + 26-site + migration + copy_for_subagent override). NEXT: P2 worktree lifecycle + in background runner, then P3 RunAgents. NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From 8bc6c6974593d9b96725e1d8324ebfe0f6de8b87 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:14:59 -0400 Subject: [PATCH 35/49] feat(subagents): enforce worktree isolation for background write agents (P2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isolation='worktree' only recorded intent; parallel coder/implementer children shared one working tree and could clobber each other. The background runner now creates a detached git worktree of HEAD per write-profile child under <session>/worktrees/<agent_id>, points the child runtime at it through the P1 work_dir seam (prompt work-dir args re-rendered), and on completion appends the worktree path plus a diff summary to the final report so the orchestrator merges deliberately. Clean worktrees are removed; changed or failed ones are retained. Non-git roots fail before any model spend with an actionable error; read-profile children log and ignore the request; resume reuses the existing worktree. Local subprocesses are safe here — the manager enforces a local backend for agent tasks. Phase P2 of tasks/worktree-isolation-design.md; P3 (RunAgents batch) remains. --- CHANGELOG.md | 1 + src/pythinker_code/background/agent_runner.py | 60 +++++++++++ src/pythinker_code/background/manager.py | 1 + src/pythinker_code/soul/permission.py | 10 ++ src/pythinker_code/subagents/builder.py | 4 + src/pythinker_code/subagents/core.py | 5 + src/pythinker_code/subagents/worktree.py | 93 ++++++++++++++++ src/pythinker_code/tools/agent/__init__.py | 6 +- tests/core/test_default_agent.py | 2 +- tests/subagents/test_worktree_isolation.py | 100 ++++++++++++++++++ tests/tools/test_tool_schemas.py | 2 +- 11 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 src/pythinker_code/subagents/worktree.py create mode 100644 tests/subagents/test_worktree_isolation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bd778ee5..a26958ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **`isolation="worktree"` is now enforced for background write agents.** Previously it only recorded intent, so parallel coders shared one working tree and could clobber each other. A write-profile child now runs in its own git worktree of HEAD; its final report names the worktree path with a diff summary so changes merge deliberately, clean worktrees are removed, non-git roots fail with an actionable error, and read-profile children ignore the request. - **MCP servers can be scoped to specific tools.** Optional `enabledTools` (exclusive allowlist) and `disabledTools` (denylist, wins on conflict) arrays per server in `mcp.json` keep a noisy server from flooding the model's tool list — filtered tools are never registered, and a call-time re-check guards shared tool maps. - **A hung MCP server can no longer stall the whole session.** Server connects are bounded by a new `mcp.client.startup_timeout_ms` (default 30s) — previously a hung connect blocked every agent turn. `/mcp` now shows one actionable line per failed server (timeout → the config knob, 401 → the exact auth command, missing binary → the command path) instead of a bare "failed". - **Provably read-only commands no longer prompt for approval.** The first `ls` or `git status` of a session used to interrupt with an approval dialog. A tight positive allowlist (read-only binaries and git subcommands, with hidden-command, write-redirection, wrapper, and fake-path rejections, fail closed) now elides the prompt in the root agent; subagents keep requesting approval as their unattended defense surface, and deny-profile decisions are never overridden. diff --git a/src/pythinker_code/background/agent_runner.py b/src/pythinker_code/background/agent_runner.py index 81f3d3ad..3c257ead 100644 --- a/src/pythinker_code/background/agent_runner.py +++ b/src/pythinker_code/background/agent_runner.py @@ -4,8 +4,11 @@ import asyncio import contextlib from dataclasses import replace +from pathlib import Path from typing import TYPE_CHECKING +from pythinker_host.path import HostPath + from pythinker_code.approval_runtime import ( ApprovalSource, reset_current_approval_source, @@ -64,6 +67,7 @@ def __init__( model_override: str | None, timeout_s: int | None = None, resumed: bool = False, + isolation: str | None = None, ) -> None: self._runtime = runtime self._manager = manager @@ -74,6 +78,8 @@ def __init__( self._model_override = model_override self._timeout_s = timeout_s self._resumed = resumed + self._isolation = isolation + self._worktree_path: Path | None = None self._builder = SubagentBuilder(runtime) self._approval_update_tasks: set[asyncio.Task[None]] = set() @@ -184,12 +190,14 @@ async def _run_core(self, output: SubagentOutputWriter) -> None: effective_model=self._model_override, ) + work_dir_override = await self._prepare_isolation_worktree(output) spec = SubagentRunSpec( agent_id=self._agent_id, type_def=type_def, launch_spec=launch_spec, prompt=self._prompt, resumed=self._resumed, + work_dir_override=work_dir_override, ) soul, prompt = await prepare_soul( spec, @@ -235,9 +243,61 @@ async def _ui_loop_fn(wire: Wire) -> None: # runner. Background results are read later via TaskOutput, so the spend rides in # the written transcript rather than the immediate (launch-stub) tool return. output.usage(format_usage_lines("child", soul.cumulative_usage, soul.model_name)) + final_response = await self._append_worktree_report(final_response) output.summary(final_response) self._finalize_safely(outcome="completed") + async def _prepare_isolation_worktree(self, output: SubagentOutputWriter) -> HostPath | None: + """Honor isolation='worktree' for write-profile children. + + Read-profile children gain nothing from isolation (they cannot + mutate), so the request is logged and skipped rather than failing + the task. Raises WorktreeError for non-git roots — actionable, and + better surfaced before any model spend. + """ + if self._isolation != "worktree": + return None + from pythinker_code.soul.permission import subagent_type_allows_file_mutation + from pythinker_code.subagents.worktree import create_agent_worktree + + if not subagent_type_allows_file_mutation(self._subagent_type): + logger.info( + "isolation='worktree' ignored for read-profile subagent type {t}", + t=self._subagent_type, + ) + return None + worktree = Path(str(self._runtime.session.dir)) / "worktrees" / self._agent_id + await create_agent_worktree(Path(str(self._runtime.work_dir)), worktree) + self._worktree_path = worktree + output.stage(f"worktree_created: {worktree}") + return HostPath.unsafe_from_local_path(worktree) + + async def _append_worktree_report(self, final_response: str) -> str: + """Tell the orchestrator where the isolated changes live. + + Changes are never auto-merged; the report carries the worktree path + and a diff summary so merging stays a deliberate decision. Clean + worktrees are removed. + """ + if self._worktree_path is None: + return final_response + from pythinker_code.subagents.worktree import ( + cleanup_agent_worktree, + worktree_change_summary, + ) + + summary = await worktree_change_summary(self._worktree_path) + disposition = await cleanup_agent_worktree( + Path(str(self._runtime.work_dir)), + self._worktree_path, + has_changes=bool(summary), + ) + return ( + f"{final_response}\n\n## Isolation worktree\n" + f"Path: {self._worktree_path} ({disposition})\n" + f"{summary or 'No changes were made.'}" + ) + def _on_approval_runtime_event(self, event: ApprovalRuntimeEvent) -> None: request = event.request if request.source.kind != "background_agent" or request.source.id != self._task_id: diff --git a/src/pythinker_code/background/manager.py b/src/pythinker_code/background/manager.py index 79b00581..a1f3c4f8 100644 --- a/src/pythinker_code/background/manager.py +++ b/src/pythinker_code/background/manager.py @@ -397,6 +397,7 @@ def mark_agent_starting(runtime: TaskRuntime) -> bool: model_override=model_override, timeout_s=effective_timeout, resumed=resumed, + isolation=isolation, ).run() ) self._live_agent_tasks[task_id] = task diff --git a/src/pythinker_code/soul/permission.py b/src/pythinker_code/soul/permission.py index 75a785ae..dda6d94b 100644 --- a/src/pythinker_code/soul/permission.py +++ b/src/pythinker_code/soul/permission.py @@ -264,6 +264,16 @@ class PermissionProfile: _TIME_VALUE_OPTS = {"-o", "-f", "--output", "--format"} +def subagent_type_allows_file_mutation(subagent_type: str) -> bool: + """Whether *subagent_type*'s permission profile permits file mutation. + + Unmapped types default to read_only (fail closed), matching + permission_profile_for_runtime. + """ + profile_name = _SUBAGENT_PROFILES.get(subagent_type, "read_only") + return _PERMISSION_PROFILES[profile_name].allow_file_mutation + + def permission_profile_for_runtime(runtime: Runtime) -> PermissionProfile: """Return the hard permission profile currently enforced for a runtime.""" if runtime.role == "subagent" and runtime.subagent_type: diff --git a/src/pythinker_code/subagents/builder.py b/src/pythinker_code/subagents/builder.py index 9c92f996..c2a64b83 100644 --- a/src/pythinker_code/subagents/builder.py +++ b/src/pythinker_code/subagents/builder.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pythinker_host.path import HostPath + from pythinker_code.llm import clone_llm_with_model_alias from pythinker_code.soul.agent import Agent, Runtime, load_agent from pythinker_code.subagents.models import AgentLaunchSpec, AgentTypeDefinition @@ -15,6 +17,7 @@ async def build_builtin_instance( agent_id: str, type_def: AgentTypeDefinition, launch_spec: AgentLaunchSpec, + work_dir_override: HostPath | None = None, ) -> Agent: effective_model = self.resolve_effective_model(type_def=type_def, launch_spec=launch_spec) llm_override = clone_llm_with_model_alias( @@ -30,6 +33,7 @@ async def build_builtin_instance( agent_id=agent_id, subagent_type=type_def.name, llm_override=llm_override, + work_dir_override=work_dir_override, ) return await load_agent( type_def.agent_file, diff --git a/src/pythinker_code/subagents/core.py b/src/pythinker_code/subagents/core.py index d5a57520..fc0202dd 100644 --- a/src/pythinker_code/subagents/core.py +++ b/src/pythinker_code/subagents/core.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING from pythinker_core.message import Message +from pythinker_host.path import HostPath from pythinker_code.notifications import is_notification_message from pythinker_code.soul.context import Context @@ -58,6 +59,9 @@ class SubagentRunSpec: # Filtered parent transcript to seed a NEW child with (spawn-time context # fork). Ignored on resume; see filter_history_for_fork. fork_history: Sequence[Message] | None = None + # Operational work-dir override (e.g. an isolation worktree); flows into + # the child runtime via copy_for_subagent. + work_dir_override: HostPath | None = None _CHECKPOINT_MARKER_RE = re.compile(r"^CHECKPOINT \d+$") @@ -129,6 +133,7 @@ async def prepare_soul( agent_id=spec.agent_id, type_def=spec.type_def, launch_spec=spec.launch_spec, + work_dir_override=spec.work_dir_override, ) if on_stage: on_stage("agent_built") diff --git a/src/pythinker_code/subagents/worktree.py b/src/pythinker_code/subagents/worktree.py new file mode 100644 index 00000000..5447d8e2 --- /dev/null +++ b/src/pythinker_code/subagents/worktree.py @@ -0,0 +1,93 @@ +"""Git worktree lifecycle for isolated write-capable child agents. + +Local subprocesses are used directly (not the host abstraction): +BackgroundTaskManager.create_agent_task enforces a local backend before +any background agent launches, and worktrees are meaningless across a +remote boundary. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from pythinker_code.utils.logging import logger + +_GIT_TIMEOUT_S = 30.0 + + +class WorktreeError(Exception): + """Worktree lifecycle failure with a user-actionable message.""" + + +async def _git(args: list[str], cwd: Path) -> tuple[int, str, str]: + process = await asyncio.create_subprocess_exec( + "git", + "-C", + str(cwd), + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=_GIT_TIMEOUT_S) + except TimeoutError: + process.kill() + await process.wait() + raise WorktreeError(f"git {' '.join(args)} timed out after {_GIT_TIMEOUT_S:g}s") from None + return ( + process.returncode or 0, + stdout.decode("utf-8", errors="replace").strip(), + stderr.decode("utf-8", errors="replace").strip(), + ) + + +async def create_agent_worktree(repo_dir: Path, dest: Path) -> None: + """Create a detached worktree of HEAD at *dest* for one child agent. + + Raises WorktreeError with an actionable message when *repo_dir* is not + a git repository or the worktree cannot be created. + """ + code, _, _ = await _git(["rev-parse", "--is-inside-work-tree"], repo_dir) + if code != 0: + raise WorktreeError( + f"isolation='worktree' requires a git repository at {repo_dir}; " + "launch without isolation, or run `git init` first" + ) + if dest.exists(): + # Resume of an isolated agent reuses its existing worktree. + return + dest.parent.mkdir(parents=True, exist_ok=True) + code, _, stderr = await _git(["worktree", "add", "--detach", str(dest), "HEAD"], repo_dir) + if code != 0: + first_line = stderr.splitlines()[0] if stderr else "unknown git error" + raise WorktreeError(f"could not create isolation worktree at {dest}: {first_line}") + + +async def worktree_change_summary(worktree: Path) -> str: + """Short human summary of changes in *worktree*; empty string when clean.""" + code, status, _ = await _git(["status", "--porcelain"], worktree) + if code != 0 or not status: + return "" + _, diff_stat, _ = await _git(["diff", "--stat", "HEAD"], worktree) + untracked = sum(1 for line in status.splitlines() if line.startswith("??")) + parts = [part for part in (diff_stat, f"{untracked} untracked file(s)" if untracked else "")] + return "\n".join(part for part in parts if part) + + +async def cleanup_agent_worktree(repo_dir: Path, worktree: Path, *, has_changes: bool) -> str: + """Remove a clean worktree; retain one that carries changes. + + Returns the disposition ("removed" or "retained") for the final report. + Removal failures degrade to retention — losing work is the only + unacceptable outcome here. + """ + if has_changes: + return "retained" + code, _, stderr = await _git(["worktree", "remove", str(worktree)], repo_dir) + if code != 0: + logger.warning( + "Could not remove clean isolation worktree {wt}: {err}", wt=worktree, err=stderr + ) + return "retained" + return "removed" diff --git a/src/pythinker_code/tools/agent/__init__.py b/src/pythinker_code/tools/agent/__init__.py index d1627cb4..ccd4f75b 100644 --- a/src/pythinker_code/tools/agent/__init__.py +++ b/src/pythinker_code/tools/agent/__init__.py @@ -98,8 +98,10 @@ class Params(BaseModel): isolation: Literal["none", "worktree"] = Field( default="none", description=( - "Optional isolation request for background agents. `worktree` records a git-worktree " - "isolation intent for orchestration/recovery; unsupported callers should leave `none`." + "Optional isolation for background agents. `worktree` runs a write-profile child " + "in its own git worktree of HEAD; its final report names the worktree path and a " + "diff summary so changes are merged deliberately (clean worktrees are removed). " + "Requires a git repository; ignored for read-profile child types." ), ) diff --git a/tests/core/test_default_agent.py b/tests/core/test_default_agent.py index 497f4c96..0e3bc498 100644 --- a/tests/core/test_default_agent.py +++ b/tests/core/test_default_agent.py @@ -496,7 +496,7 @@ async def test_default_agent_background_bash_guardrails(runtime: Runtime): }, "isolation": { "default": "none", - "description": "Optional isolation request for background agents. `worktree` records a git-worktree isolation intent for orchestration/recovery; unsupported callers should leave `none`.", + "description": "Optional isolation for background agents. `worktree` runs a write-profile child in its own git worktree of HEAD; its final report names the worktree path and a diff summary so changes are merged deliberately (clean worktrees are removed). Requires a git repository; ignored for read-profile child types.", "enum": ["none", "worktree"], "type": "string", }, diff --git a/tests/subagents/test_worktree_isolation.py b/tests/subagents/test_worktree_isolation.py new file mode 100644 index 00000000..488b9196 --- /dev/null +++ b/tests/subagents/test_worktree_isolation.py @@ -0,0 +1,100 @@ +"""Worktree isolation lifecycle (P2 of tasks/worktree-isolation-design.md).""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from pythinker_code.soul.permission import subagent_type_allows_file_mutation +from pythinker_code.subagents.worktree import ( + WorktreeError, + cleanup_agent_worktree, + create_agent_worktree, + worktree_change_summary, +) + + +async def _git(cwd: Path, *args: str) -> None: + proc = await asyncio.create_subprocess_exec( + "git", + *args, + cwd=cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await proc.communicate() + + +async def _repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + repo.mkdir() + await _git(repo, "init", "-b", "main") + await _git(repo, "config", "user.email", "t@t") + await _git(repo, "config", "user.name", "T") + (repo / "a.txt").write_text("a") + await _git(repo, "add", ".") + await _git(repo, "commit", "-m", "base") + return repo + + +class TestWorktreeLifecycle: + @pytest.mark.asyncio + async def test_clean_roundtrip_removes_worktree(self, tmp_path: Path) -> None: + repo = await _repo(tmp_path) + worktree = tmp_path / "session" / "worktrees" / "a1" + + await create_agent_worktree(repo, worktree) + assert (worktree / "a.txt").read_text() == "a" + + assert await worktree_change_summary(worktree) == "" + disposition = await cleanup_agent_worktree(repo, worktree, has_changes=False) + + assert disposition == "removed" + assert not worktree.exists() + + @pytest.mark.asyncio + async def test_changes_are_summarized_and_retained(self, tmp_path: Path) -> None: + repo = await _repo(tmp_path) + worktree = tmp_path / "wt" + await create_agent_worktree(repo, worktree) + (worktree / "a.txt").write_text("modified") + (worktree / "new.txt").write_text("untracked") + + summary = await worktree_change_summary(worktree) + disposition = await cleanup_agent_worktree(repo, worktree, has_changes=bool(summary)) + + assert "a.txt" in summary + assert "1 untracked" in summary + assert disposition == "retained" + assert worktree.exists() + + @pytest.mark.asyncio + async def test_resume_reuses_existing_worktree(self, tmp_path: Path) -> None: + repo = await _repo(tmp_path) + worktree = tmp_path / "wt" + await create_agent_worktree(repo, worktree) + (worktree / "keep.txt").write_text("work in progress") + + await create_agent_worktree(repo, worktree) # second launch (resume) + + assert (worktree / "keep.txt").read_text() == "work in progress" + + @pytest.mark.asyncio + async def test_non_git_root_is_actionable_error(self, tmp_path: Path) -> None: + plain = tmp_path / "plain" + plain.mkdir() + + with pytest.raises(WorktreeError, match="git repository"): + await create_agent_worktree(plain, tmp_path / "wt") + + +class TestWriteProfileGate: + def test_write_types_qualify(self) -> None: + assert subagent_type_allows_file_mutation("coder") is True + assert subagent_type_allows_file_mutation("implementer") is True + + def test_read_types_do_not(self) -> None: + for read_type in ("explore", "review", "verifier", "judge", "unknown-type"): + assert subagent_type_allows_file_mutation(read_type) is False diff --git a/tests/tools/test_tool_schemas.py b/tests/tools/test_tool_schemas.py index a0caf26a..222d72a4 100644 --- a/tests/tools/test_tool_schemas.py +++ b/tests/tools/test_tool_schemas.py @@ -81,7 +81,7 @@ def test_agent_params_schema(agent_tool: AgentTool): }, "isolation": { "default": "none", - "description": "Optional isolation request for background agents. `worktree` records a git-worktree isolation intent for orchestration/recovery; unsupported callers should leave `none`.", + "description": "Optional isolation for background agents. `worktree` runs a write-profile child in its own git worktree of HEAD; its final report names the worktree path and a diff summary so changes are merged deliberately (clean worktrees are removed). Requires a git repository; ignored for read-profile child types.", "enum": ["none", "worktree"], "type": "string", }, From babe8c25f4fccf79d76164e62743036febee1e57 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:15:07 -0400 Subject: [PATCH 36/49] docs(tasks): record worktree-isolation P2 checkpoint --- tasks/todo.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/todo.md b/tasks/todo.md index 6205f1e3..2ef30ed8 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -29,9 +29,9 @@ foreground children with the filtered conversational spine; background fork is a tracked follow-up). Workspace isolation: design note at tasks/worktree-isolation-design.md; - P1 work_dir seam DONE `56d6fa53` (Runtime.work_dir property + 26-site - migration + copy_for_subagent override). NEXT: P2 worktree lifecycle - in background runner, then P3 RunAgents. + P1 seam `56d6fa53` + P2 lifecycle `8bc6c697` DONE (background + write-profile children get per-agent worktrees, diff-summary reports, + recovery-aware cleanup). P3 (RunAgents batch reuse) remains. NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From 7e6c11bfc8846d3c44740744e8a9122fcfc63403 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:18:38 -0400 Subject: [PATCH 37/49] feat(subagents): document enforced isolation on RunAgents batches (P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RunAgents → Agent → create_agent_task → BackgroundAgentRunner chain already threads isolation per child, so P2 enforcement covers batch fan-outs; the parameter description now states the enforced semantics (per-child worktrees, diff-summary reports, deliberate merging) instead of 'records an intent'. Closes tasks/worktree-isolation-design.md. --- src/pythinker_code/tools/agent/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pythinker_code/tools/agent/__init__.py b/src/pythinker_code/tools/agent/__init__.py index ccd4f75b..5e1592f0 100644 --- a/src/pythinker_code/tools/agent/__init__.py +++ b/src/pythinker_code/tools/agent/__init__.py @@ -169,7 +169,12 @@ class RunAgentsParams(BaseModel): ) isolation: Literal["none", "worktree"] = Field( default="none", - description="Optional isolation request for background child agents.", + description=( + "Optional isolation for background child agents. `worktree` gives each " + "write-profile child its own git worktree of HEAD so parallel children " + "cannot clobber each other; each child's report names its worktree and " + "diff summary for deliberate merging." + ), ) From 1babdf429caaa7c0ae1fc7c16346d982aa7d6708 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:18:38 -0400 Subject: [PATCH 38/49] docs(tasks): close worktree-isolation item (P1-P3 complete) --- tasks/todo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/todo.md b/tasks/todo.md index 2ef30ed8..949c1e6d 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -31,7 +31,7 @@ Workspace isolation: design note at tasks/worktree-isolation-design.md; P1 seam `56d6fa53` + P2 lifecycle `8bc6c697` DONE (background write-profile children get per-agent worktrees, diff-summary reports, - recovery-aware cleanup). P3 (RunAgents batch reuse) remains. + recovery-aware cleanup). P3 DONE (chain verified end-to-end; descriptions state enforced semantics). Isolation item CLOSED. NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From eb13fdaeb3c349a9627683f5e9a65351d1453f7d Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:25:37 -0400 Subject: [PATCH 39/49] feat(tools): graduated fuzzy-matching ladder for edit-location recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whitespace drift or smart-punctuation mismatch in StrReplaceFile's old string hard-failed with 'not found', burning a re-read + retry turn — the drift is invisible in numbered ReadFile output. After the exact match and CRLF fallback miss, a line-window seek now retries with graduated relaxations (trailing-whitespace -> indentation -> unicode-punctuation); the first firing tier replaces the ACTUAL matched file slice — never the needle text — adopting the slice's CRLF style and trailing newline, and the tool message names the relaxation. Ambiguity contract preserved: multiple fuzzy hits without replace_all error with the tier named. Deferred (low value): opt-in final-newline normalization for whole-file writes. Plan item: patch-file-tools/graduated-fuzzy-matching-ladder (Tier 1). --- CHANGELOG.md | 1 + src/pythinker_code/tools/file/replace.py | 114 ++++++++++++++++++ tests/tools/test_replace_fuzzy.py | 142 +++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 tests/tools/test_replace_fuzzy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a26958ed..15586c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Edits recover from whitespace and smart-punctuation drift.** StrReplaceFile no longer hard-fails with "old string not found" when the only mismatch is trailing whitespace, indentation, or smart quotes/dashes: a graduated line-window ladder relocates the edit, replaces the actual file slice (preserving CRLF endings), and names the relaxation it used in the tool message. Multiple fuzzy hits without replace_all still error, so ambiguity is never silently resolved. - **`isolation="worktree"` is now enforced for background write agents.** Previously it only recorded intent, so parallel coders shared one working tree and could clobber each other. A write-profile child now runs in its own git worktree of HEAD; its final report names the worktree path with a diff summary so changes merge deliberately, clean worktrees are removed, non-git roots fail with an actionable error, and read-profile children ignore the request. - **MCP servers can be scoped to specific tools.** Optional `enabledTools` (exclusive allowlist) and `disabledTools` (denylist, wins on conflict) arrays per server in `mcp.json` keep a noisy server from flooding the model's tool list — filtered tools are never registered, and a call-time re-check guards shared tool maps. - **A hung MCP server can no longer stall the whole session.** Server connects are bounded by a new `mcp.client.startup_timeout_ms` (default 30s) — previously a hung connect blocked every agent turn. `/mcp` now shows one actionable line per failed server (timeout → the config knob, 401 → the exact auth command, missing binary → the command path) instead of a bare "failed". diff --git a/src/pythinker_code/tools/file/replace.py b/src/pythinker_code/tools/file/replace.py index 01d2bfe6..21b1d361 100644 --- a/src/pythinker_code/tools/file/replace.py +++ b/src/pythinker_code/tools/file/replace.py @@ -1,5 +1,6 @@ import json from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path from typing import Any, cast, override @@ -124,6 +125,104 @@ def _crlf_translated_edit(content: str, edit: Edit) -> Edit | None: return Edit(old=old, new=new, replace_all=edit.replace_all) +_SMART_PUNCTUATION = str.maketrans( + { + "\u2018": "'", + "\u2019": "'", + "\u201c": '"', + "\u201d": '"', + "\u2013": "-", + "\u2014": "-", + "\u2026": "...", + } +) + +# Graduated relaxations for edit-location recovery, in strictness order. +# Each tier normalizes one drift class the model cannot see in numbered +# ReadFile output; the first tier with hits wins. +_FUZZY_TIERS: tuple[tuple[str, Callable[[str], str]], ...] = ( + ("trailing-whitespace", str.rstrip), + ("indentation", str.strip), + ("unicode-punctuation", lambda line: line.translate(_SMART_PUNCTUATION).strip()), +) + + +@dataclass(frozen=True) +class _FuzzyResult: + content: str + tier: str + count: int + + +def _line_body(line: str) -> str: + return line.rstrip("\r\n") + + +def _find_fuzzy_windows( + file_lines: list[str], needle_lines: list[str], normalize: Callable[[str], str] +) -> list[int]: + targets = [normalize(line) for line in needle_lines] + width = len(targets) + hits: list[int] = [] + index = 0 + while index <= len(file_lines) - width: + if all( + normalize(_line_body(file_lines[index + offset])) == targets[offset] + for offset in range(width) + ): + hits.append(index) + index += width # non-overlapping + else: + index += 1 + return hits + + +def apply_fuzzy_edit(content: str, edit: Edit) -> _FuzzyResult | ToolError | None: + """Line-window relaxation ladder once exact (and CRLF) matching missed. + + Replaces the ACTUAL matched file slice, never the needle text; the + replacement adopts the slice's line-ending style (CRLF preserved) and + its trailing newline so the following line never glues on. Multiple + hits at the firing tier without replace_all keep the ambiguity-error + contract. Returns None when no tier matches. + """ + needle_lines = edit.old.splitlines() + if not needle_lines: + return None + file_lines = content.splitlines(keepends=True) + for tier, normalize in _FUZZY_TIERS: + hits = _find_fuzzy_windows(file_lines, needle_lines, normalize) + if not hits: + continue + if len(hits) > 1 and not edit.replace_all: + return ToolError( + message=( + f"old string {edit.old!r} occurs {len(hits)} times under " + f"{tier}-relaxed matching. Add surrounding context to make it " + "unique, or set replace_all=true." + ), + brief="Ambiguous replacement", + ) + width = len(needle_lines) + rebuilt: list[str] = [] + cursor = 0 + for start in hits: + rebuilt.extend(file_lines[cursor:start]) + slice_lines = file_lines[start : start + width] + replacement = edit.new + if any("\r\n" in line for line in slice_lines) and "\r" not in replacement: + replacement = replacement.replace("\n", "\r\n") + last = slice_lines[-1] + ending = "\r\n" if last.endswith("\r\n") else "\n" if last.endswith("\n") else "" + if ending and not replacement.endswith(("\n", "\r\n")): + replacement += ending + rebuilt.append(replacement) + cursor = start + width + rebuilt.extend(file_lines[cursor:]) + return _FuzzyResult(content="".join(rebuilt), tier=tier, count=len(hits)) + return None + + class StrReplaceFile(CallableTool2[Params]): name: str = "StrReplaceFile" description: str = _BASE_DESCRIPTION @@ -246,6 +345,7 @@ async def __call__(self, params: Params) -> ToolReturnValue: # missing or ambiguous, return before writing so the file stays # unchanged. per_edit_counts: list[int] = [] + fuzzy_notes: list[str] = [] for index, edit in enumerate(edits, start=1): if not edit.old: return ToolError( @@ -263,6 +363,19 @@ async def __call__(self, params: Params) -> ToolReturnValue: edit = crlf_edit match_count = content.count(edit.old) if match_count == 0: + fuzzy = apply_fuzzy_edit(content, edit) + if isinstance(fuzzy, ToolError): + return ToolError( + message=f"Edit {index}: {fuzzy.message}", + brief=fuzzy.brief or "Ambiguous replacement", + ) + if fuzzy is not None: + content = fuzzy.content + per_edit_counts.append(fuzzy.count if edit.replace_all else 1) + fuzzy_notes.append( + f"edit {index} matched with {fuzzy.tier}-relaxed matching" + ) + continue return ToolError( message=( f"No replacements were made for edit {index}: " @@ -322,6 +435,7 @@ async def __call__(self, params: Params) -> ToolReturnValue: message=( f"File successfully edited. " f"Applied {len(edits)} edit(s) with {total_replacements} total replacement(s)." + + (f" ({'; '.join(fuzzy_notes)})" if fuzzy_notes else "") ), display=diff_blocks, ) diff --git a/tests/tools/test_replace_fuzzy.py b/tests/tools/test_replace_fuzzy.py new file mode 100644 index 00000000..10da8269 --- /dev/null +++ b/tests/tools/test_replace_fuzzy.py @@ -0,0 +1,142 @@ +"""Graduated fuzzy-matching ladder for edit-location recovery. + +Whitespace drift or smart-punctuation mismatch used to hard-fail with +'old string not found', burning a re-read + retry turn. When the exact +match (and CRLF fallback) misses, a line-window seek retries with +graduated relaxations — trailing-whitespace, indentation, then +unicode-punctuation — replacing the ACTUAL matched file slice and naming +the fired relaxation in the tool message. Ambiguity semantics keep: +multiple fuzzy hits without replace_all still error. +""" + +from __future__ import annotations + +from pythinker_host.path import HostPath + +from pythinker_code.tools.file.replace import Edit, Params, StrReplaceFile + + +async def _edit(tool: StrReplaceFile, path: HostPath, old: str, new: str, **kw): + return await tool(Params(path=str(path), edit=Edit(old=old, new=new, **kw))) + + +async def test_trailing_whitespace_drift_recovers( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + file_path = temp_work_dir / "t.py" + await file_path.write_text("def f(): \n return 1 \n") + + result = await _edit( + str_replace_file_tool, file_path, "def f():\n return 1", "def f():\n return 2" + ) + + assert not result.is_error + assert "trailing-whitespace" in result.message + assert "return 2" in await file_path.read_text() + + +async def test_indentation_drift_recovers( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + file_path = temp_work_dir / "t.py" + await file_path.write_text("class A:\n\tdef f(self):\n\t\treturn 1\n") + + result = await _edit( + str_replace_file_tool, + file_path, + "def f(self):\n return 1", + " def f(self) -> int:\n return 2", + ) + + assert not result.is_error + assert "indentation" in result.message + content = await file_path.read_text() + assert "def f(self) -> int:" in content + assert "return 1" not in content + + +async def test_smart_punctuation_recovers( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + file_path = temp_work_dir / "t.md" + await file_path.write_text("It’s a “smart” test — really\n") + + result = await _edit( + str_replace_file_tool, + file_path, + 'It\'s a "smart" test - really', + "plain text now", + ) + + assert not result.is_error + assert "unicode-punctuation" in result.message + assert await file_path.read_text() == "plain text now\n" + + +async def test_ambiguous_fuzzy_hits_still_error( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + original = "if x: \n pass\nif x:\t\n pass\n" + file_path = temp_work_dir / "t.py" + await file_path.write_text(original) + + result = await _edit(str_replace_file_tool, file_path, "if x:\n pass", "if y:\n pass") + + assert result.is_error + assert "occurs 2 times" in result.message + assert "relaxed" in result.message + assert await file_path.read_text() == original + + +async def test_replace_all_applies_to_every_fuzzy_hit( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + file_path = temp_work_dir / "t.py" + await file_path.write_text("if x: \n pass\nif x:\t\n pass\n") + + result = await _edit( + str_replace_file_tool, + file_path, + "if x:\n pass", + "if y:\n pass", + replace_all=True, + ) + + assert not result.is_error + assert await file_path.read_text() == "if y:\n pass\nif y:\n pass\n" + + +async def test_no_match_at_any_tier_keeps_not_found_error( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + file_path = temp_work_dir / "t.py" + await file_path.write_text("alpha\nbeta\n") + + result = await _edit(str_replace_file_tool, file_path, "gamma", "delta") + + assert result.is_error + assert "not found" in result.message + + +async def test_exact_match_carries_no_relaxation_note( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + file_path = temp_work_dir / "t.py" + await file_path.write_text("value = 1\n") + + result = await _edit(str_replace_file_tool, file_path, "value = 1", "value = 2") + + assert not result.is_error + assert "relaxed" not in result.message + + +async def test_crlf_file_fuzzy_splice_preserves_line_endings( + str_replace_file_tool: StrReplaceFile, temp_work_dir: HostPath +): + file_path = temp_work_dir / "t.txt" + await file_path.write_text("first \r\nsecond\r\nthird\r\n") + + result = await _edit(str_replace_file_tool, file_path, "first\nsecond", "primary\nextra") + + assert not result.is_error + assert await file_path.read_text() == "primary\r\nextra\r\nthird\r\n" From 3161fc42da857be2abec46abd34dbe66adb1d233 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:25:45 -0400 Subject: [PATCH 40/49] docs(tasks): record fuzzy edit-ladder checkpoint --- tasks/todo.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/todo.md b/tasks/todo.md index 949c1e6d..a375c619 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -32,6 +32,8 @@ P1 seam `56d6fa53` + P2 lifecycle `8bc6c697` DONE (background write-profile children get per-agent worktrees, diff-summary reports, recovery-aware cleanup). P3 DONE (chain verified end-to-end; descriptions state enforced semantics). Isolation item CLOSED. + `eb13fdae` fuzzy edit-recovery ladder (rstrip→strip→unicode-punct + line-window seek in StrReplaceFile; ambiguity contract kept). NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From cde3c726b46576341bc579367d857d979b26bdf6 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:39:40 -0400 Subject: [PATCH 41/49] feat(soul): live permissions-state injection (posture-fingerprinted) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enforcement was rich (profiles, safe mode, yolo/auto, session approvals, shlex-based command classification) but invisible prompt-side — the model discovered policy through denied tool calls. A new PermissionsInjectionProvider renders the enforced profile, posture flags, mutation/network allowances, session-approved actions, and the command-shaping rules the gate can actually classify. Fingerprinted on (profile, yolo, auto, safe_mode, approvals): re-emits exactly on posture changes, after compaction, and on auto toggles; root-only (subagent overlays already document their constraints). Approval gains read accessors is_safe_mode/session_approved_actions. History-shape test pins scoped to their subject; wire-session e2e snapshots refreshed. Plan item: prompts-instructions/dynamic-permissions-state (Tier 1). --- CHANGELOG.md | 1 + src/pythinker_code/soul/approval.py | 8 ++ .../dynamic_injections/permissions_state.py | 99 ++++++++++++++++ src/pythinker_code/soul/pythinkersoul.py | 4 + .../test_permissions_injection_provider.py | 112 ++++++++++++++++++ tests/core/test_plan_mode.py | 8 +- tests/core/test_pythinkersoul_ralph_loop.py | 56 +++++++++ tests/core/test_pythinkersoul_steer.py | 14 ++- tests_e2e/test_wire_sessions.py | 10 +- 9 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 src/pythinker_code/soul/dynamic_injections/permissions_state.py create mode 100644 tests/core/test_permissions_injection_provider.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 15586c6f..fc624e3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **The agent now knows its own permission posture.** A live permissions-state reminder renders the enforced profile, safe-mode/yolo/auto flags, mutation/network allowances, session-approved actions, and the shell gate's command-shaping rules — re-emitted exactly when the posture changes (/yolo, /auto, /trust, new approvals) instead of the model discovering policy through denied tool calls. - **Edits recover from whitespace and smart-punctuation drift.** StrReplaceFile no longer hard-fails with "old string not found" when the only mismatch is trailing whitespace, indentation, or smart quotes/dashes: a graduated line-window ladder relocates the edit, replaces the actual file slice (preserving CRLF endings), and names the relaxation it used in the tool message. Multiple fuzzy hits without replace_all still error, so ambiguity is never silently resolved. - **`isolation="worktree"` is now enforced for background write agents.** Previously it only recorded intent, so parallel coders shared one working tree and could clobber each other. A write-profile child now runs in its own git worktree of HEAD; its final report names the worktree path with a diff summary so changes merge deliberately, clean worktrees are removed, non-git roots fail with an actionable error, and read-profile children ignore the request. - **MCP servers can be scoped to specific tools.** Optional `enabledTools` (exclusive allowlist) and `disabledTools` (denylist, wins on conflict) arrays per server in `mcp.json` keep a noisy server from flooding the model's tool list — filtered tools are never registered, and a call-time re-check guards shared tool maps. diff --git a/src/pythinker_code/soul/approval.py b/src/pythinker_code/soul/approval.py index 4b8f990b..415ca0b5 100644 --- a/src/pythinker_code/soul/approval.py +++ b/src/pythinker_code/soul/approval.py @@ -260,6 +260,14 @@ def is_runtime_auto(self) -> bool: """True only when auto mode came from this invocation.""" return self._state.runtime_auto + def is_safe_mode(self) -> bool: + """True when workspace safe mode suppresses auto-approval paths.""" + return self._state.safe_mode + + def session_approved_actions(self) -> frozenset[str]: + """Read-only view of action names approved for this session.""" + return frozenset(self._state.auto_approve_actions) + def _unattended_denial_feedback(self, action: str, tool_call: ToolCall) -> str | None: """Fail closed when an unattended run would otherwise wait for approval forever. diff --git a/src/pythinker_code/soul/dynamic_injections/permissions_state.py b/src/pythinker_code/soul/dynamic_injections/permissions_state.py new file mode 100644 index 00000000..ed2b94ef --- /dev/null +++ b/src/pythinker_code/soul/dynamic_injections/permissions_state.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from pythinker_core.message import Message + +from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider +from pythinker_code.soul.permission import PermissionProfile, permission_profile_for_runtime + +if TYPE_CHECKING: + from pythinker_code.soul.pythinkersoul import PythinkerSoul + +_INJECTION_TYPE = "permissions_state" + + +class PermissionsInjectionProvider(DynamicInjectionProvider): + """Render the live permission posture into the prompt. + + Enforcement is rich (profiles, safe mode, yolo/auto flags, session + approvals, shell-command classification) but was invisible to the model, + which discovered policy through denied tool calls. Re-injects only when + the posture fingerprint changes (covers /yolo, /auto, /trust toggles and + new session approvals), after compaction, and after auto-mode toggles. + Root-only: subagent overlays already document their profile constraints. + """ + + def __init__(self) -> None: + self._last_fingerprint: tuple[object, ...] | None = None + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + if soul.is_subagent: + return [] + profile = permission_profile_for_runtime(soul.runtime) + approval = soul.runtime.approval + approved = tuple(sorted(approval.session_approved_actions())) + fingerprint = ( + profile.name, + approval.is_yolo(), + approval.is_auto(), + approval.is_safe_mode(), + approved, + ) + if fingerprint == self._last_fingerprint: + return [] + self._last_fingerprint = fingerprint + return [ + DynamicInjection( + type=_INJECTION_TYPE, + content=_render( + profile, + approval.is_yolo(), + approval.is_auto(), + approval.is_safe_mode(), + approved, + ), + ) + ] + + async def on_context_compacted(self) -> None: + self._last_fingerprint = None + + async def on_auto_changed(self, enabled: bool) -> None: + _ = enabled + self._last_fingerprint = None + + +def _render( + profile: PermissionProfile, + yolo: bool, + auto: bool, + safe_mode: bool, + approved: tuple[str, ...], +) -> str: + def onoff(flag: bool) -> str: + return "on" if flag else "off" + + def allowed(flag: bool) -> str: + return "allowed" if flag else "denied" + + approved_text = ", ".join(approved) if approved else "none" + return ( + f"Permissions state: profile '{profile.name}' ({profile.description}). " + f"Safe mode {onoff(safe_mode)}; yolo {onoff(yolo)}; auto {onoff(auto)}. " + f"File mutation {allowed(profile.allow_file_mutation)}; shell mutation " + f"{allowed(profile.allow_shell_mutation)}; network tools " + f"{allowed(profile.allow_network)}.\n" + f"Auto-approved without prompting: provably read-only commands " + f"(ls, cat, grep, git status/log/diff, ...); session-approved actions: " + f"{approved_text}.\n" + "Command shaping: the shell gate classifies plain commands only — " + "command substitution $(...), backticks, and operators glued to words " + "are rejected as hidden commands. Write plain, separated commands so " + "the classifier can see them." + ) diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index b09799a1..595bcad9 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -84,6 +84,7 @@ from pythinker_code.soul.dynamic_injections.inline_commands import InlineCommandReminderProvider from pythinker_code.soul.dynamic_injections.model_defense import ModelDefenseInjectionProvider from pythinker_code.soul.dynamic_injections.orchestration import OrchestrationInjectionProvider +from pythinker_code.soul.dynamic_injections.permissions_state import PermissionsInjectionProvider from pythinker_code.soul.dynamic_injections.plan_mode import PlanModeInjectionProvider from pythinker_code.soul.flow_runner import FLOW_COMMAND_PREFIX, FlowRunner from pythinker_code.soul.message import ( @@ -415,6 +416,9 @@ def __init__( # Self-filtering: root-only; nudges substantial normal-mode tasks toward # direct tools, todos, RunAgents, and verification. OrchestrationInjectionProvider(), + # Self-filtering: root-only; posture-fingerprinted so it re-emits + # exactly when yolo/auto/safe-mode/profile/session-approvals change. + PermissionsInjectionProvider(), *( [] if self._runtime.config.skip_auto_prompt_injection diff --git a/tests/core/test_permissions_injection_provider.py b/tests/core/test_permissions_injection_provider.py new file mode 100644 index 00000000..4aaf42f2 --- /dev/null +++ b/tests/core/test_permissions_injection_provider.py @@ -0,0 +1,112 @@ +"""Live permissions-state injection. + +The model used to discover policy through denied tool calls; the provider +renders the enforced posture once per state change. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from pythinker_code.soul.approval import Approval, ApprovalState +from pythinker_code.soul.dynamic_injections.permissions_state import PermissionsInjectionProvider + + +def _make_soul( + *, + is_subagent: bool = False, + yolo: bool = False, + auto: bool = False, + safe_mode: bool = False, + approved: set[str] | None = None, +) -> MagicMock: + soul = MagicMock() + soul.is_subagent = is_subagent + runtime = soul.runtime + runtime.role = "root" + runtime.subagent_type = None + runtime.session.state.plan_mode = False + runtime.config.agent_execution_profile = "default" + runtime.approval = Approval( + state=ApprovalState( + yolo=yolo, auto=auto, safe_mode=safe_mode, auto_approve_actions=approved or set() + ) + ) + return soul + + +class TestPermissionsInjectionProvider: + async def test_injects_posture_on_first_step(self) -> None: + provider = PermissionsInjectionProvider() + + result = await provider.get_injections([], _make_soul(safe_mode=True)) + + assert len(result) == 1 + content = result[0].content + assert "profile 'implement'" in content + assert "Safe mode on" in content + assert "yolo off" in content + assert "Command shaping" in content + + async def test_does_not_reinject_while_posture_unchanged(self) -> None: + provider = PermissionsInjectionProvider() + soul = _make_soul() + + assert len(await provider.get_injections([], soul)) == 1 + assert await provider.get_injections([], soul) == [] + + async def test_reinjects_when_yolo_toggles(self) -> None: + provider = PermissionsInjectionProvider() + soul = _make_soul() + await provider.get_injections([], soul) + + soul.runtime.approval.set_yolo(True) + result = await provider.get_injections([], soul) + + assert len(result) == 1 + assert "yolo on" in result[0].content + + async def test_reinjects_when_session_approval_granted(self) -> None: + provider = PermissionsInjectionProvider() + soul = _make_soul() + await provider.get_injections([], soul) + + soul.runtime.approval.session_approved_actions() # no-op read + soul = _make_soul(approved={"run command"}) + provider_result = await provider.get_injections([], soul) + + assert len(provider_result) == 1 + assert "run command" in provider_result[0].content + + async def test_compaction_rearms(self) -> None: + provider = PermissionsInjectionProvider() + soul = _make_soul() + await provider.get_injections([], soul) + + await provider.on_context_compacted() + + assert len(await provider.get_injections([], soul)) == 1 + + async def test_auto_toggle_rearms(self) -> None: + provider = PermissionsInjectionProvider() + soul = _make_soul() + await provider.get_injections([], soul) + + await provider.on_auto_changed(True) + + assert len(await provider.get_injections([], soul)) == 1 + + async def test_subagents_are_excluded(self) -> None: + provider = PermissionsInjectionProvider() + + assert await provider.get_injections([], _make_soul(is_subagent=True)) == [] + + async def test_plan_mode_renders_plan_profile(self) -> None: + provider = PermissionsInjectionProvider() + soul = _make_soul() + soul.runtime.session.state.plan_mode = True + + result = await provider.get_injections([], soul) + + assert "profile 'plan'" in result[0].content + assert "File mutation denied" in result[0].content diff --git a/tests/core/test_plan_mode.py b/tests/core/test_plan_mode.py index 4a731baa..60a282a5 100644 --- a/tests/core/test_plan_mode.py +++ b/tests/core/test_plan_mode.py @@ -237,9 +237,9 @@ async def test_manual_toggle_defers_activation_to_injection( injections = await soul._collect_injections() - assert len(injections) == 1 - assert injections[0].type == "plan_mode" - assert "Plan mode is active." in injections[0].content + plan_injections = [i for i in injections if i.type.startswith("plan_mode")] + assert len(plan_injections) == 1 + assert "Plan mode is active." in plan_injections[0].content assert soul._pending_plan_activation_injection is False assert soul.context.history == [] @@ -260,7 +260,7 @@ async def test_manual_exit_clears_pending_activation_injection( assert soul._pending_plan_activation_injection is False injections = await soul._collect_injections() - assert injections == [] + assert [i for i in injections if i.type.startswith("plan_mode")] == [] async def test_tool_toggle_does_not_queue_manual_activation_injection( self, diff --git a/tests/core/test_pythinkersoul_ralph_loop.py b/tests/core/test_pythinkersoul_ralph_loop.py index 97cb0695..2347f87e 100644 --- a/tests/core/test_pythinkersoul_ralph_loop.py +++ b/tests/core/test_pythinkersoul_ralph_loop.py @@ -216,6 +216,20 @@ async def test_ralph_loop_replays_original_prompt(runtime: Runtime, tmp_path: Pa ), ], ), + Message( + role="user", + content=[ + TextPart( + text="""\ +<system-reminder> +Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. +Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. +Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. +</system-reminder>\ +""" + ) + ], + ), Message(role="assistant", content=[TextPart(text="first")]), Message( role="user", @@ -284,6 +298,20 @@ async def test_ralph_loop_stops_on_choice(runtime: Runtime, tmp_path: Path) -> N TextPart(text="do it"), ], ), + Message( + role="user", + content=[ + TextPart( + text="""\ +<system-reminder> +Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. +Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. +Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. +</system-reminder>\ +""" + ) + ], + ), Message(role="assistant", content=[TextPart(text="first")]), Message( role="user", @@ -337,6 +365,20 @@ async def test_ralph_loop_stops_on_tool_rejected(runtime: Runtime, tmp_path: Pat TextPart(text="do it"), ], ), + Message( + role="user", + content=[ + TextPart( + text="""\ +<system-reminder> +Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. +Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. +Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. +</system-reminder>\ +""" + ) + ], + ), Message( role="assistant", content=[], @@ -380,6 +422,20 @@ async def test_ralph_loop_disabled_skips_loop_prompt(runtime: Runtime, tmp_path: snapshot( [ Message(role="user", content=[TextPart(text="hello")]), + Message( + role="user", + content=[ + TextPart( + text="""\ +<system-reminder> +Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. +Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. +Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. +</system-reminder>\ +""" + ) + ], + ), Message(role="assistant", content=[TextPart(text="done")]), ] ), diff --git a/tests/core/test_pythinkersoul_steer.py b/tests/core/test_pythinkersoul_steer.py index b0e193d6..83a1fd10 100644 --- a/tests/core/test_pythinkersoul_steer.py +++ b/tests/core/test_pythinkersoul_steer.py @@ -136,9 +136,12 @@ async def test_consume_pending_steers_appends_history_before_emitting_wire_event sent: list[SteerInput] = [] def fake_wire_send(msg) -> None: - assert soul.context.history == [ - Message(role="user", content=[TextPart(text="Follow up now.")]) + persisted = [ + m + for m in soul.context.history + if not (m.role == "user" and "Permissions state:" in m.extract_text(" ")) ] + assert persisted == [Message(role="user", content=[TextPart(text="Follow up now.")])] assert isinstance(msg, SteerInput) sent.append(msg) @@ -511,7 +514,12 @@ async def ui_loop(wire: Wire) -> None: await run_soul(soul, "original question", ui_loop, asyncio.Event()) - assert soul.context.history == [ + persisted = [ + m + for m in soul.context.history + if not (m.role == "user" and "Permissions state:" in m.extract_text(" ")) + ] + assert persisted == [ Message(role="user", content=[TextPart(text="original question")]), Message(role="assistant", content=[TextPart(text="first answer")]), Message(role="user", content=[TextPart(text="follow-up steer")]), diff --git a/tests_e2e/test_wire_sessions.py b/tests_e2e/test_wire_sessions.py index 65700b0e..b3b14727 100644 --- a/tests_e2e/test_wire_sessions.py +++ b/tests_e2e/test_wire_sessions.py @@ -152,17 +152,19 @@ def test_continue_session_appends(tmp_path) -> None: "context_after": context_after, "wire_before": wire_before, "wire_after": wire_after, - } == snapshot({"context_before": 5, "context_after": 9, "wire_before": 6, "wire_after": 11}) + } == snapshot({"context_before": 6, "context_after": 11, "wire_before": 6, "wire_after": 11}) assert _read_roles(context_file) == snapshot( [ "_system_prompt", "_checkpoint", "user", "_checkpoint", + "user", "assistant", "_checkpoint", "user", "_checkpoint", + "user", "assistant", ] ) @@ -245,7 +247,7 @@ def test_clear_context_rotates(tmp_path) -> None: ) assert rotated == snapshot(["context_1.jsonl"]) assert _read_roles(session_dir / rotated[0]) == snapshot( - ["_system_prompt", "_checkpoint", "user", "_checkpoint", "assistant"] + ["_system_prompt", "_checkpoint", "user", "_checkpoint", "user", "assistant"] ) @@ -302,8 +304,8 @@ def test_manual_compact(tmp_path) -> None: "method": "event", "type": "StatusUpdate", "payload": { - "context_usage": 1e-05, - "context_tokens": 1, + "context_usage": 0.00168, + "context_tokens": 168, "max_context_tokens": 100000, "token_usage": None, "message_id": None, From a9c9059eaf4b8d11c394b98fdb25f71a84f53a87 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 07:39:47 -0400 Subject: [PATCH 42/49] docs(tasks): record permissions-state checkpoint --- tasks/todo.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/todo.md b/tasks/todo.md index a375c619..dc8fbf23 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -34,6 +34,8 @@ recovery-aware cleanup). P3 DONE (chain verified end-to-end; descriptions state enforced semantics). Isolation item CLOSED. `eb13fdae` fuzzy edit-recovery ladder (rstrip→strip→unicode-punct line-window seek in StrReplaceFile; ambiguity contract kept). + `cde3c726` live permissions-state injection (posture-fingerprinted + provider; Approval read accessors). NEXT (Tier-1 high/M, plan order): MCP startup timeout+diagnostics; MCP per-server tool filtering; subagent context fork; workspace isolation for parallel writers; turn rollup analytics; From 44a9c88cb58bf15abce5628fdda385a008fd1360 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 09:22:37 -0400 Subject: [PATCH 43/49] fix: apply external review findings across recent checkpoints - CRITICAL: GIT_CONTEXT_AGENT_TYPES used underscored reviewer names while registered type names are dashed (code-reviewer/security-reviewer), so reviewer agents silently missed the git-context injection; names fixed and a pin added asserting every gate name is a real profile key. - Foreground isolation requests now fail fast on Agent AND RunAgents instead of warning-and-proceeding unisolated (degraded behavior was presented as authoritative); warning pin updated to the new contract. - Unknown-config-key diagnostics now also run for explicit loads (--config-file / --config text) via single-source provenance. - Failure/timeout/cancel paths name the retained isolation worktree in the task output (retention is deliberate for resume, never silent). - Best-effort prune in overflow recovery logs its failure instead of contextlib.suppress. - supports_parallel flags annotated (: bool); test helpers cleaned (fail-fast _git asserts, unused _ListingClient params dropped). --- src/pythinker_code/background/agent_runner.py | 16 +++++++++++ src/pythinker_code/config.py | 17 ++++++++++++ src/pythinker_code/soul/pythinkersoul.py | 7 ++++- src/pythinker_code/subagents/core.py | 4 ++- src/pythinker_code/tools/agent/__init__.py | 26 +++++++++++++----- src/pythinker_code/tools/file/glob.py | 2 +- src/pythinker_code/tools/file/grep_local.py | 4 +-- src/pythinker_code/tools/file/read.py | 2 +- src/pythinker_code/tools/file/read_media.py | 2 +- .../tools/mcp_resource/__init__.py | 4 +-- src/pythinker_code/tools/recall/__init__.py | 2 +- src/pythinker_code/tools/think/__init__.py | 2 +- src/pythinker_code/tools/web/fetch.py | 2 +- src/pythinker_code/tools/web/search.py | 2 +- tests/subagents/test_git_context_gate.py | 11 +++++++- tests/test_git_context.py | 3 ++- tests/tools/test_agent_tool.py | 27 +++++++++---------- tests/tools/test_mcp_tool_filter.py | 6 ++--- 18 files changed, 99 insertions(+), 40 deletions(-) diff --git a/src/pythinker_code/background/agent_runner.py b/src/pythinker_code/background/agent_runner.py index 3c257ead..ec33ac13 100644 --- a/src/pythinker_code/background/agent_runner.py +++ b/src/pythinker_code/background/agent_runner.py @@ -137,11 +137,13 @@ async def run(self) -> None: output.error( _timeout_recovery_message(timeout_s=self._timeout_s, agent_id=self._agent_id) ) + self._note_retained_worktree(output) else: # Internal timeout (e.g. aiohttp request) — treat as generic failure logger.exception("Background agent runner failed") self._finalize_safely(outcome="failed", reason=str(exc)) output.error(_failure_recovery_message(reason=str(exc), agent_id=self._agent_id)) + self._note_retained_worktree(output) except asyncio.CancelledError: self._finalize_safely(outcome="killed", reason="Stopped by TaskStop") output.stage("cancelled") @@ -156,6 +158,7 @@ async def run(self) -> None: logger.exception("Background agent runner failed") self._finalize_safely(outcome="failed", reason=str(exc)) output.error(_failure_recovery_message(reason=str(exc), agent_id=self._agent_id)) + self._note_retained_worktree(output) finally: # Whatever happens in approval cleanup below, the dict pop must # run — it is the *only* place that removes this task from @@ -272,6 +275,19 @@ async def _prepare_isolation_worktree(self, output: SubagentOutputWriter) -> Hos output.stage(f"worktree_created: {worktree}") return HostPath.unsafe_from_local_path(worktree) + def _note_retained_worktree(self, output: SubagentOutputWriter) -> None: + """Name the retained worktree on failure/timeout paths. + + Retention on failure is deliberate — resume reuses the worktree and a + post-mortem may need its state — but it must never be silent. + """ + if self._worktree_path is None: + return + output.stage( + f"worktree_retained: {self._worktree_path} (resume reuses it; remove with " + f"`git worktree remove {self._worktree_path}`)" + ) + async def _append_worktree_report(self, final_response: str) -> str: """Tell the orchestrator where the isolated changes live. diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index aab327bc..eb395d07 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -185,6 +185,21 @@ def _sole_non_none_type(annotation: Any) -> Any: return annotation +def _report_unknown_keys_single_source(data: Any, source: str) -> None: + """Unknown-key diagnostics for single-source loads (--config/--config-file). + + The scoped pipeline reports through its merge provenance; explicit loads + bypass the merge, so build a one-source provenance here. Non-dict payloads + are left for Config.model_validate to reject with its own error. + """ + if not isinstance(data, dict): + return + plain = {str(key): value for key, value in cast(dict[Any, Any], data).items()} + provenance: dict[str, Any] = {} + merged = _type_based_merge({}, plain, provenance, source) + _report_unknown_config_keys(merged, provenance) + + def _report_unknown_config_keys(merged: dict[str, Any], provenance: dict[str, Any]) -> None: """Warn (or raise under PYTHINKER_STRICT_CONFIG) for unconsumed keys.""" unknown_paths = unknown_config_key_paths(Config, merged) @@ -1217,6 +1232,7 @@ def load_config(config_file: Path | None = None) -> Config: data = json.loads(config_text) else: data = tomlkit.loads(config_text) + _report_unknown_keys_single_source(data, str(config_file)) config = Config.model_validate(data) except json.JSONDecodeError as e: raise ConfigError(f"Invalid JSON in configuration file {config_file}: {e}") from e @@ -1260,6 +1276,7 @@ def load_config_from_string(config_string: str) -> Config: f"Invalid configuration text: {json_error}; {toml_error}" ) from toml_error + _report_unknown_keys_single_source(data, "--config text") try: config = Config.model_validate(data) except ValidationError as e: diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index 595bcad9..701ccf16 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -1979,8 +1979,13 @@ async def _recover_from_context_overflow(self, step_no: int) -> bool: step_no=step_no, ) try: - with contextlib.suppress(Exception): + try: await self.prune_context() + except Exception as prune_err: + logger.debug( + "Best-effort prune during overflow recovery failed: {error}", + error=prune_err, + ) await self.compact_context() except Exception as compact_err: from pythinker_code.telemetry.errors import report_handled_error diff --git a/src/pythinker_code/subagents/core.py b/src/pythinker_code/subagents/core.py index fc0202dd..8c911a7b 100644 --- a/src/pythinker_code/subagents/core.py +++ b/src/pythinker_code/subagents/core.py @@ -25,7 +25,9 @@ from pythinker_code.subagents.store import SubagentStore from pythinker_code.wire.types import TextPart, ThinkPart -GIT_CONTEXT_AGENT_TYPES = frozenset({"explore", "review", "code_reviewer", "security_reviewer"}) +# NOTE: these must match the registered type names in agents/default/agent.yaml +# (dashed), which _SUBAGENT_PROFILES also keys on — not the yaml file stems. +GIT_CONTEXT_AGENT_TYPES = frozenset({"explore", "review", "code-reviewer", "security-reviewer"}) """Read-oriented agent types whose first prompt gets a git-context prefix. Exploration and review both orient on repo state (branch, dirty files, diff --git a/src/pythinker_code/tools/agent/__init__.py b/src/pythinker_code/tools/agent/__init__.py index 5e1592f0..414781ea 100644 --- a/src/pythinker_code/tools/agent/__init__.py +++ b/src/pythinker_code/tools/agent/__init__.py @@ -297,14 +297,18 @@ async def __call__(self, params: Params) -> ToolReturnValue: ), brief="Invalid fork_context", ) + if not params.run_in_background and params.isolation != "none": + # Proceeding unisolated after an isolation request would present + # degraded behavior as authoritative; fail fast instead. + return ToolError( + message=( + "isolation='worktree' is only supported for background agents; " + "set run_in_background=true or drop isolation." + ), + brief="Invalid isolation", + ) if params.run_in_background: return await self._run_in_background(params) - if params.isolation != "none": - logger.warning( - "isolation={isolation!r} has no effect on foreground agents; " - "use run_in_background=True to enable isolation.", - isolation=params.isolation, - ) await self._journal_foreground_agent_start(params, requested_type) timeout = params.effective_timeout try: @@ -667,6 +671,16 @@ async def __call__(self, params: RunAgentsParams) -> ToolReturnValue: message="Subagents cannot launch other subagents.", brief="RunAgents unavailable", ) + if not params.run_in_background and params.isolation != "none": + # Foreground children would share one tree despite the isolation + # request; fail fast rather than proceed unisolated. + return ToolError( + message=( + "isolation='worktree' is only supported for background child " + "agents; set run_in_background=true or drop isolation." + ), + brief="Invalid isolation", + ) if params.model is not None and params.model not in self._runtime.config.models: return ToolError( message=f"Unknown model alias: {params.model}", diff --git a/src/pythinker_code/tools/file/glob.py b/src/pythinker_code/tools/file/glob.py index b70130c4..22e742e3 100644 --- a/src/pythinker_code/tools/file/glob.py +++ b/src/pythinker_code/tools/file/glob.py @@ -31,7 +31,7 @@ class Params(BaseModel): class Glob(CallableTool2[Params]): name: str = "Glob" - supports_parallel = True + supports_parallel: bool = True description: str = load_desc( Path(__file__).parent / "glob.md", { diff --git a/src/pythinker_code/tools/file/grep_local.py b/src/pythinker_code/tools/file/grep_local.py index edf546ba..3e2f49d3 100644 --- a/src/pythinker_code/tools/file/grep_local.py +++ b/src/pythinker_code/tools/file/grep_local.py @@ -719,7 +719,7 @@ def _smart_search_patterns(query: str) -> list[tuple[str, str]]: class SmartSearch(CallableTool2[SmartSearchParams]): name: str = "SmartSearch" - supports_parallel = True + supports_parallel: bool = True description: str = ( "Plan and run a small set of bounded local grep passes for a symbol or concept. " "Returns cited file/line spans and truncation guidance; use for exploration before " @@ -789,7 +789,7 @@ async def __call__(self, params: SmartSearchParams) -> ToolReturnValue: class Grep(CallableTool2[Params]): name: str = "Grep" - supports_parallel = True + supports_parallel: bool = True description: str = load_desc(Path(__file__).parent / "grep.md") params: type[Params] = Params diff --git a/src/pythinker_code/tools/file/read.py b/src/pythinker_code/tools/file/read.py index 1264cbd8..02f95368 100644 --- a/src/pythinker_code/tools/file/read.py +++ b/src/pythinker_code/tools/file/read.py @@ -63,7 +63,7 @@ def _validate_line_offset(self) -> "Params": class ReadFile(CallableTool2[Params]): name: str = "ReadFile" - supports_parallel = True + supports_parallel: bool = True params: type[Params] = Params def __init__(self, runtime: Runtime) -> None: diff --git a/src/pythinker_code/tools/file/read_media.py b/src/pythinker_code/tools/file/read_media.py index fa30340f..405e96f1 100644 --- a/src/pythinker_code/tools/file/read_media.py +++ b/src/pythinker_code/tools/file/read_media.py @@ -49,7 +49,7 @@ class Params(BaseModel): class ReadMediaFile(CallableTool2[Params]): name: str = "ReadMediaFile" - supports_parallel = True + supports_parallel: bool = True params: type[Params] = Params def __init__(self, runtime: Runtime) -> None: diff --git a/src/pythinker_code/tools/mcp_resource/__init__.py b/src/pythinker_code/tools/mcp_resource/__init__.py index dbdda56d..fd56a72f 100644 --- a/src/pythinker_code/tools/mcp_resource/__init__.py +++ b/src/pythinker_code/tools/mcp_resource/__init__.py @@ -26,7 +26,7 @@ class ListParams(BaseModel): class ListMcpResources(CallableTool2[ListParams]): name: str = "ListMcpResources" - supports_parallel = True + supports_parallel: bool = True params: type[ListParams] = ListParams def __init__(self, toolset: PythinkerToolset) -> None: @@ -75,7 +75,7 @@ class ReadParams(BaseModel): class ReadMcpResource(CallableTool2[ReadParams]): name: str = "ReadMcpResource" - supports_parallel = True + supports_parallel: bool = True params: type[ReadParams] = ReadParams def __init__(self, toolset: PythinkerToolset) -> None: diff --git a/src/pythinker_code/tools/recall/__init__.py b/src/pythinker_code/tools/recall/__init__.py index 316cd497..69dcd949 100644 --- a/src/pythinker_code/tools/recall/__init__.py +++ b/src/pythinker_code/tools/recall/__init__.py @@ -111,7 +111,7 @@ def _render_transcript(context_file: Path, budget: int) -> str: class Recall(CallableTool2[Params]): name: str = NAME - supports_parallel = True + supports_parallel: bool = True params: type[Params] = Params def __init__(self, runtime: Runtime): diff --git a/src/pythinker_code/tools/think/__init__.py b/src/pythinker_code/tools/think/__init__.py index d2ddeb23..3b2abb45 100644 --- a/src/pythinker_code/tools/think/__init__.py +++ b/src/pythinker_code/tools/think/__init__.py @@ -13,7 +13,7 @@ class Params(BaseModel): class Think(CallableTool2[Params]): name: str = "Think" - supports_parallel = True + supports_parallel: bool = True description: str = load_desc(Path(__file__).parent / "think.md", {}) params: type[Params] = Params diff --git a/src/pythinker_code/tools/web/fetch.py b/src/pythinker_code/tools/web/fetch.py index eda83848..302fdb2c 100644 --- a/src/pythinker_code/tools/web/fetch.py +++ b/src/pythinker_code/tools/web/fetch.py @@ -130,7 +130,7 @@ class Params(BaseModel): class FetchURL(CallableTool2[Params]): name: str = "FetchURL" - supports_parallel = True + supports_parallel: bool = True description: str = load_desc(Path(__file__).parent / "fetch.md", {}) params: type[Params] = Params diff --git a/src/pythinker_code/tools/web/search.py b/src/pythinker_code/tools/web/search.py index 9c0fe8dd..f9fc78b4 100644 --- a/src/pythinker_code/tools/web/search.py +++ b/src/pythinker_code/tools/web/search.py @@ -43,7 +43,7 @@ class Params(BaseModel): class SearchWeb(CallableTool2[Params]): name: str = "SearchWeb" - supports_parallel = True + supports_parallel: bool = True description: str = load_desc(Path(__file__).parent / "search.md", {}) params: type[Params] = Params diff --git a/tests/subagents/test_git_context_gate.py b/tests/subagents/test_git_context_gate.py index 0772c8d2..a069df06 100644 --- a/tests/subagents/test_git_context_gate.py +++ b/tests/subagents/test_git_context_gate.py @@ -11,7 +11,16 @@ def test_explore_and_reviewer_types_receive_git_context() -> None: - assert {"explore", "review", "code_reviewer", "security_reviewer"} <= GIT_CONTEXT_AGENT_TYPES + assert {"explore", "review", "code-reviewer", "security-reviewer"} <= GIT_CONTEXT_AGENT_TYPES + + +def test_gate_names_match_registered_profile_keys() -> None: + """The gate is keyed on spec.type_def.name; a name that is not also a + profile key would silently never match a real agent type.""" + from pythinker_code.soul.permission import _SUBAGENT_PROFILES + + unmatched = GIT_CONTEXT_AGENT_TYPES - set(_SUBAGENT_PROFILES) + assert unmatched == set(), unmatched def test_write_capable_types_do_not() -> None: diff --git a/tests/test_git_context.py b/tests/test_git_context.py index f3d682eb..afcdfb4e 100644 --- a/tests/test_git_context.py +++ b/tests/test_git_context.py @@ -346,7 +346,8 @@ async def _git(cwd: Path, *args: str) -> str: stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, _ = await proc.communicate() + stdout, stderr = await proc.communicate() + assert proc.returncode == 0, f"git {' '.join(args)} failed: {stderr.decode().strip()}" return stdout.decode().strip() diff --git a/tests/tools/test_agent_tool.py b/tests/tools/test_agent_tool.py index 4da76270..eb31a174 100644 --- a/tests/tools/test_agent_tool.py +++ b/tests/tools/test_agent_tool.py @@ -3,7 +3,6 @@ import asyncio import re from types import SimpleNamespace -from unittest.mock import patch import pytest from pythinker_core.chat_provider import APIConnectionError, APIStatusError, ChatProviderError @@ -2318,8 +2317,9 @@ def fake_create_agent_task(**kwargs): # --------------------------------------------------------------------------- -async def test_agent_tool_warns_when_isolation_set_for_foreground(agent_tool, runtime, monkeypatch): - """isolation='worktree' on a foreground agent logs a warning (isolation only applies to background).""" +async def test_agent_tool_rejects_isolation_for_foreground(agent_tool, runtime, monkeypatch): + """isolation='worktree' on a foreground agent fails fast — proceeding + unisolated would present degraded behavior as authoritative.""" runtime.labor_market.add_builtin_type( AgentTypeDefinition( name="coder", @@ -2344,20 +2344,17 @@ async def fake_run_soul( monkeypatch.setattr("pythinker_code.subagents.builder.load_agent", fake_load_agent) monkeypatch.setattr("pythinker_code.subagents.runner.run_soul", fake_run_soul) - with patch("pythinker_code.tools.agent.logger") as mock_log: - result = await agent_tool( - agent_tool.params( - description="task", - prompt="do it", - run_in_background=False, - isolation="worktree", - ) + result = await agent_tool( + agent_tool.params( + description="task", + prompt="do it", + run_in_background=False, + isolation="worktree", ) + ) - assert not result.is_error - mock_log.warning.assert_called_once() - warning_msg = str(mock_log.warning.call_args) - assert "isolation" in warning_msg.lower() + assert result.is_error + assert "run_in_background" in result.message # --------------------------------------------------------------------------- diff --git a/tests/tools/test_mcp_tool_filter.py b/tests/tools/test_mcp_tool_filter.py index 77ed1ee8..656c5a72 100644 --- a/tests/tools/test_mcp_tool_filter.py +++ b/tests/tools/test_mcp_tool_filter.py @@ -22,10 +22,8 @@ class _ListingClient: - def __init__(self, tool_names: list[str], *, fail_on_call: bool = False) -> None: + def __init__(self, tool_names: list[str]) -> None: self._tool_names = tool_names - self.calls: list[str] = [] - self._fail_on_call = fail_on_call async def __aenter__(self) -> _ListingClient: return self @@ -116,7 +114,7 @@ async def test_denied_tool_errors_without_reaching_server(self, runtime) -> None tool = MCPTool( "srv", mcp.types.Tool(name="drop_db", inputSchema={}), - cast(Any, _ListingClient([], fail_on_call=True)), + cast(Any, _ListingClient([])), runtime=runtime, tool_filter=McpToolFilter(deny=frozenset({"drop_db"})), ) From ee75926b37566dfda4bf5ab13250e33e612810ea Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 09:48:16 -0400 Subject: [PATCH 44/49] fix: harden isolation, elision, and concurrency per adversarial arc review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 16-agent adversarial review (4 dimensions, every finding refuted-or- confirmed against live code) confirmed 11 findings; all fixed except one deliberate deferral (exclusive gate held across approval waits — needs the approval-split refactor; recorded in tasks/todo.md). - DATA LOSS (high): a child that committed its work left a clean worktree, so cleanup removed it and orphaned the commits. Creation now records a base-SHA sidecar (next to the worktree, never inside it); commits ahead of base count as changes and force retention, with unknown provenance failing closed to retention. - FALSE ISOLATION (high): foreground shell inherited the process cwd and relative file-tool paths resolved against it, so isolated children mutated the original repo. Host exec (protocol, local, ssh, ACP fallback) gained a cwd argument; foreground shell passes the runtime work dir, and write/replace/read resolve relative paths against it while preserving the relative-escape error contract. - REGRESSION (high): safe mode now disables the read-only-command prompt elision — users who disabled auto-approval keep every checkpoint. - REGRESSION (high): untrusted-project hook stripping now publishes a session notification (web/ACP visible), not just a shell log line. - MCP readOnlyHint annotations enable supports_parallel via property; worktree add/remove serializes per repo; CHANGELOG documents the same-step serialization and stderr-diagnostics behavior changes. --- CHANGELOG.md | 1 + .../src/pythinker_host/__init__.py | 12 ++- .../src/pythinker_host/local.py | 5 +- .../pythinker-host/src/pythinker_host/ssh.py | 9 ++- src/pythinker_code/acp/host.py | 6 +- src/pythinker_code/app.py | 24 ++++++ src/pythinker_code/config.py | 14 ++++ src/pythinker_code/soul/toolset.py | 10 +++ src/pythinker_code/subagents/worktree.py | 73 ++++++++++++++++--- src/pythinker_code/tools/file/read.py | 8 +- src/pythinker_code/tools/file/replace.py | 8 +- src/pythinker_code/tools/file/write.py | 8 +- src/pythinker_code/tools/shell/__init__.py | 14 +++- tasks/todo.md | 7 ++ tests/core/test_safe_command_elision.py | 22 ++++++ tests/subagents/test_worktree_isolation.py | 33 +++++++++ tests/tools/test_mcp_tool_filter.py | 26 +++++++ tests/tools/test_write_file.py | 21 ++++++ 18 files changed, 274 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc624e3a..df4f26fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- **Adversarial branch review hardening.** A multi-agent review pass confirmed and fixed: committed-but-clean isolation worktrees are now retained (commits ahead of the creation base count as changes — they were previously orphaned on cleanup); foreground shell commands and relative-path file edits now resolve against the agent's work dir, so worktree isolation actually binds them (host exec gained a `cwd` argument); safe mode now also disables the read-only-command prompt elision; MCP tools annotated `readOnlyHint` run in parallel in the same-step gate; worktree add/remove serializes per repo. Behavior notes: same-step tool calls without `supports_parallel` now serialize deterministically (previously fully concurrent), and text-mode error diagnostics moved to stderr — capture `2>&1` or use `--output-format stream-json` if you scraped stdout. - **The agent now knows its own permission posture.** A live permissions-state reminder renders the enforced profile, safe-mode/yolo/auto flags, mutation/network allowances, session-approved actions, and the shell gate's command-shaping rules — re-emitted exactly when the posture changes (/yolo, /auto, /trust, new approvals) instead of the model discovering policy through denied tool calls. - **Edits recover from whitespace and smart-punctuation drift.** StrReplaceFile no longer hard-fails with "old string not found" when the only mismatch is trailing whitespace, indentation, or smart quotes/dashes: a graduated line-window ladder relocates the edit, replaces the actual file slice (preserving CRLF endings), and names the relaxation it used in the tool message. Multiple fuzzy hits without replace_all still error, so ambiguity is never silently resolved. - **`isolation="worktree"` is now enforced for background write agents.** Previously it only recorded intent, so parallel coders shared one working tree and could clobber each other. A write-profile child now runs in its own git worktree of HEAD; its final report names the worktree path with a diff summary so changes merge deliberately, clean worktrees are removed, non-git roots fail with an actionable error, and read-profile children ignore the request. diff --git a/packages/pythinker-host/src/pythinker_host/__init__.py b/packages/pythinker-host/src/pythinker_host/__init__.py index 94151a07..d63ab5ce 100644 --- a/packages/pythinker-host/src/pythinker_host/__init__.py +++ b/packages/pythinker-host/src/pythinker_host/__init__.py @@ -220,7 +220,9 @@ async def mkdir( """Create a directory at the given path.""" ... - async def exec(self, *args: str, env: Mapping[str, str] | None = None) -> HostProcess: + async def exec( + self, *args: str, env: Mapping[str, str] | None = None, cwd: str | None = None + ) -> HostProcess: """ Execute a command with arguments and return the running process. @@ -228,6 +230,8 @@ async def exec(self, *args: str, env: Mapping[str, str] | None = None) -> HostPr *args: Command and its arguments. env: Environment variables for the subprocess. If None, inherits from the parent process. + cwd: Working directory for the subprocess. If None, inherits the + backend's current working directory (process cwd locally). """ ... @@ -347,8 +351,10 @@ async def mkdir(path: StrOrHostPath, parents: bool = False, exist_ok: bool = Fal return await get_current_host().mkdir(path, parents=parents, exist_ok=exist_ok) -async def exec(*args: str, env: Mapping[str, str] | None = None) -> HostProcess: - return await get_current_host().exec(*args, env=env) +async def exec( + *args: str, env: Mapping[str, str] | None = None, cwd: str | None = None +) -> HostProcess: + return await get_current_host().exec(*args, env=env, cwd=cwd) from pythinker_host._current import current_host as current_host # noqa: E402 diff --git a/packages/pythinker-host/src/pythinker_host/local.py b/packages/pythinker-host/src/pythinker_host/local.py index d48e1fd6..3dee3358 100644 --- a/packages/pythinker-host/src/pythinker_host/local.py +++ b/packages/pythinker-host/src/pythinker_host/local.py @@ -190,7 +190,9 @@ async def mkdir( local_path = path.unsafe_to_local_path() if isinstance(path, HostPath) else Path(path) await asyncio.to_thread(local_path.mkdir, parents=parents, exist_ok=exist_ok) - async def exec(self, *args: str, env: Mapping[str, str] | None = None) -> HostProcess: + async def exec( + self, *args: str, env: Mapping[str, str] | None = None, cwd: str | None = None + ) -> HostProcess: if not args: raise ValueError("At least one argument (the program to execute) is required.") @@ -208,6 +210,7 @@ async def exec(self, *args: str, env: Mapping[str, str] | None = None) -> HostPr stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env, + cwd=cwd, **process_options, ) return self.Process(process) diff --git a/packages/pythinker-host/src/pythinker_host/ssh.py b/packages/pythinker-host/src/pythinker_host/ssh.py index d2b1a2f4..809c7576 100644 --- a/packages/pythinker-host/src/pythinker_host/ssh.py +++ b/packages/pythinker-host/src/pythinker_host/ssh.py @@ -303,7 +303,9 @@ async def mkdir( raise FileExistsError(f"{path} already exists") await self._sftp.mkdir(str(path)) - async def exec(self, *args: str, env: Mapping[str, str] | None = None) -> HostProcess: + async def exec( + self, *args: str, env: Mapping[str, str] | None = None, cwd: str | None = None + ) -> HostProcess: if not args: raise ValueError("At least one argument (the program to execute) is required.") command = " ".join(shlex.quote(arg) for arg in args) @@ -313,8 +315,9 @@ async def exec(self, *args: str, env: Mapping[str, str] | None = None) -> HostPr # cwd before running the command. # # This is intentionally strict: if cwd doesn't exist, the command fails. - if self._cwd: - command = f"cd {shlex.quote(self._cwd)} && {command}" + effective_cwd = cwd or self._cwd + if effective_cwd: + command = f"cd {shlex.quote(effective_cwd)} && {command}" process = await self._connection.create_process(command, encoding=None, env=env) return self.Process(process) diff --git a/src/pythinker_code/acp/host.py b/src/pythinker_code/acp/host.py index 423d00a8..81508d68 100644 --- a/src/pythinker_code/acp/host.py +++ b/src/pythinker_code/acp/host.py @@ -296,8 +296,10 @@ async def mkdir( ) -> None: await self._fallback.mkdir(path, parents=parents, exist_ok=exist_ok) - async def exec(self, *args: str, env: Mapping[str, str] | None = None) -> HostProcess: - return await self._fallback.exec(*args, env=env) + async def exec( + self, *args: str, env: Mapping[str, str] | None = None, cwd: str | None = None + ) -> HostProcess: + return await self._fallback.exec(*args, env=env, cwd=cwd) def _abs_path(self, path: StrOrHostPath) -> str: host_path = path if isinstance(path, HostPath) else HostPath(path) diff --git a/src/pythinker_code/app.py b/src/pythinker_code/app.py index a5f9774d..5e9f1090 100644 --- a/src/pythinker_code/app.py +++ b/src/pythinker_code/app.py @@ -388,6 +388,30 @@ async def create( from pythinker_code.hooks.engine import HookEngine hook_engine = HookEngine(config.hooks, cwd=str(session.work_dir)) + if config.disabled_project_hooks: + # The load-time logger.warning only reaches shell users; publish a + # notification so web/ACP frontends also learn why their project + # hooks did not run and how to enable them. + from pythinker_code.notifications.models import NotificationEvent + + runtime.notifications.publish( + NotificationEvent( + id=f"project-hooks-disabled:{session.id}", + category="system", + type="project_hooks_disabled", + source_kind="config", + source_id="project_trust", + title="Project hooks disabled (untrusted project)", + body=( + "Hooks defined in " + + ", ".join(config.disabled_project_hooks) + + " are disabled until you trust this project. Run /trust to " + "enable them (takes effect on /reload or next start)." + ), + severity="warning", + dedupe_key=f"project-hooks-disabled:{session.id}", + ) + ) soul.set_hook_engine(hook_engine) runtime.hook_engine = hook_engine diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index eb395d07..0ef9564e 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -379,6 +379,8 @@ def _read_toml(path: Path) -> dict[str, Any]: local_file: Path | None = None project_dict: dict[str, Any] = {} local_dict: dict[str, Any] = {} + project_trusted = True + stripped_hook_files: list[str] = [] if project_root is not None: from pythinker_code.project_trust import is_project_trusted @@ -406,6 +408,7 @@ def _read_toml(path: Path) -> dict[str, Any]: local_dict = {} for scope_dict, scope_file in ((project_dict, project_file), (local_dict, local_file)): if scope_dict.pop("hooks", None) is not None: + stripped_hook_files.append(str(scope_file)) logger.warning( "Project hooks in {file} are disabled until the project is " "trusted; run /trust to enable them", @@ -444,6 +447,8 @@ def _read_toml(path: Path) -> dict[str, Any]: raise ConfigError("Invalid configuration:\n" + "\n".join(enriched)) from exc # ── METADATA ────────────────────────────────────────────────────────── + if project_root is not None and not project_trusted: + config.disabled_project_hooks = stripped_hook_files config.is_from_default_location = True config.source_file = user_file if user_file.exists(): @@ -1103,6 +1108,15 @@ class Config(BaseModel): mcp: MCPConfig = Field(default_factory=MCPConfig, description="MCP configuration") tui: TUIConfig = Field(default_factory=TUIConfig, description="TUI rendering configuration") hooks: list[HookDef] = Field(default_factory=list, description="Hook definitions") # pyright: ignore[reportUnknownVariableType] + disabled_project_hooks: list[str] = Field( + default_factory=list, + exclude=True, + description=( + "Config files whose project-scope hooks were stripped because the " + "project is untrusted. Populated at load; surfaced as a " + "notification so non-shell frontends see it too." + ), + ) merge_all_available_skills: bool = Field( default=True, description=( diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index a0947fb5..25967df2 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -1296,6 +1296,16 @@ def mcp_server_name(self) -> str: """Name of the MCP server this tool belongs to.""" return self._mcp_server_name + @property + def supports_parallel(self) -> bool: + """Honor the MCP readOnlyHint annotation in the same-step gate. + + Read-only server tools (doc/resource lookups) may overlap instead of + serializing; anything unannotated stays exclusive (safe default). + """ + annotations = getattr(self._mcp_tool, "annotations", None) + return bool(getattr(annotations, "readOnlyHint", False)) + async def __call__(self, *args: Any, **kwargs: Any) -> ToolReturnValue: # Call-time re-check of the list-time filter: defense in depth for # tool maps shared across agents (e.g. runtime.mcp_tools handed to diff --git a/src/pythinker_code/subagents/worktree.py b/src/pythinker_code/subagents/worktree.py index 5447d8e2..51efa88a 100644 --- a/src/pythinker_code/subagents/worktree.py +++ b/src/pythinker_code/subagents/worktree.py @@ -9,11 +9,21 @@ from __future__ import annotations import asyncio +from collections import defaultdict from pathlib import Path from pythinker_code.utils.logging import logger _GIT_TIMEOUT_S = 30.0 +# Serialize worktree add/remove per repo: git's internal locking is reliable +# on current versions, but concurrent isolated agents should not depend on it. +_REPO_LOCKS: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock) + + +def _base_sha_file(worktree: Path) -> Path: + # Sidecar NEXT TO the worktree, never inside it — an untracked file inside + # would make every clean worktree look dirty. + return worktree.parent / f"{worktree.name}.base-sha" class WorktreeError(Exception): @@ -58,21 +68,58 @@ async def create_agent_worktree(repo_dir: Path, dest: Path) -> None: # Resume of an isolated agent reuses its existing worktree. return dest.parent.mkdir(parents=True, exist_ok=True) - code, _, stderr = await _git(["worktree", "add", "--detach", str(dest), "HEAD"], repo_dir) - if code != 0: - first_line = stderr.splitlines()[0] if stderr else "unknown git error" - raise WorktreeError(f"could not create isolation worktree at {dest}: {first_line}") + async with _REPO_LOCKS[str(repo_dir)]: + code, _, stderr = await _git(["worktree", "add", "--detach", str(dest), "HEAD"], repo_dir) + if code != 0: + first_line = stderr.splitlines()[0] if stderr else "unknown git error" + raise WorktreeError(f"could not create isolation worktree at {dest}: {first_line}") + # Record the creation base so committed-but-clean child work is detected + # later; commits ahead of this SHA must never be silently removed. + code, base_sha, _ = await _git(["rev-parse", "HEAD"], dest) + if code == 0 and base_sha: + _base_sha_file(dest).write_text(base_sha + "\n", encoding="utf-8") async def worktree_change_summary(worktree: Path) -> str: - """Short human summary of changes in *worktree*; empty string when clean.""" + """Short human summary of changes in *worktree*; empty string when clean. + + "Changes" includes commits the child made on its detached HEAD: a child + that commits its work leaves a clean working tree, and `worktree remove` + would orphan those commits as dangling objects. + """ + parts: list[str] = [] + commits_ahead = await _commits_ahead_of_base(worktree) + if commits_ahead: + parts.append(f"{commits_ahead} commit(s) ahead of the creation base") code, status, _ = await _git(["status", "--porcelain"], worktree) - if code != 0 or not status: - return "" - _, diff_stat, _ = await _git(["diff", "--stat", "HEAD"], worktree) - untracked = sum(1 for line in status.splitlines() if line.startswith("??")) - parts = [part for part in (diff_stat, f"{untracked} untracked file(s)" if untracked else "")] - return "\n".join(part for part in parts if part) + if code == 0 and status: + _, diff_stat, _ = await _git(["diff", "--stat", "HEAD"], worktree) + if diff_stat: + parts.append(diff_stat) + untracked = sum(1 for line in status.splitlines() if line.startswith("??")) + if untracked: + parts.append(f"{untracked} untracked file(s)") + return "\n".join(parts) + + +async def _commits_ahead_of_base(worktree: Path) -> int: + """Commits on the worktree's detached HEAD since creation. + + Missing or unreadable sidecar fails CLOSED (pretend one commit exists) + when HEAD cannot be compared — losing work is the only unacceptable + outcome, so unknown provenance means retain. + """ + sidecar = _base_sha_file(worktree) + try: + base_sha = sidecar.read_text(encoding="utf-8").strip() + except OSError: + base_sha = "" + if not base_sha: + return 1 # unknown provenance — retain + code, count, _ = await _git(["rev-list", "--count", f"{base_sha}..HEAD"], worktree) + if code != 0: + return 1 + return int(count or 0) async def cleanup_agent_worktree(repo_dir: Path, worktree: Path, *, has_changes: bool) -> str: @@ -84,10 +131,12 @@ async def cleanup_agent_worktree(repo_dir: Path, worktree: Path, *, has_changes: """ if has_changes: return "retained" - code, _, stderr = await _git(["worktree", "remove", str(worktree)], repo_dir) + async with _REPO_LOCKS[str(repo_dir)]: + code, _, stderr = await _git(["worktree", "remove", str(worktree)], repo_dir) if code != 0: logger.warning( "Could not remove clean isolation worktree {wt}: {err}", wt=worktree, err=stderr ) return "retained" + _base_sha_file(worktree).unlink(missing_ok=True) return "removed" diff --git a/src/pythinker_code/tools/file/read.py b/src/pythinker_code/tools/file/read.py index 02f95368..24525e6e 100644 --- a/src/pythinker_code/tools/file/read.py +++ b/src/pythinker_code/tools/file/read.py @@ -111,7 +111,13 @@ async def __call__(self, params: Params) -> ToolReturnValue: try: raw = HostPath(params.path).expanduser() - p = raw.canonical() + # Relative tool paths resolve against the runtime work dir + # (override-aware), NOT the process cwd — an isolated child's + # relative write must land in its worktree. `raw` keeps the + # original form: the workspace-escape rule for relative paths + # checks (and reports) what the caller actually passed. + base_joined = raw if raw.is_absolute() else self._work_dir.joinpath(str(raw)) + p = base_joined.canonical() # Resolve the real (symlink-followed) path for security checks only. # os.path.realpath follows symlinks at every component including the leaf, diff --git a/src/pythinker_code/tools/file/replace.py b/src/pythinker_code/tools/file/replace.py index 21b1d361..49c8a987 100644 --- a/src/pythinker_code/tools/file/replace.py +++ b/src/pythinker_code/tools/file/replace.py @@ -286,7 +286,13 @@ async def __call__(self, params: Params) -> ToolReturnValue: try: raw = HostPath(params.path).expanduser() - p = raw.canonical() + # Relative tool paths resolve against the runtime work dir + # (override-aware), NOT the process cwd — an isolated child's + # relative write must land in its worktree. `raw` keeps the + # original form: the workspace-escape rule for relative paths + # checks (and reports) what the caller actually passed. + base_joined = raw if raw.is_absolute() else self._work_dir.joinpath(str(raw)) + p = base_joined.canonical() # Resolve the real (symlink-followed) path for security checks only. # os.path.realpath follows symlinks at every component including the leaf. diff --git a/src/pythinker_code/tools/file/write.py b/src/pythinker_code/tools/file/write.py index f291e9ac..1b49912b 100644 --- a/src/pythinker_code/tools/file/write.py +++ b/src/pythinker_code/tools/file/write.py @@ -97,7 +97,13 @@ async def __call__(self, params: Params) -> ToolReturnValue: try: raw = HostPath(params.path).expanduser() - p = raw.canonical() + # Relative tool paths resolve against the runtime work dir + # (override-aware), NOT the process cwd — an isolated child's + # relative write must land in its worktree. `raw` keeps the + # original form: the workspace-escape rule for relative paths + # checks (and reports) what the caller actually passed. + base_joined = raw if raw.is_absolute() else self._work_dir.joinpath(str(raw)) + p = base_joined.canonical() # Resolve the real (symlink-followed) path for security checks only. # os.path.realpath follows symlinks at every component including the leaf. diff --git a/src/pythinker_code/tools/shell/__init__.py b/src/pythinker_code/tools/shell/__init__.py index 3e0e3e1e..23b2e482 100644 --- a/src/pythinker_code/tools/shell/__init__.py +++ b/src/pythinker_code/tools/shell/__init__.py @@ -156,9 +156,15 @@ async def __call__(self, params: Params) -> ToolReturnValue: if params.run_in_background: return await self._run_in_background(params, scrub_secrets=restricted_profile) - if self._runtime.role == "root" and is_known_safe_command(params.command): + if ( + self._runtime.role == "root" + and not self._approval.is_safe_mode() + and is_known_safe_command(params.command) + ): # Provably read-only — elide the approval prompt for the root - # agent, where prompt fatigue hits the human. Subagents keep the + # agent, where prompt fatigue hits the human. Safe mode keeps every + # prompt: a user who explicitly disabled auto-approval depends on + # them as checkpoints. Subagents keep the # request: their approval path is part of the unattended-denial # defense surface (mutation parsing is best-effort there). The # deny-path gate (check_shell_command_allowed) already ran above, @@ -381,7 +387,9 @@ async def _read_stream(stream: AsyncReadable, cb: Callable[[bytes], None]): env = get_noninteractive_env() if scrub_secrets: env = scrub_secret_env(env) - process = await pythinker_host.exec(*self._shell_args(command), env=env) + process = await pythinker_host.exec( + *self._shell_args(command), env=env, cwd=str(self._runtime.work_dir) + ) # Close stdin immediately so interactive prompts (e.g. git password) get # EOF instead of hanging forever waiting for input that will never come. diff --git a/tasks/todo.md b/tasks/todo.md index dc8fbf23..169a5e64 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -109,6 +109,13 @@ Done: `mythos-enhancements` PR #118 merged (d51ef649). ### Deferred (documented, not silently dropped) +- Arc-review finding (medium, deliberate deferral): the same-step exclusive + gate is held across a mutating tool's approval wait, blocking sibling + parallel-safe reads in that batch. Proper fix = split approval from + execution per tool (approval ungated, only the mutation gated) — an + invasive per-tool refactor; the cost today is bounded (elision/auto modes + remove most waits). Revisit with the approval-split refactor. + - Read-only MCP doc-lookup carve-out for offline roles — today ALL MCP tools are fail-closed below the implement profile (deliberate); reviewer specs route doc needs to the parent/`scout` instead. Revisit only with a real diff --git a/tests/core/test_safe_command_elision.py b/tests/core/test_safe_command_elision.py index 47aeee41..9c41aa87 100644 --- a/tests/core/test_safe_command_elision.py +++ b/tests/core/test_safe_command_elision.py @@ -105,3 +105,25 @@ async def _raise(*args: object, **kwargs: object) -> object: with pytest.raises(_ApprovalReached): await shell_tool(Params(command="touch /tmp/should-not-run")) + + +class TestSafeModeKeepsPrompts: + @pytest.mark.asyncio + async def test_safe_mode_blocks_elision(self, shell_tool) -> None: + """A user who enabled safe mode depends on prompts as checkpoints; + provably-safe classification must not bypass them.""" + from pythinker_code.tools.shell import Params + + class _ApprovalReached(Exception): + pass + + async def _raise(*args: object, **kwargs: object) -> object: + raise _ApprovalReached + + shell_tool._approval.set_safe_mode(True) + shell_tool._approval.request = _raise # type: ignore[method-assign] + + import pytest + + with pytest.raises(_ApprovalReached): + await shell_tool(Params(command="echo checkpoint")) diff --git a/tests/subagents/test_worktree_isolation.py b/tests/subagents/test_worktree_isolation.py index 488b9196..b151454e 100644 --- a/tests/subagents/test_worktree_isolation.py +++ b/tests/subagents/test_worktree_isolation.py @@ -98,3 +98,36 @@ def test_write_types_qualify(self) -> None: def test_read_types_do_not(self) -> None: for read_type in ("explore", "review", "verifier", "judge", "unknown-type"): assert subagent_type_allows_file_mutation(read_type) is False + + +class TestCommittedChangesRetention: + @pytest.mark.asyncio + async def test_committed_clean_worktree_is_retained(self, tmp_path: Path) -> None: + """A child that commits its work leaves a CLEAN tree; removing the + worktree would orphan those commits as dangling objects.""" + repo = await _repo(tmp_path) + worktree = tmp_path / "wt" + await create_agent_worktree(repo, worktree) + (worktree / "a.txt").write_text("committed work") + await _git(worktree, "add", ".") + await _git(worktree, "config", "user.email", "t@t") + await _git(worktree, "config", "user.name", "T") + await _git(worktree, "commit", "-m", "child work") + + summary = await worktree_change_summary(worktree) + disposition = await cleanup_agent_worktree(repo, worktree, has_changes=bool(summary)) + + assert "commit(s) ahead" in summary + assert disposition == "retained" + assert worktree.exists() + + @pytest.mark.asyncio + async def test_missing_base_sidecar_fails_closed_to_retention(self, tmp_path: Path) -> None: + repo = await _repo(tmp_path) + worktree = tmp_path / "wt" + await create_agent_worktree(repo, worktree) + (worktree.parent / f"{worktree.name}.base-sha").unlink() + + summary = await worktree_change_summary(worktree) + + assert "commit(s) ahead" in summary # unknown provenance counts as changes diff --git a/tests/tools/test_mcp_tool_filter.py b/tests/tools/test_mcp_tool_filter.py index 656c5a72..a44fd9bb 100644 --- a/tests/tools/test_mcp_tool_filter.py +++ b/tests/tools/test_mcp_tool_filter.py @@ -123,3 +123,29 @@ async def test_denied_tool_errors_without_reaching_server(self, runtime) -> None assert result.is_error assert "disabled" in (result.message or "").lower() + + +class TestReadOnlyHintParallel: + def test_read_only_hint_enables_parallel(self, runtime) -> None: + tool = MCPTool( + "srv", + mcp.types.Tool( + name="lookup", + inputSchema={}, + annotations=mcp.types.ToolAnnotations(readOnlyHint=True), + ), + cast(Any, _ListingClient([])), + runtime=runtime, + ) + + assert tool.supports_parallel is True + + def test_unannotated_tool_stays_exclusive(self, runtime) -> None: + tool = MCPTool( + "srv", + mcp.types.Tool(name="mutate", inputSchema={}), + cast(Any, _ListingClient([])), + runtime=runtime, + ) + + assert tool.supports_parallel is False diff --git a/tests/tools/test_write_file.py b/tests/tools/test_write_file.py index 7506d39b..b6891303 100644 --- a/tests/tools/test_write_file.py +++ b/tests/tools/test_write_file.py @@ -218,3 +218,24 @@ async def capture_request(tool_name, action, description, **kwargs): # type: ig assert captured_actions[0] == FileActions.EDIT_OUTSIDE, ( f"Expected EDIT_OUTSIDE for symlink escaping workspace, got {captured_actions[0]}" ) + + +async def test_relative_path_resolves_against_work_dir( + write_file_tool: WriteFile, temp_work_dir: HostPath, tmp_path: Path +): + """An isolated child's relative write must land in its (overridden) work + dir, not wherever the process cwd happens to be.""" + import os + + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + previous_cwd = os.getcwd() + os.chdir(elsewhere) + try: + result = await write_file_tool(Params(path="rel_note.txt", content="hi")) + finally: + os.chdir(previous_cwd) + + assert not result.is_error + assert (Path(str(temp_work_dir)) / "rel_note.txt").read_text() == "hi" + assert not (elsewhere / "rel_note.txt").exists() From 07ac5a022ade526025619f0ae5f1de5b8de7e59c Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 10:56:13 -0400 Subject: [PATCH 45/49] fix: resolve PR #122 review findings (CodeRabbit, CodeQL, typos) Address all bot review feedback on the agent-harness branch: Security / CI gates: - project_trust: store SHA-256 digests instead of clear-text paths (CodeQL clear-text-storage) and serialize read-modify-write behind a cross-process file lock; read legacy clear-text stores for compat. - typos: rename intentional config-key fixtures to validly-spelled unknown keys; fix `unparsable`/`default_yolo_typo` prose in planning doc. - api_errors: drop redundant `400 <= status < 500` guard (always true after the >=500 early return). Correctness: - ssh: resolve relative cwd against the host's tracked cwd, not the SSH login dir. - config: read untrusted project/local scopes independently so one bad file no longer discards the other. - context: re-repair the post-usage slice so token accounting reflects the repaired history; keep tool messages with no call id. - soul/agent + subagents/builder: recompute AGENTS.md payload for a child worktree override instead of inheriting the parent's. - subagents/core: import TextPart/ThinkPart from pythinker_core.message; collect git context from the effective child work dir. - subagents/runner: fail an explicit context fork loudly instead of silently degrading to a blank child. - subagents/worktree: validate a pre-existing dest is a registered worktree before reusing it. - background/agent_runner: report retained isolation worktrees on the early failure/empty-output exits too. - slash: persist the carry-over summary as a system turn, not a user turn. Safety hardening: - permission: refuse prompt elision for read-only commands with path-bearing operands (cat /etc/shadow, git -C /other, ../secret). - toolset: keep MCP tools exclusive in the same-step gate (ignore untrusted remote readOnlyHint); stage MCP inventory locally until connect succeeds. - read_media: keep ReadMediaFile serialized (large in-memory payloads). - permissions_state: include agent_execution_profile in the injection fingerprint so profile switches reinject. plan_mode: complete the truncated exit-rule sentence and wrap multi-line reminder literals in parentheses (fixes the implicit-concat warning without splitting sentences across rendered lines). Tests: cover the trust-store hashing/legacy path, the work_dir and runtime.work_dir seams by behavior not identity, the MCP exclusive default, legacy StatusUpdate deserialization, and the new unsafe path-operand commands; stop pinning full reminder text in loop tests. --- CHANGELOG.md | 4 +- .../pythinker-host/src/pythinker_host/ssh.py | 8 +- src/pythinker_code/background/agent_runner.py | 2 + src/pythinker_code/config.py | 8 +- src/pythinker_code/project_trust.py | 110 +++++++++++++++--- src/pythinker_code/soul/agent.py | 4 + src/pythinker_code/soul/api_errors.py | 2 +- src/pythinker_code/soul/context.py | 6 +- .../dynamic_injections/permissions_state.py | 1 + .../soul/dynamic_injections/plan_mode.py | 28 +++-- src/pythinker_code/soul/permission.py | 32 +++++ src/pythinker_code/soul/toolset.py | 19 +-- src/pythinker_code/subagents/builder.py | 12 ++ src/pythinker_code/subagents/core.py | 6 +- src/pythinker_code/subagents/runner.py | 10 +- src/pythinker_code/subagents/worktree.py | 25 +++- src/pythinker_code/tools/file/read_media.py | 2 +- src/pythinker_code/ui/shell/slash.py | 2 +- tasks/agent-harness-adoption-plan.md | 4 +- tests/acp/test_acp_tool_visibility.py | 5 +- tests/core/test_config_unknown_keys.py | 20 ++-- tests/core/test_project_trust.py | 27 +++++ tests/core/test_pythinkersoul_ralph_loop.py | 74 +++++------- tests/core/test_pythinkersoul_steer.py | 20 ++-- tests/core/test_safe_command_elision.py | 6 +- tests/core/test_toolset_concurrency.py | 17 ++- tests/core/test_wire_message.py | 19 +++ tests/core/test_work_dir_seam.py | 4 +- tests/tools/test_mcp_tool_filter.py | 4 +- tests/tools/test_memory_tool.py | 15 ++- 30 files changed, 357 insertions(+), 139 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df4f26fd..2861e7d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased -- **Adversarial branch review hardening.** A multi-agent review pass confirmed and fixed: committed-but-clean isolation worktrees are now retained (commits ahead of the creation base count as changes — they were previously orphaned on cleanup); foreground shell commands and relative-path file edits now resolve against the agent's work dir, so worktree isolation actually binds them (host exec gained a `cwd` argument); safe mode now also disables the read-only-command prompt elision; MCP tools annotated `readOnlyHint` run in parallel in the same-step gate; worktree add/remove serializes per repo. Behavior notes: same-step tool calls without `supports_parallel` now serialize deterministically (previously fully concurrent), and text-mode error diagnostics moved to stderr — capture `2>&1` or use `--output-format stream-json` if you scraped stdout. +- **Adversarial branch review hardening.** A multi-agent review pass confirmed and fixed: committed-but-clean isolation worktrees are now retained (commits ahead of the creation base count as changes — they were previously orphaned on cleanup); foreground shell commands and relative-path file edits now resolve against the agent's work dir, so worktree isolation actually binds them (host exec gained a `cwd` argument); safe mode now also disables the read-only-command prompt elision; locally parallel-safe MCP/tools run in parallel in the same-step gate; worktree add/remove serializes per repo. Behavior notes: same-step tool calls without `supports_parallel` now serialize deterministically (previously fully concurrent), and text-mode error diagnostics moved to stderr — capture `2>&1` or use `--output-format stream-json` if you scraped stdout. - **The agent now knows its own permission posture.** A live permissions-state reminder renders the enforced profile, safe-mode/yolo/auto flags, mutation/network allowances, session-approved actions, and the shell gate's command-shaping rules — re-emitted exactly when the posture changes (/yolo, /auto, /trust, new approvals) instead of the model discovering policy through denied tool calls. - **Edits recover from whitespace and smart-punctuation drift.** StrReplaceFile no longer hard-fails with "old string not found" when the only mismatch is trailing whitespace, indentation, or smart quotes/dashes: a graduated line-window ladder relocates the edit, replaces the actual file slice (preserving CRLF endings), and names the relaxation it used in the tool message. Multiple fuzzy hits without replace_all still error, so ambiguity is never silently resolved. - **`isolation="worktree"` is now enforced for background write agents.** Previously it only recorded intent, so parallel coders shared one working tree and could clobber each other. A write-profile child now runs in its own git worktree of HEAD; its final report names the worktree path with a diff summary so changes merge deliberately, clean worktrees are removed, non-git roots fail with an actionable error, and read-profile children ignore the request. @@ -23,7 +23,7 @@ GitHub Releases page; `0.8.0` is the new starting line. - **A hung MCP server can no longer stall the whole session.** Server connects are bounded by a new `mcp.client.startup_timeout_ms` (default 30s) — previously a hung connect blocked every agent turn. `/mcp` now shows one actionable line per failed server (timeout → the config knob, 401 → the exact auth command, missing binary → the command path) instead of a bare "failed". - **Provably read-only commands no longer prompt for approval.** The first `ls` or `git status` of a session used to interrupt with an approval dialog. A tight positive allowlist (read-only binaries and git subcommands, with hidden-command, write-redirection, wrapper, and fake-path rejections, fail closed) now elides the prompt in the root agent; subagents keep requesting approval as their unattended defense surface, and deny-profile decisions are never overridden. - **Switching models keeps your conversation.** `/model` used to start a fresh session, discarding all context. The switch now seeds the new session with a plain-text summary written by the outgoing model (so provider-specific message formats never cross the boundary), falling back to the old fresh start if summarization fails; disable with `model_switch_carryover = false`. -- **Typo'd config keys are no longer silently ignored.** Loading now diffs the merged config against the model schema and warns with the dotted path and originating scope file for every unrecognized key (`defaut_yolo = true` names itself and its file instead of silently changing nothing); `PYTHINKER_STRICT_CONFIG=1` escalates the findings to a startup error for CI. +- **Typo'd config keys are no longer silently ignored.** Loading now diffs the merged config against the model schema and warns with the dotted path and originating scope file for every unrecognized key (`default_yolo_typo = true` names itself and its file instead of silently changing nothing); `PYTHINKER_STRICT_CONFIG=1` escalates the findings to a startup error for CI. - **Project-scope hooks now require trusting the project.** A cloned repository's `.pythinker/config.toml` could previously auto-execute its shell hooks at session start. Hooks from project and local scopes now load only after `/trust` records a durable per-project decision (stored user-side in `trusted_projects.json`, keyed by the resolved repo root); until then they are stripped with a warning naming the fix. Broken TOML in an untrusted project no longer blocks startup — the scope is treated as empty with a warning, while trusted projects keep the loud error. - **Agent orchestration guidance is sharper for substantial tasks.** The default prompt now sharpens work-shaping guidance, and a new root-only runtime reminder nudges substantial normal-mode tasks toward the lightest effective path — direct tools, `SetTodoList`, foreground `RunAgents`, or verification — while backing off for plan mode, `/goal`, auto mode, and subagents. - **The in-app updater recovers from Homebrew's untrusted-tap refusal.** Homebrew 5.0 (`HOMEBREW_REQUIRE_TAP_TRUST`) refuses to load formulas from third-party taps until `brew trust <tap>` is run once, which made the in-app `brew upgrade` fail with only a generic "run manually" hint. The updater now detects both the hard `Refusing to load … from untrusted tap` refusal and the soft `Skipping … not trusted` warning, offers to run `brew trust pythoughts-labs/pythinker` and retry the upgrade once on an interactive terminal, and otherwise prints the exact remediation. It also catches the silent no-op where an untrusted tap is skipped during `brew update` and `brew upgrade` exits 0 without advancing the version. diff --git a/packages/pythinker-host/src/pythinker_host/ssh.py b/packages/pythinker-host/src/pythinker_host/ssh.py index 809c7576..42911a16 100644 --- a/packages/pythinker-host/src/pythinker_host/ssh.py +++ b/packages/pythinker-host/src/pythinker_host/ssh.py @@ -315,7 +315,13 @@ async def exec( # cwd before running the command. # # This is intentionally strict: if cwd doesn't exist, the command fails. - effective_cwd = cwd or self._cwd + if cwd is None: + effective_cwd = self._cwd + elif posixpath.isabs(cwd): + effective_cwd = cwd + else: + base_cwd = self._cwd or "/" + effective_cwd = posixpath.normpath(posixpath.join(base_cwd, cwd)) if effective_cwd: command = f"cd {shlex.quote(effective_cwd)} && {command}" process = await self._connection.create_process(command, encoding=None, env=env) diff --git a/src/pythinker_code/background/agent_runner.py b/src/pythinker_code/background/agent_runner.py index ec33ac13..06455062 100644 --- a/src/pythinker_code/background/agent_runner.py +++ b/src/pythinker_code/background/agent_runner.py @@ -231,6 +231,7 @@ async def _ui_loop_fn(wire: Wire) -> None: if failure is not None: self._finalize_safely(outcome="failed", reason=failure.message) output.error(_failure_recovery_message(reason=failure.message, agent_id=self._agent_id)) + self._note_retained_worktree(output) output.stage(f"failed: {failure.brief}") return output.stage("run_soul_finished") @@ -239,6 +240,7 @@ async def _ui_loop_fn(wire: Wire) -> None: self._finalize_safely( outcome="failed", reason="Agent completed but produced no output." ) + self._note_retained_worktree(output) output.stage("failed: empty output") return # Surface this child's total LLM spend so the orchestrating parent can budget a diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index 0ef9564e..75c666d5 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -398,13 +398,19 @@ def _read_toml(path: Path) -> dict[str, Any]: # the user records trust (/trust). try: project_dict = _read_toml(project_file) - local_dict = _read_toml(local_file) except ConfigError as exc: logger.warning( "Ignoring unreadable project config in untrusted project: {error}", error=exc, ) project_dict = {} + try: + local_dict = _read_toml(local_file) + except ConfigError as exc: + logger.warning( + "Ignoring unreadable project config in untrusted project: {error}", + error=exc, + ) local_dict = {} for scope_dict, scope_file in ((project_dict, project_file), (local_dict, local_file)): if scope_dict.pop("hooks", None) is not None: diff --git a/src/pythinker_code/project_trust.py b/src/pythinker_code/project_trust.py index c48c0ae6..cd899666 100644 --- a/src/pythinker_code/project_trust.py +++ b/src/pythinker_code/project_trust.py @@ -3,32 +3,87 @@ A cloned repository's project-scope config (``.pythinker/config.toml``) carries auto-executed surfaces — shell hooks above all — so those load only after the user trusts the project root. The decision persists -across sessions in a user-scope file keyed by the normalized root path; -it is never stored inside the project, where the repo could edit it. +across sessions in a user-scope file keyed by a digest of the normalized +root path; it is never stored inside the project, where the repo could +edit it, and it does not persist local filesystem paths in clear text. """ from __future__ import annotations +import contextlib +import hashlib import json +import os import tempfile +from collections.abc import Generator from pathlib import Path -from typing import cast +from typing import IO, cast from pythinker_code.share import get_share_dir from pythinker_code.utils.logging import logger _TRUST_FILE_NAME = "trusted_projects.json" +_LOCK_FILE_NAME = f"{_TRUST_FILE_NAME}.lock" +_STORE_KEY = "trusted_project_ids" +_LEGACY_STORE_KEY = "trusted_roots" def _trust_file() -> Path: return get_share_dir() / _TRUST_FILE_NAME +def _lock_file() -> Path: + return get_share_dir() / _LOCK_FILE_NAME + + def _normalize(project_root: Path) -> str: return str(project_root.expanduser().resolve(strict=False)) -def _read_trusted_roots() -> set[str]: +def _project_id(project_root: Path) -> str: + return hashlib.sha256(_normalize(project_root).encode("utf-8")).hexdigest() + + +@contextlib.contextmanager +def _locked_trust_store() -> Generator[None]: + """Serialize trust-store read/modify/write across local processes.""" + path = _lock_file() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+b") as lock_file: + _lock_file_exclusive(lock_file) + try: + yield + finally: + _unlock_file(lock_file) + + +def _lock_file_exclusive(lock_file: IO[bytes]) -> None: + if os.name == "nt": + import msvcrt + + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) + return + + import fcntl + + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + + +def _unlock_file(lock_file: IO[bytes]) -> None: + if os.name == "nt": + import msvcrt + + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + return + + import fcntl + + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def _read_trusted_project_ids() -> set[str]: path = _trust_file() if not path.exists(): return set() @@ -43,16 +98,35 @@ def _read_trusted_roots() -> set[str]: return set() if not isinstance(data, dict): return set() - roots: object = cast("dict[str, object]", data).get("trusted_roots") - if not isinstance(roots, list): + + project_ids: object = cast("dict[str, object]", data).get(_STORE_KEY) + if isinstance(project_ids, list): + return { + value + for value in cast("list[object]", project_ids) + if isinstance(value, str) and _looks_like_sha256(value) + } + + # Backward compatibility for pre-hash stores. Read legacy clear-text paths, + # return their digests, and let the next write persist only hashed ids. + legacy_roots: object = cast("dict[str, object]", data).get(_LEGACY_STORE_KEY) + if not isinstance(legacy_roots, list): return set() - return {root for root in cast("list[object]", roots) if isinstance(root, str)} + return { + hashlib.sha256(root.encode("utf-8")).hexdigest() + for root in cast("list[object]", legacy_roots) + if isinstance(root, str) + } + + +def _looks_like_sha256(value: str) -> bool: + return len(value) == 64 and all(char in "0123456789abcdef" for char in value) -def _write_trusted_roots(roots: set[str]) -> None: +def _write_trusted_project_ids(project_ids: set[str]) -> None: path = _trust_file() path.parent.mkdir(parents=True, exist_ok=True) - payload = json.dumps({"trusted_roots": sorted(roots)}, indent=2) + "\n" + payload = json.dumps({_STORE_KEY: sorted(project_ids)}, indent=2) + "\n" # Atomic replace so a crash mid-write cannot corrupt the trust store. fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=path.name, suffix=".tmp") tmp_path = Path(tmp_name) @@ -67,15 +141,17 @@ def _write_trusted_roots(roots: set[str]) -> None: def is_project_trusted(project_root: Path) -> bool: """Whether the user has durably trusted *project_root*.""" - return _normalize(project_root) in _read_trusted_roots() + with _locked_trust_store(): + return _project_id(project_root) in _read_trusted_project_ids() def set_project_trusted(project_root: Path, trusted: bool) -> None: """Durably record (or revoke) trust for *project_root*.""" - roots = _read_trusted_roots() - normalized = _normalize(project_root) - if trusted: - roots.add(normalized) - else: - roots.discard(normalized) - _write_trusted_roots(roots) + with _locked_trust_store(): + project_ids = _read_trusted_project_ids() + project_id = _project_id(project_root) + if trusted: + project_ids.add(project_id) + else: + project_ids.discard(project_id) + _write_trusted_project_ids(project_ids) diff --git a/src/pythinker_code/soul/agent.py b/src/pythinker_code/soul/agent.py index a6ceb851..7b9c9845 100644 --- a/src/pythinker_code/soul/agent.py +++ b/src/pythinker_code/soul/agent.py @@ -396,6 +396,7 @@ def copy_for_subagent( llm_override: LLM | None = None, work_dir_override: HostPath | None = None, work_dir_ls: str | None = None, + work_dir_agents_md: str | None = None, ) -> Runtime: """Clone runtime for a subagent. @@ -405,10 +406,13 @@ def copy_for_subagent( """ builtin_args = self.builtin_args if work_dir_override is not None: + agents_md = work_dir_agents_md or "" builtin_args = replace( builtin_args, PYTHINKER_WORK_DIR=work_dir_override, PYTHINKER_WORK_DIR_LS=work_dir_ls or "", + PYTHINKER_AGENTS_MD=agents_md, + PYTHINKER_AGENTS_MD_FENCE=_agents_md_fence(agents_md), ) return Runtime( config=self.config, diff --git a/src/pythinker_code/soul/api_errors.py b/src/pythinker_code/soul/api_errors.py index 5ffb0eb1..4902d405 100644 --- a/src/pythinker_code/soul/api_errors.py +++ b/src/pythinker_code/soul/api_errors.py @@ -41,7 +41,7 @@ def classify_api_error(e: Exception) -> tuple[str, int | None]: return "auth", status_code if status >= 500: return "5xx_server", status_code - if 400 <= status < 500: + if status < 500: msg_lower = str(e).lower() if any(marker in msg_lower for marker in _CONTEXT_OVERFLOW_MARKERS): return "context_overflow", status_code diff --git a/src/pythinker_code/soul/context.py b/src/pythinker_code/soul/context.py index 80f20052..3e491e68 100644 --- a/src/pythinker_code/soul/context.py +++ b/src/pythinker_code/soul/context.py @@ -56,7 +56,9 @@ def _synthesize_lost_results() -> None: for message in history: if message.role == "tool": - if message.tool_call_id in open_call_ids: + if message.tool_call_id is None: + repaired.append(message) + elif message.tool_call_id in open_call_ids: open_call_ids.remove(message.tool_call_id) repaired.append(message) else: @@ -130,6 +132,7 @@ async def restore(self) -> bool: ) self._history[:] = repair_history_invariants(self._history) + messages_after_last_usage[:] = repair_history_invariants(messages_after_last_usage) self._pending_token_estimate = estimate_text_tokens(messages_after_last_usage) return True @@ -282,6 +285,7 @@ async def revert_to(self, checkpoint_id: int): await new_file.write(line) self._history[:] = repair_history_invariants(self._history) + messages_after_last_usage[:] = repair_history_invariants(messages_after_last_usage) self._pending_token_estimate = estimate_text_tokens(messages_after_last_usage) async def clear(self): diff --git a/src/pythinker_code/soul/dynamic_injections/permissions_state.py b/src/pythinker_code/soul/dynamic_injections/permissions_state.py index ed2b94ef..b3e7e2c4 100644 --- a/src/pythinker_code/soul/dynamic_injections/permissions_state.py +++ b/src/pythinker_code/soul/dynamic_injections/permissions_state.py @@ -40,6 +40,7 @@ async def get_injections( approved = tuple(sorted(approval.session_approved_actions())) fingerprint = ( profile.name, + soul.runtime.config.agent_execution_profile, approval.is_yolo(), approval.is_auto(), approval.is_safe_mode(), diff --git a/src/pythinker_code/soul/dynamic_injections/plan_mode.py b/src/pythinker_code/soul/dynamic_injections/plan_mode.py index bd090e0c..5cb332de 100644 --- a/src/pythinker_code/soul/dynamic_injections/plan_mode.py +++ b/src/pythinker_code/soul/dynamic_injections/plan_mode.py @@ -154,23 +154,29 @@ def _full_reminder( "check that would prove it worked end-to-end. Keep the plan file " "skimmable: 3-5 short sections (including Assumptions and Verification), " "bullets grouped by subsystem, naming only load-bearing files", - "5. Exit — call ExitPlanMode for user approval, only once the plan is " - "decision-complete: an implementer could execute it without making any " - 'decision themselves. "Figure out X during implementation" is not a plan ' - "step — resolve it now by exploring, or ask", + ( + "5. Exit — call ExitPlanMode for user approval, only once the plan is " + "decision-complete: an implementer could execute it without making any " + 'decision themselves. "Figure out X during implementation" is not a plan ' + "step — resolve it now by exploring, or ask via AskUserQuestion." + ), ] ) lines.extend( [ "", "## Resolving unknowns", - "Unknowns come in two kinds. Repo-discoverable facts (current behavior, " - "existing patterns, file locations): explore and answer them yourself — " - "never ask the user. Preference or scope decisions (product behavior, " - "tradeoff priorities, rollout): surface them early with AskUserQuestion, " - "offering 2-4 concrete options with your recommended default first.", - "If a preference question stays unanswered, proceed with your recommended " - "default and record it in the plan's Assumptions section.", + ( + "Unknowns come in two kinds. Repo-discoverable facts (current behavior, " + "existing patterns, file locations): explore and answer them yourself — " + "never ask the user. Preference or scope decisions (product behavior, " + "tradeoff priorities, rollout): surface them early with AskUserQuestion, " + "offering 2-4 concrete options with your recommended default first." + ), + ( + "If a preference question stays unanswered, proceed with your recommended " + "default and record it in the plan's Assumptions section." + ), "", "## Handling multiple approaches", "Keep it focused: at most 2-3 meaningfully different approaches. " diff --git a/src/pythinker_code/soul/permission.py b/src/pythinker_code/soul/permission.py index dda6d94b..4b26e813 100644 --- a/src/pythinker_code/soul/permission.py +++ b/src/pythinker_code/soul/permission.py @@ -569,6 +569,32 @@ def _shell_hidden_command_reason(command: str) -> str | None: _SAFE_INLINE_ENV_VARS = frozenset({"LANG", "LC_ALL", "LC_COLLATE", "LC_CTYPE", "LC_MESSAGES", "TZ"}) +_WINDOWS_DRIVE_PATH_RE = re.compile(r"^[A-Za-z]:[\\/]") + + +def _has_unvalidated_path_operand(args: Sequence[str]) -> bool: + """Whether args contain a path that prompt elision cannot safely bound. + + Prompt elision has no runtime workspace root, so it must fail closed for + absolute paths, home paths, parent traversal, or explicit path separators. + The command can still run after normal approval. + """ + for arg in args: + if arg == "--": + continue + value = arg.split("=", 1)[1] if arg.startswith("--") and "=" in arg else arg + if ( + value.startswith(("/", "~")) + or "../" in value + or value == ".." + or "/" in value + or "\\" in value + or _WINDOWS_DRIVE_PATH_RE.match(value) + ): + return True + return False + + def is_known_safe_command(command: str) -> bool: """Whether *command* is provably read-only, qualifying for prompt elision. @@ -625,7 +651,13 @@ def _is_safe_readonly_segment(tokens: list[str]) -> bool: else: base = command base = base.lower() + if _has_unvalidated_path_operand(args): + return False if base == "git": + if any(arg in {"-C", "--git-dir", "--work-tree"} for arg in args): + return False + if any(arg.startswith(("--git-dir=", "--work-tree=")) for arg in args): + return False subcommand = _git_subcommand(args) if subcommand not in _SAFE_GIT_SUBCOMMANDS: return False diff --git a/src/pythinker_code/soul/toolset.py b/src/pythinker_code/soul/toolset.py index 25967df2..86789a80 100644 --- a/src/pythinker_code/soul/toolset.py +++ b/src/pythinker_code/soul/toolset.py @@ -1068,13 +1068,14 @@ async def _connect_server( async def _open_and_inventory() -> None: async with server_info.client as client: skipped: list[str] = [] + local_tools: list[MCPTool[Any]] = [] for tool in await client.list_tools(): if server_info.tool_filter and not server_info.tool_filter.allows( tool.name ): skipped.append(tool.name) continue - server_info.tools.append( + local_tools.append( MCPTool( server_name, tool, @@ -1096,12 +1097,15 @@ async def _open_and_inventory() -> None: # still connect, so capture them best-effort (mcpext-1). A # METHOD_NOT_FOUND means the capability is genuinely absent; any # other error is surfaced (WARNING) rather than masked as "none". - server_info.resources = await _discover_optional_capability( + local_resources = await _discover_optional_capability( server_name, "resources", client.list_resources ) - server_info.prompts = await _discover_optional_capability( + local_prompts = await _discover_optional_capability( server_name, "prompts", client.list_prompts ) + server_info.tools = local_tools + server_info.resources = local_resources + server_info.prompts = local_prompts try: # Bound connect+inventory: a hung server would otherwise block @@ -1298,13 +1302,12 @@ def mcp_server_name(self) -> str: @property def supports_parallel(self) -> bool: - """Honor the MCP readOnlyHint annotation in the same-step gate. + """MCP tools stay exclusive unless locally proven safe. - Read-only server tools (doc/resource lookups) may overlap instead of - serializing; anything unannotated stays exclusive (safe default). + Server-supplied annotations are untrusted remote metadata, so they must + not relax local write serialization. """ - annotations = getattr(self._mcp_tool, "annotations", None) - return bool(getattr(annotations, "readOnlyHint", False)) + return False async def __call__(self, *args: Any, **kwargs: Any) -> ToolReturnValue: # Call-time re-check of the list-time filter: defense in depth for diff --git a/src/pythinker_code/subagents/builder.py b/src/pythinker_code/subagents/builder.py index c2a64b83..8a68992b 100644 --- a/src/pythinker_code/subagents/builder.py +++ b/src/pythinker_code/subagents/builder.py @@ -29,11 +29,23 @@ async def build_builtin_instance( thinking=launch_spec.thinking, thinking_effort=launch_spec.thinking_effort, ) + work_dir_ls: str | None = None + work_dir_agents_md: str | None = None + if work_dir_override is not None: + from pythinker_code.soul.agent import load_agents_md + from pythinker_code.utils.path import list_directory + + work_dir_ls, work_dir_agents_md = ( + await list_directory(work_dir_override), + await load_agents_md(work_dir_override), + ) runtime = self._root_runtime.copy_for_subagent( agent_id=agent_id, subagent_type=type_def.name, llm_override=llm_override, work_dir_override=work_dir_override, + work_dir_ls=work_dir_ls, + work_dir_agents_md=work_dir_agents_md, ) return await load_agent( type_def.agent_file, diff --git a/src/pythinker_code/subagents/core.py b/src/pythinker_code/subagents/core.py index 8c911a7b..c06ff915 100644 --- a/src/pythinker_code/subagents/core.py +++ b/src/pythinker_code/subagents/core.py @@ -13,7 +13,7 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING -from pythinker_core.message import Message +from pythinker_core.message import Message, TextPart, ThinkPart from pythinker_host.path import HostPath from pythinker_code.notifications import is_notification_message @@ -23,7 +23,6 @@ from pythinker_code.subagents.builder import SubagentBuilder from pythinker_code.subagents.models import AgentLaunchSpec, AgentTypeDefinition from pythinker_code.subagents.store import SubagentStore -from pythinker_code.wire.types import TextPart, ThinkPart # NOTE: these must match the registered type names in agents/default/agent.yaml # (dashed), which _SUBAGENT_PROFILES also keys on — not the yaml file stems. @@ -160,7 +159,8 @@ async def prepare_soul( if spec.type_def.name in GIT_CONTEXT_AGENT_TYPES and not spec.resumed: from pythinker_code.subagents.git_context import collect_git_context - git_ctx = await collect_git_context(runtime.builtin_args.PYTHINKER_WORK_DIR) + git_context_dir = spec.work_dir_override or runtime.builtin_args.PYTHINKER_WORK_DIR + git_ctx = await collect_git_context(git_context_dir) if git_ctx: prompt = f"{git_ctx}\n\n{prompt}" prompt = _prepend_output_language_instruction(prompt) diff --git a/src/pythinker_code/subagents/runner.py b/src/pythinker_code/subagents/runner.py index 30062dec..383b9b3e 100644 --- a/src/pythinker_code/subagents/runner.py +++ b/src/pythinker_code/subagents/runner.py @@ -282,8 +282,8 @@ async def _load_fork_history(self) -> list[Message] | None: Reads the persisted parent context file rather than live soul state, so the fork inherits exactly what would survive a parent restore - (including the restore-time pairing repair). Best-effort: a read - failure degrades to a blank child rather than failing the spawn. + (including the restore-time pairing repair). An explicit fork must not + silently degrade to a blank child: read failures abort the spawn. """ from pythinker_code.soul.context import Context from pythinker_code.subagents.core import filter_history_for_fork @@ -291,9 +291,11 @@ async def _load_fork_history(self) -> list[Message] | None: try: parent_context = Context(file_backend=self._runtime.session.context_file) await parent_context.restore() - except Exception: + except Exception as exc: logger.warning("Context fork: failed to read parent history", exc_info=True) - return None + raise RuntimeError( + "Context fork requested, but the parent history could not be restored." + ) from exc forked = filter_history_for_fork(parent_context.history) return forked or None diff --git a/src/pythinker_code/subagents/worktree.py b/src/pythinker_code/subagents/worktree.py index 51efa88a..50c5cc06 100644 --- a/src/pythinker_code/subagents/worktree.py +++ b/src/pythinker_code/subagents/worktree.py @@ -52,6 +52,20 @@ async def _git(args: list[str], cwd: Path) -> tuple[int, str, str]: ) +async def _is_registered_worktree(repo_dir: Path, dest: Path) -> bool: + code, stdout, _ = await _git(["worktree", "list", "--porcelain"], repo_dir) + if code != 0: + return False + wanted = str(dest.resolve(strict=False)) + for line in stdout.splitlines(): + if not line.startswith("worktree "): + continue + registered = Path(line.removeprefix("worktree ")).resolve(strict=False) + if str(registered) == wanted: + return True + return False + + async def create_agent_worktree(repo_dir: Path, dest: Path) -> None: """Create a detached worktree of HEAD at *dest* for one child agent. @@ -64,11 +78,16 @@ async def create_agent_worktree(repo_dir: Path, dest: Path) -> None: f"isolation='worktree' requires a git repository at {repo_dir}; " "launch without isolation, or run `git init` first" ) - if dest.exists(): - # Resume of an isolated agent reuses its existing worktree. - return dest.parent.mkdir(parents=True, exist_ok=True) async with _REPO_LOCKS[str(repo_dir)]: + if dest.exists(): + # Resume of an isolated agent may reuse its existing worktree, but + # only after verifying the path is still registered for this repo. + if await _is_registered_worktree(repo_dir, dest): + return + raise WorktreeError( + f"isolation worktree path exists but is not a registered worktree: {dest}" + ) code, _, stderr = await _git(["worktree", "add", "--detach", str(dest), "HEAD"], repo_dir) if code != 0: first_line = stderr.splitlines()[0] if stderr else "unknown git error" diff --git a/src/pythinker_code/tools/file/read_media.py b/src/pythinker_code/tools/file/read_media.py index 405e96f1..f109a6c5 100644 --- a/src/pythinker_code/tools/file/read_media.py +++ b/src/pythinker_code/tools/file/read_media.py @@ -49,7 +49,7 @@ class Params(BaseModel): class ReadMediaFile(CallableTool2[Params]): name: str = "ReadMediaFile" - supports_parallel: bool = True + supports_parallel: bool = False params: type[Params] = Params def __init__(self, runtime: Runtime) -> None: diff --git a/src/pythinker_code/ui/shell/slash.py b/src/pythinker_code/ui/shell/slash.py index a8c4c8ef..a30b78a0 100644 --- a/src/pythinker_code/ui/shell/slash.py +++ b/src/pythinker_code/ui/shell/slash.py @@ -403,7 +403,7 @@ async def _carry_context_to_session(soul: PythinkerSoul, new_session: Any) -> bo from pythinker_core.message import Message seed = Message( - role="user", + role="system", content=[ system( "Summary of the conversation so far, carried over from the previous " diff --git a/tasks/agent-harness-adoption-plan.md b/tasks/agent-harness-adoption-plan.md index 3313f6a8..7dd0fe43 100644 --- a/tasks/agent-harness-adoption-plan.md +++ b/tasks/agent-harness-adoption-plan.md @@ -80,7 +80,7 @@ generic pythinker agent enhancements — no external product names in code, comm ### `config-features/unknown-config-key-detection-with-source-located-diagnostics` — missing, M, high -**Today.** Missing. Config models in src/pythinker_code/config.py use Pydantic's default extra='ignore', so a typo'd key (e.g. 'defaut_yolo') silently vanishes and changes agent behavior with no signal; validation errors are scope-attributed via _lookup_provenance but carry no positions, and there is no strict mode. +**Today.** Missing. Config models in src/pythinker_code/config.py use Pydantic's default extra='ignore', so a typo'd key (e.g. 'default_yolo_typo') silently vanishes and changes agent behavior with no signal; validation errors are scope-attributed via _lookup_provenance but carry no positions, and there is no strict mode. **Verifier note.** Claim confirmed. No extra='forbid'/strict mode anywhere in the config models; typo'd keys are silently dropped. Provenance enrichment attaches only scope file-path strings to validation errors, with no line/column positions. @@ -378,7 +378,7 @@ generic pythinker agent enhancements — no external product names in code, comm **Verifier note.** Claim confirmed. Scope locks exist and cover the claimed keys, but enforcement is hard-fail (ConfigError raise), not sanitize-and-warn; invalid project TOML also hard-fails, so a repo-controlled .pythinker/config.toml can block startup in that directory. -**Adopt.** In the GUARD step, strip locked paths from project/local dicts and collect startup warnings ('ignored providers in .pythinker/config.toml; move to ~/.pythinker/config.toml') instead of raising; degrade unparseable project/local TOML to an empty scope with a warning. Keep hard failure for the user scope only. Surface accumulated warnings once in the shell banner. +**Adopt.** In the GUARD step, strip locked paths from project/local dicts and collect startup warnings ('ignored providers in .pythinker/config.toml; move to ~/.pythinker/config.toml') instead of raising; degrade unparsable project/local TOML to an empty scope with a warning. Keep hard failure for the user scope only. Surface accumulated warnings once in the shell banner. **Files.** `src/pythinker_code/config.py`, `<ref>/config/src/loader/mod.rs` diff --git a/tests/acp/test_acp_tool_visibility.py b/tests/acp/test_acp_tool_visibility.py index 9c2c0890..47a0f5d8 100644 --- a/tests/acp/test_acp_tool_visibility.py +++ b/tests/acp/test_acp_tool_visibility.py @@ -14,7 +14,6 @@ from pythinker_host.local import local_host import pythinker_code.acp.tools as acp_tools -from pythinker_code.acp.tools import replace_tools from pythinker_code.soul.toolset import PythinkerToolset from pythinker_code.tools.ask_user import AskUserQuestion @@ -34,7 +33,7 @@ def test_ask_user_question_is_hidden_from_model(self, monkeypatch) -> None: monkeypatch.setattr(acp_tools, "get_current_host", lambda: local_host) toolset = _make_toolset() - replace_tools(_capabilities(), MagicMock(), "sid", toolset, MagicMock()) + acp_tools.replace_tools(_capabilities(), MagicMock(), "sid", toolset, MagicMock()) visible = [tool.name for tool in toolset.tools] assert "AskUserQuestion" not in visible @@ -43,6 +42,6 @@ def test_ask_user_question_remains_registered_for_graceful_fallback(self, monkey monkeypatch.setattr(acp_tools, "get_current_host", lambda: local_host) toolset = _make_toolset() - replace_tools(_capabilities(), MagicMock(), "sid", toolset, MagicMock()) + acp_tools.replace_tools(_capabilities(), MagicMock(), "sid", toolset, MagicMock()) assert toolset.find(AskUserQuestion) is not None diff --git a/tests/core/test_config_unknown_keys.py b/tests/core/test_config_unknown_keys.py index 386af197..8ab2db55 100644 --- a/tests/core/test_config_unknown_keys.py +++ b/tests/core/test_config_unknown_keys.py @@ -1,6 +1,6 @@ """Unknown-config-key detection with source-located diagnostics. -Config models ignore extra keys, so a typo'd key ('defaut_yolo') silently +Config models ignore extra keys, so an unknown key ('default_yolo_typo') silently vanishes and changes behavior with no signal. Loading now diffs the raw merged dict against the model field tree and warns with the dotted path and originating scope file; PYTHINKER_STRICT_CONFIG escalates to an error. @@ -28,9 +28,9 @@ def _isolated_share_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None class TestUnknownKeyPaths: def test_top_level_typo_detected(self) -> None: - unknown = unknown_config_key_paths(Config, {"defaut_yolo": True}) + unknown = unknown_config_key_paths(Config, {"default_yolo_typo": True}) - assert ("defaut_yolo",) in unknown + assert ("default_yolo_typo",) in unknown def test_nested_typo_detected(self) -> None: unknown = unknown_config_key_paths(Config, {"tui": {"statuslin": {}}}) @@ -48,12 +48,14 @@ def test_valid_keys_produce_no_findings(self) -> None: def test_map_fields_allow_arbitrary_keys_but_check_values(self) -> None: data = { - "providers": {"mine": {"type": "openai", "base_url": "x", "api_key": "k", "tpyo": 1}} + "providers": { + "mine": {"type": "openai", "base_url": "x", "api_key": "k", "typo_unknown": 1} + } } unknown = unknown_config_key_paths(Config, data) - assert ("providers", "mine", "tpyo") in unknown + assert ("providers", "mine", "typo_unknown") in unknown assert all(path[:2] != ("providers", "mine") or len(path) == 3 for path in unknown) def test_list_of_models_checks_items(self) -> None: @@ -68,20 +70,20 @@ class TestLoadTimeDiagnostics: def test_unknown_key_warned_with_scope(self, tmp_path: Path, monkeypatch) -> None: from unittest.mock import patch - _write(tmp_path / "share" / "config.toml", "defaut_yolo = true\n") + _write(tmp_path / "share" / "config.toml", "default_yolo_typo = true\n") with patch("pythinker_code.config.logger") as mock_logger: _load_scoped(None) joined = " ".join(str(call) for call in mock_logger.warning.call_args_list) - assert "defaut_yolo" in joined + assert "default_yolo_typo" in joined assert "config.toml" in joined def test_strict_mode_escalates_to_error(self, tmp_path: Path, monkeypatch) -> None: monkeypatch.setenv("PYTHINKER_STRICT_CONFIG", "1") - _write(tmp_path / "share" / "config.toml", "defaut_yolo = true\n") + _write(tmp_path / "share" / "config.toml", "default_yolo_typo = true\n") - with pytest.raises(ConfigError, match="defaut_yolo"): + with pytest.raises(ConfigError, match="default_yolo_typo"): _load_scoped(None) def test_clean_config_loads_silently_in_strict_mode(self, tmp_path: Path, monkeypatch) -> None: diff --git a/tests/core/test_project_trust.py b/tests/core/test_project_trust.py index 6a291ff5..8da20fd4 100644 --- a/tests/core/test_project_trust.py +++ b/tests/core/test_project_trust.py @@ -8,6 +8,8 @@ from __future__ import annotations +import hashlib +import json from pathlib import Path import pytest @@ -66,6 +68,31 @@ def test_corrupt_trust_file_is_tolerated(self, tmp_path: Path) -> None: assert is_project_trusted(root) is False + def test_trust_store_uses_hashed_project_ids(self, tmp_path: Path) -> None: + root = tmp_path / "repo" + root.mkdir() + + set_project_trusted(root, True) + + trust_file = tmp_path / "share" / "trusted_projects.json" + payload = json.loads(trust_file.read_text(encoding="utf-8")) + normalized = str(root.expanduser().resolve(strict=False)) + expected = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + assert payload == {"trusted_project_ids": [expected]} + assert normalized not in trust_file.read_text(encoding="utf-8") + + def test_legacy_cleartext_trust_store_is_read(self, tmp_path: Path) -> None: + root = tmp_path / "repo" + root.mkdir() + trust_file = tmp_path / "share" / "trusted_projects.json" + trust_file.parent.mkdir(parents=True) + trust_file.write_text( + json.dumps({"trusted_roots": [str(root.resolve(strict=False))]}), + encoding="utf-8", + ) + + assert is_project_trusted(root) is True + class TestUntrustedProjectConfigGating: def test_untrusted_project_hooks_are_stripped(self, tmp_path: Path) -> None: diff --git a/tests/core/test_pythinkersoul_ralph_loop.py b/tests/core/test_pythinkersoul_ralph_loop.py index 2347f87e..e8ebfe5f 100644 --- a/tests/core/test_pythinkersoul_ralph_loop.py +++ b/tests/core/test_pythinkersoul_ralph_loop.py @@ -18,6 +18,7 @@ from pythinker_code.soul.agent import Agent, Runtime from pythinker_code.soul.approval import Approval from pythinker_code.soul.context import Context +from pythinker_code.soul.message import is_system_reminder_message from pythinker_code.soul.pythinkersoul import PythinkerSoul from pythinker_code.tools.utils import ToolRejectedError from pythinker_code.utils.aioqueue import QueueShutDown @@ -43,6 +44,23 @@ def expect_snapshot[T](value: T, expected: Snapshot[T]) -> None: pytest.fail(f"Snapshot mismatch: {value!r} != {expected!r}") +def _normalize_permissions_reminders(history: Sequence[Message]) -> list[Message]: + normalized: list[Message] = [] + for message in history: + if is_system_reminder_message(message) and "Permissions state:" in message.extract_text( + " " + ): + normalized.append( + Message( + role=message.role, + content=[TextPart(text="<permissions-state-reminder>")], + ) + ) + else: + normalized.append(message) + return normalized + + class SequenceStreamedMessage: def __init__(self, parts: Sequence[StreamedMessagePart]) -> None: self._iter = self._to_stream(list(parts)) @@ -204,7 +222,7 @@ async def test_ralph_loop_replays_original_prompt(runtime: Runtime, tmp_path: Pa await _run_and_collect_turns(soul, user_input) expect_snapshot( - context.history, + _normalize_permissions_reminders(context.history), snapshot( [ Message( @@ -218,17 +236,7 @@ async def test_ralph_loop_replays_original_prompt(runtime: Runtime, tmp_path: Pa ), Message( role="user", - content=[ - TextPart( - text="""\ -<system-reminder> -Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. -Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. -Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. -</system-reminder>\ -""" - ) - ], + content=[TextPart(text="<permissions-state-reminder>")], ), Message(role="assistant", content=[TextPart(text="first")]), Message( @@ -289,7 +297,7 @@ async def test_ralph_loop_stops_on_choice(runtime: Runtime, tmp_path: Path) -> N await _run_and_collect_turns(soul, "do it") expect_snapshot( - context.history, + _normalize_permissions_reminders(context.history), snapshot( [ Message( @@ -300,17 +308,7 @@ async def test_ralph_loop_stops_on_choice(runtime: Runtime, tmp_path: Path) -> N ), Message( role="user", - content=[ - TextPart( - text="""\ -<system-reminder> -Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. -Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. -Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. -</system-reminder>\ -""" - ) - ], + content=[TextPart(text="<permissions-state-reminder>")], ), Message(role="assistant", content=[TextPart(text="first")]), Message( @@ -356,7 +354,7 @@ async def test_ralph_loop_stops_on_tool_rejected(runtime: Runtime, tmp_path: Pat await _run_and_collect_turns(soul, "do it") expect_snapshot( - context.history, + _normalize_permissions_reminders(context.history), snapshot( [ Message( @@ -367,17 +365,7 @@ async def test_ralph_loop_stops_on_tool_rejected(runtime: Runtime, tmp_path: Pat ), Message( role="user", - content=[ - TextPart( - text="""\ -<system-reminder> -Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. -Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. -Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. -</system-reminder>\ -""" - ) - ], + content=[TextPart(text="<permissions-state-reminder>")], ), Message( role="assistant", @@ -418,23 +406,13 @@ async def test_ralph_loop_disabled_skips_loop_prompt(runtime: Runtime, tmp_path: await _run_and_collect_turns(soul, "hello") expect_snapshot( - context.history, + _normalize_permissions_reminders(context.history), snapshot( [ Message(role="user", content=[TextPart(text="hello")]), Message( role="user", - content=[ - TextPart( - text="""\ -<system-reminder> -Permissions state: profile 'implement' (implementation mode). Safe mode off; yolo off; auto off. File mutation allowed; shell mutation allowed; network tools allowed. -Auto-approved without prompting: provably read-only commands (ls, cat, grep, git status/log/diff, ...); session-approved actions: none. -Command shaping: the shell gate classifies plain commands only — command substitution $(...), backticks, and operators glued to words are rejected as hidden commands. Write plain, separated commands so the classifier can see them. -</system-reminder>\ -""" - ) - ], + content=[TextPart(text="<permissions-state-reminder>")], ), Message(role="assistant", content=[TextPart(text="done")]), ] diff --git a/tests/core/test_pythinkersoul_steer.py b/tests/core/test_pythinkersoul_steer.py index 83a1fd10..00357d47 100644 --- a/tests/core/test_pythinkersoul_steer.py +++ b/tests/core/test_pythinkersoul_steer.py @@ -74,6 +74,14 @@ def _runtime_with_llm(runtime: Runtime, llm: LLM) -> Runtime: ) +def _is_permissions_state_injection(message: Message) -> bool: + return ( + message.role == "user" + and is_system_reminder_message(message) + and "Permissions state:" in message.extract_text(" ") + ) + + def _llm_with_capabilities(runtime: Runtime, capabilities: set[ModelCapability]) -> LLM: assert runtime.llm is not None return LLM( @@ -136,11 +144,7 @@ async def test_consume_pending_steers_appends_history_before_emitting_wire_event sent: list[SteerInput] = [] def fake_wire_send(msg) -> None: - persisted = [ - m - for m in soul.context.history - if not (m.role == "user" and "Permissions state:" in m.extract_text(" ")) - ] + persisted = [m for m in soul.context.history if not _is_permissions_state_injection(m)] assert persisted == [Message(role="user", content=[TextPart(text="Follow up now.")])] assert isinstance(msg, SteerInput) sent.append(msg) @@ -514,11 +518,7 @@ async def ui_loop(wire: Wire) -> None: await run_soul(soul, "original question", ui_loop, asyncio.Event()) - persisted = [ - m - for m in soul.context.history - if not (m.role == "user" and "Permissions state:" in m.extract_text(" ")) - ] + persisted = [m for m in soul.context.history if not _is_permissions_state_injection(m)] assert persisted == [ Message(role="user", content=[TextPart(text="original question")]), Message(role="assistant", content=[TextPart(text="first answer")]), diff --git a/tests/core/test_safe_command_elision.py b/tests/core/test_safe_command_elision.py index 9c41aa87..1ed88690 100644 --- a/tests/core/test_safe_command_elision.py +++ b/tests/core/test_safe_command_elision.py @@ -38,6 +38,10 @@ "git commit -m x", "git branch new-branch", "git status --output=/tmp/f", + "git -C /other/repo status", + "cat /etc/shadow", + "cat ~/.ssh/id_rsa", + "cat ../secret.txt", "git log --output=/tmp/f --oneline", "git -c core.pager=rm status", "ls > /tmp/out", @@ -123,7 +127,5 @@ async def _raise(*args: object, **kwargs: object) -> object: shell_tool._approval.set_safe_mode(True) shell_tool._approval.request = _raise # type: ignore[method-assign] - import pytest - with pytest.raises(_ApprovalReached): await shell_tool(Params(command="echo checkpoint")) diff --git a/tests/core/test_toolset_concurrency.py b/tests/core/test_toolset_concurrency.py index ecc3cbee..b5b9f23f 100644 --- a/tests/core/test_toolset_concurrency.py +++ b/tests/core/test_toolset_concurrency.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio +from pathlib import Path from pythinker_core.tooling import ToolReturnValue @@ -37,9 +38,9 @@ async def call(self, arguments: object) -> ToolReturnValue: return ToolReturnValue(is_error=False, output="ok", message="ok", display=[]) -def _toolset(*tools: _RecordingTool) -> PythinkerToolset: +def _toolset(*tools: _RecordingTool, cwd: Path) -> PythinkerToolset: toolset = PythinkerToolset() - toolset._hook_engine = HookEngine([], cwd="/tmp") + toolset._hook_engine = HookEngine([], cwd=str(cwd)) for tool in tools: toolset._tool_dict[tool.name] = tool # type: ignore[assignment] return toolset @@ -57,11 +58,12 @@ async def _dispatch(toolset: PythinkerToolset, *names: str) -> None: class TestSameStepConcurrencyPolicy: - async def test_mutating_tools_serialize_in_dispatch_order(self) -> None: + async def test_mutating_tools_serialize_in_dispatch_order(self, tmp_path: Path) -> None: events: list[tuple[str, str]] = [] toolset = _toolset( _RecordingTool("WriteA", events, parallel=False), _RecordingTool("WriteB", events, parallel=False), + cwd=tmp_path, ) await _dispatch(toolset, "WriteA", "WriteB") @@ -73,33 +75,36 @@ async def test_mutating_tools_serialize_in_dispatch_order(self) -> None: ("exit", "WriteB"), ] - async def test_parallel_safe_tools_overlap(self) -> None: + async def test_parallel_safe_tools_overlap(self, tmp_path: Path) -> None: events: list[tuple[str, str]] = [] toolset = _toolset( _RecordingTool("ReadA", events, parallel=True), _RecordingTool("ReadB", events, parallel=True), + cwd=tmp_path, ) await _dispatch(toolset, "ReadA", "ReadB") assert {events[0][0], events[1][0]} == {"enter"}, events - async def test_reader_waits_for_earlier_writer(self) -> None: + async def test_reader_waits_for_earlier_writer(self, tmp_path: Path) -> None: events: list[tuple[str, str]] = [] toolset = _toolset( _RecordingTool("Write", events, parallel=False), _RecordingTool("Read", events, parallel=True), + cwd=tmp_path, ) await _dispatch(toolset, "Write", "Read") assert events.index(("exit", "Write")) < events.index(("enter", "Read")) - async def test_writer_waits_for_inflight_readers(self) -> None: + async def test_writer_waits_for_inflight_readers(self, tmp_path: Path) -> None: events: list[tuple[str, str]] = [] toolset = _toolset( _RecordingTool("Read", events, parallel=True), _RecordingTool("Write", events, parallel=False), + cwd=tmp_path, ) await _dispatch(toolset, "Read", "Write") diff --git a/tests/core/test_wire_message.py b/tests/core/test_wire_message.py index e9d39e3c..0fc0c253 100644 --- a/tests/core/test_wire_message.py +++ b/tests/core/test_wire_message.py @@ -190,6 +190,25 @@ async def test_wire_message_serde(): ) _test_serde(msg) + legacy_status = deserialize_wire_message( + { + "type": "StatusUpdate", + "payload": { + "context_usage": 0.5, + "mcp_status": { + "loading": True, + "connected": 0, + "total": 1, + "tools": 0, + "servers": [{"name": "context7", "status": "connecting", "tools": []}], + }, + }, + } + ) + assert isinstance(legacy_status, StatusUpdate) + assert legacy_status.mcp_status is not None + assert legacy_status.mcp_status.servers[0].error is None + msg = Notification( id="n1234567", category="task", diff --git a/tests/core/test_work_dir_seam.py b/tests/core/test_work_dir_seam.py index 3028d842..882352cd 100644 --- a/tests/core/test_work_dir_seam.py +++ b/tests/core/test_work_dir_seam.py @@ -18,7 +18,9 @@ def test_subagent_clone_inherits_by_default(self, runtime) -> None: child = runtime.copy_for_subagent(agent_id="a1", subagent_type="coder") assert child.work_dir == runtime.session.work_dir - assert child.builtin_args is runtime.builtin_args + for field in runtime.builtin_args.__dataclass_fields__: + if field.startswith("PYTHINKER_"): + assert getattr(child.builtin_args, field) == getattr(runtime.builtin_args, field) def test_override_redirects_child_only(self, runtime, tmp_path) -> None: worktree = HostPath.unsafe_from_local_path(tmp_path / "wt") diff --git a/tests/tools/test_mcp_tool_filter.py b/tests/tools/test_mcp_tool_filter.py index a44fd9bb..eb90d4ca 100644 --- a/tests/tools/test_mcp_tool_filter.py +++ b/tests/tools/test_mcp_tool_filter.py @@ -126,7 +126,7 @@ async def test_denied_tool_errors_without_reaching_server(self, runtime) -> None class TestReadOnlyHintParallel: - def test_read_only_hint_enables_parallel(self, runtime) -> None: + def test_remote_read_only_hint_stays_exclusive(self, runtime) -> None: tool = MCPTool( "srv", mcp.types.Tool( @@ -138,7 +138,7 @@ def test_read_only_hint_enables_parallel(self, runtime) -> None: runtime=runtime, ) - assert tool.supports_parallel is True + assert tool.supports_parallel is False def test_unannotated_tool_stays_exclusive(self, runtime) -> None: tool = MCPTool( diff --git a/tests/tools/test_memory_tool.py b/tests/tools/test_memory_tool.py index a290391d..ee948b85 100644 --- a/tests/tools/test_memory_tool.py +++ b/tests/tools/test_memory_tool.py @@ -24,12 +24,12 @@ async def __call__(self, argv): return GitResult(ok=True, exit_code=1, stdout="") -def _runtime(tmp_path, role="root"): +def _runtime(tmp_path, role="root", work_dir=None): session = SimpleNamespace(id="sess1", title="t", work_dir=_hp(tmp_path / "repo")) return SimpleNamespace( role=role, session=session, - work_dir=session.work_dir, + work_dir=work_dir or session.work_dir, rearmed=[], rearm_injection=lambda key: None, ) @@ -73,3 +73,14 @@ async def test_memory_tool_blocks_subagent(tmp_path, monkeypatch): res = await tool(Params(action="add", target="memory", content="x")) assert res.is_error is True assert "root" in res.message.lower() + + +def test_memory_tool_uses_runtime_work_dir_seam(tmp_path, monkeypatch): + monkeypatch.setenv("PYTHINKER_SHARE_DIR", str(tmp_path / "share")) + from pythinker_code.tools.memory import Memory + + runtime = _runtime(tmp_path, work_dir=_hp(tmp_path / "different_repo")) + tool = Memory(cast(Any, runtime)) + + assert tool._store._work_dir == runtime.work_dir + assert tool._store._work_dir != runtime.session.work_dir From e66f43f8a5761c6adbf650bbf65132093de39190 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 11:07:37 -0400 Subject: [PATCH 46/49] fix(tui): hide thinking shimmer while a foreground tool runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verb spinner (shimmering "Working…/Thinking…") was gated only on `_active_turn_depth > 0`, i.e. the whole turn. When the agent started a long-running foreground command — a dev server via npm/docker, a watch task — the agent coroutine just awaits the subprocess, but the shimmer kept animating for the full turn, falsely signalling active agent cognition. The tool card already shows an animated running marker plus streaming output, so the shimmer was redundant and misleading. Suppress the working indicator while any foreground tool is mid-execution (execution started, no result yet, not a detached background agent) on both render surfaces — the non-interactive Rich Live path and the interactive pinned status tail. The shimmer now means "the agent is thinking" and reappears the moment the command returns. Platform-agnostic: the root cause was turn-level gating, not Windows-specific. Adds `_ToolCallBlock.is_executing` and `_LiveView._foreground_tool_executing()`, and a test pinning that the pinned tail is empty mid-execution and returns once the tool finishes. --- CHANGELOG.md | 1 + .../ui/shell/visualize/_blocks.py | 13 ++++++++ .../ui/shell/visualize/_interactive.py | 2 +- .../ui/shell/visualize/_live_view.py | 23 +++++++++++-- .../test_visualize_running_prompt.py | 32 +++++++++++++++++++ 5 files changed, 68 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2861e7d8..26a2b065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ GitHub Releases page; `0.8.0` is the new starting line. - **Session exports redact secrets surfaced by tool output.** A tool result (e.g. `grep`/`cat` over a `.env`) could write a secret value into an exported transcript in plaintext. `/export` now redacts the value of secret-named keys (`password`, `token`, `api_key`, `secret`, …) to `[REDACTED]` in both the markdown and YAML formats, while leaving non-secret keys such as `token_count`, usernames, and ports intact. - **The welcome logo's antenna blinks a fixed number of times on launch, then settles.** Replaces the terminal's indefinite slow-blink with a bounded boot animation — the antenna ball blinks seven times after the banner prints and then holds steady. It is skipped under reduced motion, on non-interactive output, and when the terminal is too short to keep the antenna row on screen. - **Inline `/command` references get acted on, not just explained away.** When a message mentions a slash command mid-sentence (e.g. "your `/goal` today is to `/plan` and build the page"), the command doesn't auto-run — but the agent no longer leads its reply by reporting it as failed. The per-turn reminder and the system prompt now steer the agent to act on the intent: call the real `EnterPlanMode` tool for `/plan` (clarified as a genuine, callable tool so models stop doubting it exists), pursue the described objective for `/goal`, load `/skill:<name>` via `ReadSkill`, and apply equivalent guidance for other commands — only surfacing how to invoke the literal command when genuinely needed. +- **The "thinking" shimmer no longer runs while a foreground command does.** When the agent started a long-running foreground process — a dev server via `npm`/`docker`, a watch task — the shimmering verb spinner ("Working…/Thinking…") kept animating for the whole turn, implying the agent was busy when it was really just awaiting the subprocess. The spinner is now suppressed while any foreground tool is mid-execution; the tool card's own animated running marker (and its streaming output) carries the liveness, so the shimmer means "the agent is thinking" again and reappears the moment the command returns. ## 0.41.0 (2026-06-11) diff --git a/src/pythinker_code/ui/shell/visualize/_blocks.py b/src/pythinker_code/ui/shell/visualize/_blocks.py index e36ea865..c689e1f0 100644 --- a/src/pythinker_code/ui/shell/visualize/_blocks.py +++ b/src/pythinker_code/ui/shell/visualize/_blocks.py @@ -649,6 +649,19 @@ def finished(self) -> bool: def is_background_pending(self) -> bool: return self._is_background_pending + @property + def is_executing(self) -> bool: + """True while the tool body is running: execution has started, no result + has arrived, and it is not a detached background agent. + + In this window the agent coroutine is awaiting the subprocess (most + visibly a long-lived server started via the shell tool) rather than + thinking, and the tool card already shows an animated running marker. + Callers suppress the shimmering verb spinner here so it does not falsely + imply active agent cognition. + """ + return self._execution_started and not self.finished and not self._is_background_pending + @property def has_expandable_card(self) -> bool: return self._tui_card is not None and self._tui_card.can_expand diff --git a/src/pythinker_code/ui/shell/visualize/_interactive.py b/src/pythinker_code/ui/shell/visualize/_interactive.py index 7ca05a50..808b2cfa 100644 --- a/src/pythinker_code/ui/shell/visualize/_interactive.py +++ b/src/pythinker_code/ui/shell/visualize/_interactive.py @@ -540,7 +540,7 @@ def render_agent_status(self, columns: int) -> ANSI: def render_pinned_status_tail(self, columns: int) -> ANSI: """Render the trailing verb spinner that the prompt keeps pinned below a (possibly clipped) agent stream, so it stays visible above the input.""" - if self._turn_ended or self._active_turn_depth <= 0: + if self._turn_ended or self._active_turn_depth <= 0 or self._foreground_tool_executing(): return ANSI("") body = render_to_ansi(self._working_indicator(), columns=columns).rstrip("\n") return ANSI(body if body else "") diff --git a/src/pythinker_code/ui/shell/visualize/_live_view.py b/src/pythinker_code/ui/shell/visualize/_live_view.py index da57f812..84b65f73 100644 --- a/src/pythinker_code/ui/shell/visualize/_live_view.py +++ b/src/pythinker_code/ui/shell/visualize/_live_view.py @@ -570,10 +570,17 @@ def compose_agent_output( _append_action_block(blocks, tool_call.compose(), leading=True) for hook_block in getattr(self, "_hook_blocks", {}).values(): _append_action_block(blocks, hook_block.compose(), leading=True) - if include_working_indicator and self._active_turn_depth > 0: + if ( + include_working_indicator + and self._active_turn_depth > 0 + and not self._foreground_tool_executing() + ): # Keep a stable activity indicator visible even while content or # tool cards are already on-screen. This makes long-running - # background waits feel alive instead of frozen. + # background waits feel alive instead of frozen. A foreground tool + # mid-execution is the exception: the agent is awaiting it (not + # thinking), so the tool card's running marker owns the liveness + # and the shimmer verb spinner stays hidden. _append_action_block(blocks, self._working_indicator(), leading=True) for notification in list(self._live_notification_blocks): _append_action_block(blocks, notification.compose()) @@ -611,6 +618,18 @@ def _print_turn_recap(self) -> None: ) console.print() + def _foreground_tool_executing(self) -> bool: + """Whether a foreground tool is mid-execution (agent awaiting it, not thinking). + + While a tool body runs — most visibly a long-lived server started via the + shell tool — the agent is blocked awaiting the subprocess rather than + thinking, and the tool card already shows an animated running marker. The + shimmering verb spinner would falsely signal active agent cognition, so it + is suppressed in this window. Detached background agents are excluded: they + run independently of the current turn. + """ + return any(block.is_executing for block in getattr(self, "_tool_call_blocks", {}).values()) + def _working_indicator(self) -> RenderableType: now = time.monotonic() elapsed = 0.0 if self._turn_start_time is None else now - self._turn_start_time diff --git a/tests/ui_and_conv/test_visualize_running_prompt.py b/tests/ui_and_conv/test_visualize_running_prompt.py index 4c61f338..2c1dabc7 100644 --- a/tests/ui_and_conv/test_visualize_running_prompt.py +++ b/tests/ui_and_conv/test_visualize_running_prompt.py @@ -143,6 +143,38 @@ def test_render_pinned_status_tail_empty_when_turn_inactive() -> None: assert view2.render_pinned_status_tail(80).value == "" +def test_pinned_tail_hidden_while_foreground_tool_executes() -> None: + """A long-running foreground tool (e.g. a server started via the shell tool) + must not animate the shimmer verb spinner: the agent is awaiting the + subprocess, not thinking. The tool card's own running marker carries the + liveness instead, so the spinner reappears only once the tool finishes and + the agent is processing the result again.""" + import time as _time + + from pythinker_core.message import ToolCall + from pythinker_core.tooling import ToolReturnValue + + from pythinker_code.ui.shell.visualize._blocks import _ToolCallBlock + + view = object.__new__(_PromptLiveView) + view._turn_ended = False + view._active_turn_depth = 1 + view._turn_start_time = _time.monotonic() + + block = _ToolCallBlock( + ToolCall(id="tc-1", function=ToolCall.FunctionBody(name="Shell", arguments="{}")) + ) + block.mark_execution_started() + view._tool_call_blocks = {block.tool_call_id: block} + + # While the foreground command runs, the shimmer verb spinner is suppressed. + assert view.render_pinned_status_tail(80).value == "" + + # Once the tool finishes, the agent is processing again → spinner returns. + block.finish(ToolReturnValue(is_error=False, output="ok", message="ok", display=[])) + assert view.render_pinned_status_tail(80).value.strip() != "" + + @pytest.mark.asyncio async def test_prompt_live_view_status_refresh_invalidates_active_turn(monkeypatch) -> None: invalidations: list[str] = [] From 2519f9003ad0acdf5738d032e8ab09f1371067a7 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 11:27:25 -0400 Subject: [PATCH 47/49] fix(api-errors): keep 4xx lower bound; drop redundant 500 check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The earlier `if status < 500` simplification was behaviour-changing: after the `status >= 500` early return the upper bound is always true (CodeQL "redundant comparison" + "unreachable code" on the `return "api"` fallback), but dropping the `>= 400` lower bound also routed sub-400 statuses (e.g. the `status=0` default for non-HTTP-ish errors) into `4xx_client` instead of the generic `api` bucket. Use `if status >= 400` — equivalent to the original `400 <= status < 500` given the preceding return, with the `api` fallback reachable again for status < 400. Also parenthesize the remaining sparse plan-mode reminder concatenation so CodeQL's implicit-string-concatenation check stays quiet without splitting the line across the rendered output. --- src/pythinker_code/soul/api_errors.py | 2 +- src/pythinker_code/soul/dynamic_injections/plan_mode.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pythinker_code/soul/api_errors.py b/src/pythinker_code/soul/api_errors.py index 4902d405..4af78b6f 100644 --- a/src/pythinker_code/soul/api_errors.py +++ b/src/pythinker_code/soul/api_errors.py @@ -41,7 +41,7 @@ def classify_api_error(e: Exception) -> tuple[str, int | None]: return "auth", status_code if status >= 500: return "5xx_server", status_code - if status < 500: + if status >= 400: msg_lower = str(e).lower() if any(marker in msg_lower for marker in _CONTEXT_OVERFLOW_MARKERS): return "context_overflow", status_code diff --git a/src/pythinker_code/soul/dynamic_injections/plan_mode.py b/src/pythinker_code/soul/dynamic_injections/plan_mode.py index 5cb332de..1a0db9f9 100644 --- a/src/pythinker_code/soul/dynamic_injections/plan_mode.py +++ b/src/pythinker_code/soul/dynamic_injections/plan_mode.py @@ -225,8 +225,10 @@ def _sparse_reminder(plan_file_path: str | None = None) -> str: ) parts.extend( [ - "Exit only with a decision-complete plan; " - "record unconfirmed defaults under Assumptions.", + ( + "Exit only with a decision-complete plan; " + "record unconfirmed defaults under Assumptions." + ), "Use AskUserQuestion to clarify user preferences " "when it helps you write a better plan.", "If the plan has multiple approaches, " From 6a3ecfe2b55e0a375c720a3098a2c327c4447344 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 11:42:09 -0400 Subject: [PATCH 48/49] fix: address CodeRabbit re-review on harness changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - soul/agent: when overriding a child's work dir, only replace PYTHINKER_AGENTS_MD when an explicit value is provided; None now keeps the parent payload instead of silently clearing inherited context. - soul/context: drop tool results with no tool_call_id during pairing repair. Keeping them left malformed history that re-broke the next provider request — the exact failure the repair exists to prevent. - subagents/worktree: a failed `git worktree list` no longer collapses to "not a registered worktree" (which could send the operator to delete a path holding the child's only work); raise WorktreeError with the git stderr instead. - tests/memory: annotate the `_runtime` helper return type (ANN202). --- src/pythinker_code/soul/agent.py | 9 ++++++++- src/pythinker_code/soul/context.py | 5 ++++- src/pythinker_code/subagents/worktree.py | 8 ++++++-- tests/tools/test_memory_tool.py | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/pythinker_code/soul/agent.py b/src/pythinker_code/soul/agent.py index 7b9c9845..43702c01 100644 --- a/src/pythinker_code/soul/agent.py +++ b/src/pythinker_code/soul/agent.py @@ -406,7 +406,14 @@ def copy_for_subagent( """ builtin_args = self.builtin_args if work_dir_override is not None: - agents_md = work_dir_agents_md or "" + # An explicit value (including "") replaces the payload for the + # child's work dir; None means "not provided" and keeps the parent's + # so inherited policy/instruction context is never silently dropped. + agents_md = ( + work_dir_agents_md + if work_dir_agents_md is not None + else builtin_args.PYTHINKER_AGENTS_MD + ) builtin_args = replace( builtin_args, PYTHINKER_WORK_DIR=work_dir_override, diff --git a/src/pythinker_code/soul/context.py b/src/pythinker_code/soul/context.py index 3e491e68..16978531 100644 --- a/src/pythinker_code/soul/context.py +++ b/src/pythinker_code/soul/context.py @@ -57,7 +57,10 @@ def _synthesize_lost_results() -> None: for message in history: if message.role == "tool": if message.tool_call_id is None: - repaired.append(message) + # An unpaired tool result cannot satisfy the call/result pairing + # invariant this repair enforces; keeping it would re-break the + # next provider request, so drop it like an orphan. + logger.warning("Context repair: dropping tool result without tool_call_id") elif message.tool_call_id in open_call_ids: open_call_ids.remove(message.tool_call_id) repaired.append(message) diff --git a/src/pythinker_code/subagents/worktree.py b/src/pythinker_code/subagents/worktree.py index 50c5cc06..cab530ed 100644 --- a/src/pythinker_code/subagents/worktree.py +++ b/src/pythinker_code/subagents/worktree.py @@ -53,9 +53,13 @@ async def _git(args: list[str], cwd: Path) -> tuple[int, str, str]: async def _is_registered_worktree(repo_dir: Path, dest: Path) -> bool: - code, stdout, _ = await _git(["worktree", "list", "--porcelain"], repo_dir) + code, stdout, stderr = await _git(["worktree", "list", "--porcelain"], repo_dir) if code != 0: - return False + # A git failure here is not evidence that dest is unregistered. Collapsing + # it into "not a worktree" would let the caller delete a path that may + # still hold the child's only work; surface the real error instead. + first_line = stderr.splitlines()[0] if stderr else "unknown git error" + raise WorktreeError(f"could not verify isolation worktree at {dest}: {first_line}") wanted = str(dest.resolve(strict=False)) for line in stdout.splitlines(): if not line.startswith("worktree "): diff --git a/tests/tools/test_memory_tool.py b/tests/tools/test_memory_tool.py index ee948b85..9565e4db 100644 --- a/tests/tools/test_memory_tool.py +++ b/tests/tools/test_memory_tool.py @@ -24,7 +24,7 @@ async def __call__(self, argv): return GitResult(ok=True, exit_code=1, stdout="") -def _runtime(tmp_path, role="root", work_dir=None): +def _runtime(tmp_path, role="root", work_dir=None) -> SimpleNamespace: session = SimpleNamespace(id="sess1", title="t", work_dir=_hp(tmp_path / "repo")) return SimpleNamespace( role=role, From e14bd5201ed6e160f08696fc388633963a4b38ac Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy <moelkholy1995@gmail.com> Date: Fri, 12 Jun 2026 11:58:41 -0400 Subject: [PATCH 49/49] test(context): use well-formed tool pairs in pending-token fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drop of id-less tool results during pairing repair (previous commit) correctly removes malformed history, but three pending-token tests fed bare `tool` messages (no tool_call_id, no opening assistant tool call) as token ballast through the restore/repair path, so they now under-counted. Production tool results always carry the originating tool_call_id, so model the fixtures realistically: an assistant message that opens a tool call plus a paired tool result, both after the last `_usage`. They survive pairing repair and keep the pending estimate intact — exercising the post-`_usage` slice accounting without depending on malformed history. --- tests/core/test_context_pending_tokens.py | 44 +++++++++++++++++------ 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/tests/core/test_context_pending_tokens.py b/tests/core/test_context_pending_tokens.py index d7ba4365..3ce8ef46 100644 --- a/tests/core/test_context_pending_tokens.py +++ b/tests/core/test_context_pending_tokens.py @@ -11,7 +11,7 @@ from pathlib import Path import pytest -from pythinker_core.message import Message, Role +from pythinker_core.message import Message, Role, ToolCall from pythinker_code.soul.compaction import estimate_text_tokens, should_auto_compact from pythinker_code.soul.context import Context @@ -22,6 +22,26 @@ def _msg(role: Role, text: str) -> Message: return Message(role=role, content=[TextPart(text=text)]) +def _assistant_call(text: str, call_id: str) -> Message: + """A well-formed assistant message that opens a tool call (id ``call_id``).""" + return Message( + role="assistant", + content=[TextPart(text=text)], + tool_calls=[ + ToolCall(id=call_id, function=ToolCall.FunctionBody(name="ReadFile", arguments="{}")) + ], + ) + + +def _tool_result(text: str, call_id: str) -> Message: + """A well-formed tool result paired to ``call_id`` (survives pairing repair).""" + return Message(role="tool", content=[TextPart(text=text)], tool_call_id=call_id) + + +def _dict(msg: Message) -> dict: + return json.loads(msg.model_dump_json(exclude_none=True)) + + def _write_lines(path: Path, lines: list[dict]) -> None: path.write_text( "".join(json.dumps(line) + "\n" for line in lines), @@ -169,8 +189,8 @@ async def test_revert_to_rebuilds_pending_from_messages_after_usage(tmp_path: Pa [ _message_dict("user", "question"), {"role": "_usage", "token_count": 5000}, - _message_dict("assistant", "let me check"), - _message_dict("tool", tool_text), + _dict(_assistant_call("let me check", "tc0")), + _dict(_tool_result(tool_text, "tc0")), {"role": "_checkpoint", "id": 0}, _message_dict("user", "follow up"), {"role": "_checkpoint", "id": 1}, @@ -184,8 +204,8 @@ async def test_revert_to_rebuilds_pending_from_messages_after_usage(tmp_path: Pa # After revert to checkpoint 1: assistant, tool, and "follow up" are all after _usage expected_pending = estimate_text_tokens( [ - _msg("assistant", "let me check"), - _msg("tool", tool_text), + _assistant_call("let me check", "tc0"), + _tool_result(tool_text, "tc0"), _msg("user", "follow up"), ] ) @@ -248,8 +268,8 @@ async def test_restore_rebuilds_pending_for_messages_after_usage(tmp_path: Path) [ _message_dict("user", "hello"), {"role": "_usage", "token_count": 10000}, - _message_dict("assistant", "let me read that file"), - _message_dict("tool", tool_text), + _dict(_assistant_call("let me read that file", "tc0")), + _dict(_tool_result(tool_text, "tc0")), ], ) @@ -258,8 +278,8 @@ async def test_restore_rebuilds_pending_for_messages_after_usage(tmp_path: Path) expected_pending = estimate_text_tokens( [ - _msg("assistant", "let me read that file"), - _msg("tool", tool_text), + _assistant_call("let me read that file", "tc0"), + _tool_result(tool_text, "tc0"), ] ) assert ctx.token_count == 10000 @@ -383,11 +403,13 @@ async def test_pending_rebuilt_on_restore(tmp_path: Path) -> None: path = tmp_path / "ctx.jsonl" path.touch() - tool_msg = _msg("tool", "b" * 800) + assistant_msg = _assistant_call("let me check", "tc0") + tool_msg = _tool_result("b" * 800, "tc0") ctx1 = Context(file_backend=path) await ctx1.append_message(_msg("user", "a" * 400)) await ctx1.update_token_count(1000) + await ctx1.append_message(assistant_msg) await ctx1.append_message(tool_msg) assert ctx1.token_count_with_pending > 1000 @@ -395,7 +417,7 @@ async def test_pending_rebuilt_on_restore(tmp_path: Path) -> None: ctx2 = Context(file_backend=path) await ctx2.restore() assert ctx2.token_count == 1000 - expected_pending = estimate_text_tokens([tool_msg]) + expected_pending = estimate_text_tokens([assistant_msg, tool_msg]) assert ctx2.token_count_with_pending == 1000 + expected_pending