|
26 | 26 | from pythinker_code.soul.agent import Agent, BuiltinSystemPromptArgs, Runtime |
27 | 27 | from pythinker_code.soul.context import Context |
28 | 28 | from pythinker_code.soul.pythinkersoul import PythinkerSoul, TurnStopReason |
| 29 | +from pythinker_code.soul.toolset import PythinkerToolset |
29 | 30 | from pythinker_code.utils.aioqueue import QueueShutDown |
30 | 31 | from pythinker_code.wire import Wire |
31 | 32 |
|
@@ -159,6 +160,43 @@ def _make_soul( |
159 | 160 | return context, soul |
160 | 161 |
|
161 | 162 |
|
| 163 | +def _make_soul_with_pythinker_toolset( |
| 164 | + runtime: Runtime, provider: _ScriptedToolCallProvider, tmp_path: Path |
| 165 | +) -> tuple[Context, PythinkerSoul]: |
| 166 | + """Like `_make_soul`, but with a real `PythinkerToolset` — required to exercise the |
| 167 | + identical-call repeat backstop, which is tracked on `PythinkerToolset` specifically.""" |
| 168 | + llm = LLM(chat_provider=provider, max_context_size=100_000, capabilities=set()) |
| 169 | + runtime = Runtime( |
| 170 | + config=runtime.config, |
| 171 | + llm=llm, |
| 172 | + session=runtime.session, |
| 173 | + builtin_args=runtime.builtin_args, |
| 174 | + denwa_renji=runtime.denwa_renji, |
| 175 | + approval=runtime.approval, |
| 176 | + labor_market=runtime.labor_market, |
| 177 | + environment=runtime.environment, |
| 178 | + notifications=runtime.notifications, |
| 179 | + background_tasks=runtime.background_tasks, |
| 180 | + skills=runtime.skills, |
| 181 | + oauth=runtime.oauth, |
| 182 | + additional_dirs=runtime.additional_dirs, |
| 183 | + skills_dirs=runtime.skills_dirs, |
| 184 | + role=runtime.role, |
| 185 | + ) |
| 186 | + toolset = PythinkerToolset() |
| 187 | + toolset.add(_BoomTool()) |
| 188 | + toolset.add(_OkTool()) |
| 189 | + agent = Agent( |
| 190 | + name="Stuck Test Agent", |
| 191 | + system_prompt="Stuck test prompt.", |
| 192 | + toolset=toolset, |
| 193 | + runtime=runtime, |
| 194 | + ) |
| 195 | + context = Context(file_backend=tmp_path / "history.jsonl") |
| 196 | + soul = PythinkerSoul(agent, context=context) |
| 197 | + return context, soul |
| 198 | + |
| 199 | + |
162 | 200 | async def _drain_ui_messages(wire: Wire) -> None: |
163 | 201 | wire_ui = wire.ui_side(merge=True) |
164 | 202 | while True: |
@@ -506,6 +544,44 @@ async def test_max_consecutive_failures_zero_disables_backstop( |
506 | 544 | assert record_turn.call_args.kwargs["stop_reason"] == "no_tool_calls" |
507 | 545 |
|
508 | 546 |
|
| 547 | +@pytest.mark.asyncio |
| 548 | +async def test_consecutive_identical_calls_yield_stuck_outcome( |
| 549 | + runtime: Runtime, tmp_path: Path |
| 550 | +) -> None: |
| 551 | + """N consecutive identical-argument tool calls stop the turn with `stuck`, even |
| 552 | + though every call reports success — this backstop is independent of the |
| 553 | + all-error check above, so it still catches a tool that falsely reports success |
| 554 | + on a call that made no progress.""" |
| 555 | + runtime.config.loop_control.max_consecutive_identical_calls = 3 |
| 556 | + runtime.config.loop_control.max_steps_per_turn = 50 |
| 557 | + provider = _ScriptedToolCallProvider(["Ok"] * 10) |
| 558 | + context, soul = _make_soul_with_pythinker_toolset(runtime, provider, tmp_path) |
| 559 | + |
| 560 | + with patch("pythinker_code.telemetry.metrics.record_turn") as record_turn: |
| 561 | + await run_soul(soul, "go", _drain_ui_messages, asyncio.Event()) |
| 562 | + |
| 563 | + assert provider.generate_attempts == 3 |
| 564 | + assert record_turn.call_args.kwargs["stop_reason"] == "stuck" |
| 565 | + assert "identical" in context.history[-1].extract_text(" ").lower() |
| 566 | + |
| 567 | + |
| 568 | +@pytest.mark.asyncio |
| 569 | +async def test_max_consecutive_identical_calls_zero_disables_backstop( |
| 570 | + runtime: Runtime, tmp_path: Path |
| 571 | +) -> None: |
| 572 | + """A threshold of 0 disables the identical-call backstop entirely.""" |
| 573 | + runtime.config.loop_control.max_consecutive_identical_calls = 0 |
| 574 | + runtime.config.loop_control.max_steps_per_turn = 50 |
| 575 | + provider = _ScriptedToolCallProvider(["Ok", "Ok", "Ok", "Ok", None]) |
| 576 | + context, soul = _make_soul_with_pythinker_toolset(runtime, provider, tmp_path) |
| 577 | + |
| 578 | + with patch("pythinker_code.telemetry.metrics.record_turn") as record_turn: |
| 579 | + await run_soul(soul, "go", _drain_ui_messages, asyncio.Event()) |
| 580 | + |
| 581 | + assert provider.generate_attempts == 5 |
| 582 | + assert record_turn.call_args.kwargs["stop_reason"] == "no_tool_calls" |
| 583 | + |
| 584 | + |
509 | 585 | @pytest.mark.asyncio |
510 | 586 | async def test_truncated_response_nudges_continuation(runtime: Runtime, tmp_path: Path) -> None: |
511 | 587 | """A response cut off by the output-token limit (no tool calls) nudges the model to |
|
0 commit comments