Implement P2-T8: back off broker tools probe retries

Author: SoundBlaster

src/mcpbridge_wrapper/broker/daemon.py

@@ -43,7 +43,8 @@
 # broker_id is 1_048_576.  These negative/zero values can never collide.
 _BROKER_INIT_ID = 0
 _BROKER_TOOLS_ID = -1
-_TOOLS_PROBE_RETRY_DELAY_SECONDS = 0.25
+_TOOLS_PROBE_RETRY_BASE_DELAY_SECONDS = 0.25
+_TOOLS_PROBE_RETRY_MAX_DELAY_SECONDS = 2.0
 
 
 class BrokerDaemon:
@@ -95,6 +96,7 @@ def __init__(
         self._tools_catalog_ready: asyncio.Event = asyncio.Event()
         # Background retry task for broker-internal tools/list warm-up probes.
         self._tools_probe_retry_task: asyncio.Task[None] | None = None
+        self._tools_probe_retry_attempt: int = 0
 
     # ------------------------------------------------------------------
     # Public API
@@ -398,12 +400,26 @@ def _schedule_tools_list_probe(self, *, delay: float = 0.0) -> None:
             self._retry_tools_list_probe_after_delay(delay)
         )
 
+    def _reset_tools_probe_retry_backoff(self) -> None:
+        """Reset retry state for broker-internal tools/list warm-up probing."""
+        self._tools_probe_retry_attempt = 0
+
+    def _next_tools_probe_retry_delay(self) -> float:
+        """Return the next bounded backoff delay for broker tools/list retries."""
+        delay = min(
+            _TOOLS_PROBE_RETRY_BASE_DELAY_SECONDS * (2**self._tools_probe_retry_attempt),
+            _TOOLS_PROBE_RETRY_MAX_DELAY_SECONDS,
+        )
+        self._tools_probe_retry_attempt += 1
+        return float(delay)
+
     def _cancel_tools_probe_retry(self) -> None:
         """Cancel any pending retry for the broker-internal tools/list probe."""
         task = self._tools_probe_retry_task
         if task is not None and not task.done():
             task.cancel()
         self._tools_probe_retry_task = None
+        self._reset_tools_probe_retry_backoff()
 
     async def _rollback_startup(self) -> None:
         """Roll back a failed :meth:`start` sequence.
@@ -565,6 +581,7 @@ async def _read_upstream_loop(self) -> None:
                     # Now fetch tools/list for the cache.
                     upstream = self._upstream
                     if upstream is not None and upstream.stdin is not None:
+                        self._reset_tools_probe_retry_backoff()
                         self._schedule_tools_list_probe()
                     continue
 
@@ -585,18 +602,32 @@ async def _read_upstream_loop(self) -> None:
                         else:
                             self._tools_list_cache = None
                             self._tools_catalog_ready.clear()
-                            logger.warning(
+                            delay = self._next_tools_probe_retry_delay()
+                            log_fn = (
+                                logger.warning
+                                if self._tools_probe_retry_attempt == 1
+                                else logger.debug
+                            )
+                            log_fn(
                                 "Broker tools/list probe returned an empty or invalid "
-                                "tool catalog; retrying warm-up probe."
+                                "tool catalog; retry %d in %.2fs.",
+                                self._tools_probe_retry_attempt,
+                                delay,
                             )
-                            self._schedule_tools_list_probe(delay=_TOOLS_PROBE_RETRY_DELAY_SECONDS)
+                            self._schedule_tools_list_probe(delay=delay)
                     else:
                         self._tools_list_cache = None
                         self._tools_catalog_ready.clear()
-                        logger.warning(
-                            "Broker tools/list probe returned no result; retrying warm-up probe."
+                        delay = self._next_tools_probe_retry_delay()
+                        log_fn = (
+                            logger.warning if self._tools_probe_retry_attempt == 1 else logger.debug
+                        )
+                        log_fn(
+                            "Broker tools/list probe returned no result; retry %d in %.2fs.",
+                            self._tools_probe_retry_attempt,
+                            delay,
                         )
-                        self._schedule_tools_list_probe(delay=_TOOLS_PROBE_RETRY_DELAY_SECONDS)
+                        self._schedule_tools_list_probe(delay=delay)
                     continue
 
                 if self._transport is not None:

tests/unit/test_broker_daemon.py

@@ -1335,7 +1335,10 @@ def _write(data: bytes) -> None:
         proc.stdin.write = MagicMock(side_effect=_write)
 
         with patch(
-            "mcpbridge_wrapper.broker.daemon._TOOLS_PROBE_RETRY_DELAY_SECONDS",
+            "mcpbridge_wrapper.broker.daemon._TOOLS_PROBE_RETRY_BASE_DELAY_SECONDS",
+            0.0,
+        ), patch(
+            "mcpbridge_wrapper.broker.daemon._TOOLS_PROBE_RETRY_MAX_DELAY_SECONDS",
             0.0,
         ), patch(
             "mcpbridge_wrapper.broker.daemon.asyncio.create_subprocess_exec",
@@ -1350,6 +1353,25 @@ def _write(data: bytes) -> None:
         assert daemon.tools_catalog_ready.is_set()
         assert sum('"method":"tools/list"' in msg for msg in sent_messages) == 2
 
+    def test_tools_list_probe_retry_backoff_is_bounded_and_resets(self, tmp_path: Path) -> None:
+        """Retry delays should back off and reset after cancellation/success transitions."""
+        cfg = _make_config(tmp_path)
+        daemon = BrokerDaemon(cfg)
+
+        with patch(
+            "mcpbridge_wrapper.broker.daemon._TOOLS_PROBE_RETRY_BASE_DELAY_SECONDS",
+            0.25,
+        ), patch(
+            "mcpbridge_wrapper.broker.daemon._TOOLS_PROBE_RETRY_MAX_DELAY_SECONDS",
+            1.0,
+        ):
+            assert daemon._next_tools_probe_retry_delay() == 0.25
+            assert daemon._next_tools_probe_retry_delay() == 0.5
+            assert daemon._next_tools_probe_retry_delay() == 1.0
+            assert daemon._next_tools_probe_retry_delay() == 1.0
+            daemon._cancel_tools_probe_retry()
+            assert daemon._next_tools_probe_retry_delay() == 0.25
+
     @pytest.mark.asyncio
     async def test_upstream_initialized_cleared_on_reconnect(self, tmp_path: Path) -> None:
         """_upstream_initialized is cleared at the start of _reconnect()."""