fix(simulation): Add error message when LlmBackedUserSimulator returns empty response

PiperOrigin-RevId: 911999966

Author: google-genai-bot

src/google/adk/evaluation/simulation/llm_backed_user_simulator.py

@@ -16,7 +16,6 @@
 
 import logging
 from typing import ClassVar
-from typing import Optional
 
 from google.genai import types as genai_types
 from pydantic import Field
@@ -72,7 +71,7 @@ class LlmBackedUserSimulatorConfig(BaseUserSimulatorConfig):
 (Not recommended) If you don't want a limit, you can set the value to -1.""",
   )
 
-  custom_instructions: Optional[str] = Field(
+  custom_instructions: str | None = Field(
       default=None,
       description="""Custom instructions for the LlmBackedUserSimulator. The
 instructions must contain the following formatting placeholders following Jinja syntax:
@@ -88,7 +87,7 @@ class LlmBackedUserSimulatorConfig(BaseUserSimulatorConfig):
 
   @field_validator("custom_instructions")
   @classmethod
-  def validate_custom_instructions(cls, value: Optional[str]) -> Optional[str]:
+  def validate_custom_instructions(cls, value: str | None) -> str | None:
     if value is None:
       return value
     if not is_valid_user_simulator_template(
@@ -158,11 +157,11 @@ def _summarize_conversation(
   async def _get_llm_response(
       self,
       rewritten_dialogue: str,
-  ) -> str:
-    """Sends a user message generation request to the LLM and returns the full response."""
+  ) -> tuple[str, str | None]:
+    """Sends a user message generation request to the LLM and returns the full response and potential error reason."""
     if self._invocation_count == 0:
       # first invocation - send the static starting prompt
-      return self._conversation_scenario.starting_prompt
+      return self._conversation_scenario.starting_prompt, None
 
     user_agent_instructions = get_llm_backed_user_simulator_prompt(
         conversation_plan=self._conversation_scenario.conversation_plan,
@@ -187,19 +186,44 @@ async def _get_llm_response(
     add_default_retry_options_if_not_present(llm_request)
 
     response = ""
+    error_reason = None
+    has_thought_tokens = False
     async with Aclosing(self._llm.generate_content_async(llm_request)) as agen:
       async for llm_response in agen:
+        error_code = llm_response.error_code
+        if error_code:
+          logger.warning(
+              "User simulator LLM returned error: code=%s, message=%s",
+              error_code,
+              getattr(llm_response, "error_message", ""),
+          )
+          error_reason = f"safety filters or other error (code={error_code})"
+          response = ""
+          break
+
         generated_content: genai_types.Content = llm_response.content
         if (
             not generated_content
             or not hasattr(generated_content, "parts")
             or not generated_content.parts
         ):
           continue
+
         for part in generated_content.parts:
-          if part.text and not part.thought:
+          if part.thought:
+            has_thought_tokens = True
+          elif part.text:
             response += part.text
-    return response
+
+    if not response:
+      if error_reason:
+        pass  # Keep the error reason from error_code
+      elif has_thought_tokens:
+        error_reason = "LLM returned only thinking tokens"
+      else:
+        error_reason = "LLM returned empty response"
+
+    return response, error_reason
 
   @override
   async def get_next_user_message(
@@ -234,11 +258,11 @@ async def get_next_user_message(
     rewritten_dialogue = self._summarize_conversation(events)
 
     # query the LLM for the next user message
-    response = await self._get_llm_response(rewritten_dialogue)
+    response, error_reason = await self._get_llm_response(rewritten_dialogue)
     self._invocation_count += 1
 
     # is the conversation over? (Has the user simulator output the stop signal?)
-    if _STOP_SIGNAL.lower() in response.lower():
+    if response and _STOP_SIGNAL.lower() in response.lower():
       logger.info(
           "Stopping user message generation as the stop signal was detected."
       )
@@ -256,11 +280,11 @@ async def get_next_user_message(
 
     # if we are here, the user agent failed to generate a message, which is not
     # a valid result for the LLM backed user simulator.
-    raise RuntimeError("Failed to generate a user message")
+    raise RuntimeError(f"Failed to generate a user message: {error_reason}")
 
   @override
   def get_simulation_evaluator(
       self,
-  ) -> Optional[Evaluator]:
+  ) -> Evaluator | None:
     """Returns an Evaluator that evaluates if the simulation was successful or not."""
     raise NotImplementedError()

tests/unittests/evaluation/simulation/test_llm_backed_user_simulator.py

@@ -129,7 +129,8 @@ async def to_async_iter(items):
 def mock_llm_agent(mocker):
   """Provides a mock LLM agent."""
   mock_llm_registry_cls = mocker.patch(
-      "google.adk.evaluation.simulation.llm_backed_user_simulator.LLMRegistry"
+      "google.adk.evaluation.simulation.llm_backed_user_simulator.LLMRegistry",
+      autospec=True,
   )
   mock_llm_registry = mocker.MagicMock()
   mock_llm_registry_cls.return_value = mock_llm_registry
@@ -207,18 +208,25 @@ async def test_get_llm_response_return_value(
       self, simulator, mock_llm_agent, mocker
   ):
     """Tests that _get_llm_response returns the full response correctly."""
-    mock_llm_response = mocker.MagicMock()
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
     mock_llm_response.content = types.Content(
         parts=[
             types.Part(text="some thought", thought=True),
             types.Part(text="Hello world!"),
         ]
     )
+    mock_llm_response.parts = mock_llm_response.content.parts
     mock_llm_agent.generate_content_async.return_value = to_async_iter(
         [mock_llm_response]
     )
-    response = await simulator._get_llm_response(rewritten_dialogue="")
+    response, error_reason = await simulator._get_llm_response(
+        rewritten_dialogue=""
+    )
     assert response == "Hello world!"
+    assert error_reason is None
 
   @pytest.mark.asyncio
   async def test_get_next_user_message_first_invocation(
@@ -257,10 +265,14 @@ async def test_turn_limit_reached(self, conversation_scenario):
   @pytest.mark.asyncio
   async def test_stop_signal_detected(self, simulator, mock_llm_agent, mocker):
     """Tests get_next_user_message when the stop signal is detected."""
-    mock_llm_response = mocker.MagicMock()
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
     mock_llm_response.content = types.Content(
         parts=[types.Part(text="Thanks! Bye!</finished>")]
     )
+    mock_llm_response.parts = mock_llm_response.content.parts
     mock_llm_agent.generate_content_async.return_value = to_async_iter(
         [mock_llm_response]
     )
@@ -273,22 +285,84 @@ async def test_stop_signal_detected(self, simulator, mock_llm_agent, mocker):
     assert next_user_message.user_message is None
 
   @pytest.mark.asyncio
-  async def test_no_message_generated(self, simulator, mock_llm_agent):
-    """Tests get_next_user_message when no message is generated."""
+  async def test_no_message_generated_empty_response(
+      self, simulator, mock_llm_agent
+  ):
+    """Tests get_next_user_message when no message is generated (empty stream)."""
     mock_llm_agent.generate_content_async.return_value = to_async_iter([])
 
-    with pytest.raises(RuntimeError, match="Failed to generate a user message"):
+    with pytest.raises(
+        RuntimeError,
+        match="Failed to generate a user message: LLM returned empty response",
+    ):
+      await simulator.get_next_user_message(events=_INPUT_EVENTS)
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_safety_blocked(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests get_next_user_message when response is safety blocked."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.content = None
+    mock_llm_response.error_code = "SAFETY"
+    mock_llm_response.error_message = "Blocked by safety"
+    mock_llm_response.parts = []
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match=(
+            "Failed to generate a user message: safety filters or other error"
+            " \\(code=SAFETY\\)"
+        ),
+    ):
+      await simulator.get_next_user_message(events=_INPUT_EVENTS)
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_thinking_only(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests get_next_user_message when response contains only thinking tokens."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.content = types.Content(
+        parts=[
+            types.Part(text="thinking...", thought=True),
+        ]
+    )
+    mock_llm_response.error_code = None
+    mock_llm_response.parts = mock_llm_response.content.parts
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match=(
+            "Failed to generate a user message: LLM returned only thinking"
+            " tokens"
+        ),
+    ):
       await simulator.get_next_user_message(events=_INPUT_EVENTS)
 
   @pytest.mark.asyncio
   async def test_get_next_user_message_success(
       self, simulator, mock_llm_agent, mocker
   ):
     """Tests get_next_user_message when the user message is generated successfully."""
-    mock_llm_response = mocker.MagicMock()
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
     mock_llm_response.content = types.Content(
         parts=[types.Part(text="I need to book a flight.")]
     )
+    mock_llm_response.parts = mock_llm_response.content.parts
     mock_llm_agent.generate_content_async.return_value = to_async_iter(
         [mock_llm_response]
     )
@@ -309,10 +383,14 @@ async def test_get_next_user_message_with_persona_success(
       self, simulator_with_persona, mock_llm_agent, mocker
   ):
     """Tests get_next_user_message when the user message is generated successfully."""
-    mock_llm_response = mocker.MagicMock()
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
     mock_llm_response.content = types.Content(
         parts=[types.Part(text="I need to book a flight.")]
     )
+    mock_llm_response.parts = mock_llm_response.content.parts
     mock_llm_agent.generate_content_async.return_value = to_async_iter(
         [mock_llm_response]
     )