From 555c7e4b26cdd287ded95454eb624c1a45e6fc8c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 17 May 2026 18:34:28 -0500 Subject: [PATCH] fix: guard choices[0] and message=None before content access in llm_as_judge OpenAI-compatible APIs can return empty choices on error or content filtering; Gemini returns choices[0].message=None on PROHIBITED_CONTENT (HTTP 200). Both cause unhandled exceptions inside the retry loop. Adds guard before response.choices[0].message.content in the API call retry path so the ValueError propagates cleanly. --- src/lighteval/metrics/utils/llm_as_judge.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lighteval/metrics/utils/llm_as_judge.py b/src/lighteval/metrics/utils/llm_as_judge.py index 0f9b3315c..5776fc56d 100644 --- a/src/lighteval/metrics/utils/llm_as_judge.py +++ b/src/lighteval/metrics/utils/llm_as_judge.py @@ -436,6 +436,8 @@ def __call_api(self, prompt): max_tokens=self.max_tokens, n=1, ) + if not response.choices or response.choices[0].message is None: + raise ValueError("LLM returned empty or filtered response") text = response.choices[0].message.content return text except Exception as e: