Added failed flag to ModelResponse.

Author: JoelNiklaus

src/lighteval/metrics/llm_as_judge.py

@@ -206,18 +206,20 @@ def __call_api(prompt):
                     }
                     response = litellm.completion(**kwargs)
                     text = response.choices[0].message.content
-                    if text is None:
+                    if not text or response.failed:
                         kwargs["caching"] = False
                         response = litellm.completion(**kwargs)
                         text = response.choices[0].message.content
-                        if text is None:
+                        if not text or response.failed:
                             # Just return an error response if the second attempt fails too
-                            return ModelResponse(text="Failed to get response from the API.", model=self.model)
+                            return ModelResponse(
+                                text="Failed to get response from the API.", model=self.model, failed=True
+                            )
                     return text
                 except Exception as e:
                     logger.warning(f"{type(e), e}")
                     time.sleep(self.API_RETRY_SLEEP)
-            return ModelResponse(text="Failed to get response from the API.", model=self.model)
+            return ModelResponse(text="Failed to get response from the API.", model=self.model, failed=True)
 
         results = []
         with ThreadPoolExecutor(100) as executor:

src/lighteval/models/model_output.py

@@ -33,6 +33,7 @@ class ModelResponse:
     generated_tokens: list[int] = field(default_factory=list)  # model generations
     truncated_tokens_count: Optional[int] = 0  # How many tokens truncated
     padded_tokens_count: Optional[int] = 0  # How many tokens of padding
+    failed: bool = False
 
     def get_result_for_eval(self):
         raise NotImplementedError()