diff --git a/js/ragas.test.ts b/js/ragas.test.ts index 9a2aaa3..e205819 100644 --- a/js/ragas.test.ts +++ b/js/ragas.test.ts @@ -327,3 +327,80 @@ describe("AnswerCorrectness custom embedding model", () => { expect(capturedEmbeddingModel).toBe("text-embedding-3-large"); }); }); + +describe("AnswerCorrectness empty classification", () => { + const server = setupServer(); + + beforeAll(() => { + server.listen({ + onUnhandledRequest: (req) => { + throw new Error(`Unhandled request ${req.method}, ${req.url}`); + }, + }); + }); + + afterEach(() => { + server.resetHandlers(); + init(); + }); + + afterAll(() => { + server.close(); + }); + + test("scores 0 when the judge classifies nothing", async () => { + server.use( + http.post("https://api.openai.com/v1/chat/completions", async () => { + return HttpResponse.json({ + id: "test-id", + object: "chat.completion", + created: Date.now(), + model: "gpt-5-mini", + choices: [ + { + index: 0, + message: { + role: "assistant", + tool_calls: [ + { + id: "call_test", + type: "function", + function: { + name: "classify_statements", + arguments: JSON.stringify({ + TP: [], + FP: [], + FN: [], + }), + }, + }, + ], + }, + finish_reason: "tool_calls", + }, + ], + }); + }), + ); + + init({ + client: new OpenAI({ + apiKey: "test-api-key", + baseURL: "https://api.openai.com/v1", + }), + }); + + // Regression test: computeF1Score divided by tp + 0.5 * (fp + fn) + // unguarded, so an empty classification returned NaN, which serializes + // to null and reads as a skipped score. answerSimilarityWeight 0 keeps + // the test off the embeddings API. + const result = await AnswerCorrectness({ + input: "What is the capital of France?", + output: "I don't know.", + expected: "Paris is the capital of France", + answerSimilarityWeight: 0, + }); + + expect(result.score).toBe(0); + }); +}); diff --git a/js/ragas.ts b/js/ragas.ts index e961460..115ec67 100644 --- a/js/ragas.ts +++ b/js/ragas.ts @@ -850,7 +850,11 @@ function computeF1Score(classification: AnswerCorrectnessClassification) { const tp = classification.TP.length; const fp = classification.FP.length; const fn = classification.FN.length; - return tp / (tp + 0.5 * (fp + fn)); + const denominator = tp + 0.5 * (fp + fn); + // A judge can classify nothing at all (an answer with no checkable claims, + // or an empty classification). Score that 0 rather than dividing by zero + // and returning NaN, matching the guard in ContextRecall. + return denominator > 0 ? tp / denominator : 0; } /** diff --git a/py/autoevals/ragas.py b/py/autoevals/ragas.py index 4726159..2790752 100644 --- a/py/autoevals/ragas.py +++ b/py/autoevals/ragas.py @@ -1341,7 +1341,11 @@ def extract_correctness_request(question, answer, ground_truth, **extra_args): def compute_f1_score(factuality): tp, fp, fn = len(factuality["TP"]), len(factuality["FP"]), len(factuality["FN"]) - return tp / (tp + 0.5 * (fp + fn)) + denominator = tp + 0.5 * (fp + fn) + # A judge can classify nothing at all (an answer with no checkable claims, + # or an empty classification). Score that 0 rather than dividing by zero, + # matching the guard in ContextRecall._postprocess. + return tp / denominator if denominator > 0 else 0 class AnswerCorrectness(OpenAILLMScorer): diff --git a/py/autoevals/test_ragas.py b/py/autoevals/test_ragas.py index 74db4e5..44086fc 100644 --- a/py/autoevals/test_ragas.py +++ b/py/autoevals/test_ragas.py @@ -262,3 +262,47 @@ def mock_responses_api(request): ) assert captured_embedding_model == "text-embedding-3-large" + + +def test_compute_f1_score_empty_classification(): + """An empty classification has no correct statements, so it scores 0. + + Regression test: compute_f1_score divided by tp + 0.5 * (fp + fn) + unguarded, so a judge that classified nothing raised ZeroDivisionError. + """ + assert ragas_module.compute_f1_score({"TP": [], "FP": [], "FN": []}) == 0 + # Normal cases are unchanged. + assert ragas_module.compute_f1_score({"TP": ["a", "b"], "FP": ["c"], "FN": []}) == 0.8 + assert ragas_module.compute_f1_score({"TP": [], "FP": ["a"], "FN": ["b"]}) == 0 + + +@pytest.mark.parametrize("is_async", [False, True]) +def test_answer_correctness_empty_classification_scores_zero(monkeypatch, is_async): + """A judge that classifies nothing scores 0 instead of crashing. + + The TypeScript scorer returned NaN for the same input, which serializes + to null and reads as a skipped score. + """ + + def fake_load_function_call_request(client=None, **kwargs): + return {"TP": [], "FP": [], "FN": []} + + async def fake_aload_function_call_request(client=None, **kwargs): + return {"TP": [], "FP": [], "FN": []} + + monkeypatch.setattr(ragas_module, "load_function_call_request", fake_load_function_call_request) + monkeypatch.setattr(ragas_module, "aload_function_call_request", fake_aload_function_call_request) + + # answer_similarity_weight=0 keeps the test off the embeddings API. + metric = AnswerCorrectness(answer_similarity_weight=0) + kwargs = dict( + input="What is the capital of France?", + output="I don't know.", + expected="Paris is the capital of France", + ) + if is_async: + score = asyncio.run(metric.eval_async(**kwargs)) + else: + score = metric.eval(**kwargs) + + assert score.score == 0