From 98576fd65e5d7938b1dac7fb14d785022efc0665 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Wed, 21 Oct 2026 18:55:25 -0400 Subject: [PATCH] fix(python): do not send a default temperature from AnswerRelevancy AnswerRelevancy defaulted temperature to 0.5 and always sent it, so a judge model that rejects a non-default temperature (Claude 4.7+ and 5.x) failed with a 400 unless the caller knew to pass one it accepts. Default it to None and send it only when given, as the other scorers in llm.py do ("If not specified, uses the model's default"). Co-Authored-By: Claude Sonnet 5.5 --- py/autoevals/ragas.py | 5 +++-- py/autoevals/test_ragas.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/py/autoevals/ragas.py b/py/autoevals/ragas.py index 4726159..911d81f 100644 --- a/py/autoevals/ragas.py +++ b/py/autoevals/ragas.py @@ -1127,7 +1127,8 @@ class AnswerRelevancy(OpenAILLMScorer): output: The generated answer to evaluate context: The context document(s) to evaluate against strictness: Optional float between 0-1, higher values enforce stricter relevancy - temperature: Optional float between 0-1, lower values make evaluation more deterministic + temperature: Optional float between 0-1, lower values make evaluation more deterministic. + If not specified, uses the model's default embedding_model: Optional model to use for embeddings, defaults to text-embedding-3-small """ @@ -1135,7 +1136,7 @@ def __init__( self, model: str | None = None, strictness=3, - temperature=0.5, + temperature=None, embedding_model=None, client: Client | None = None, **kwargs, diff --git a/py/autoevals/test_ragas.py b/py/autoevals/test_ragas.py index 74db4e5..40d022d 100644 --- a/py/autoevals/test_ragas.py +++ b/py/autoevals/test_ragas.py @@ -262,3 +262,18 @@ def mock_responses_api(request): ) assert captured_embedding_model == "text-embedding-3-large" + + +def test_answer_relevancy_does_not_send_a_default_temperature(): + # Claude 4.7+ and 5.x reject a non-default temperature, so the request only carries one the caller asked for. + scorer = AnswerRelevancy(model="claude-opus-5-5", client=object()) + request = ragas_module.extract_question_gen_request( + answer="Paris", context="France", model=scorer.model, **scorer.extra_args + ) + assert "temperature" not in request + + scorer = AnswerRelevancy(model="gpt-4o", temperature=0.2, client=object()) + request = ragas_module.extract_question_gen_request( + answer="Paris", context="France", model=scorer.model, **scorer.extra_args + ) + assert request["temperature"] == 0.2