diff --git a/py/autoevals/ragas.py b/py/autoevals/ragas.py index 4726159..911d81f 100644 --- a/py/autoevals/ragas.py +++ b/py/autoevals/ragas.py @@ -1127,7 +1127,8 @@ class AnswerRelevancy(OpenAILLMScorer): output: The generated answer to evaluate context: The context document(s) to evaluate against strictness: Optional float between 0-1, higher values enforce stricter relevancy - temperature: Optional float between 0-1, lower values make evaluation more deterministic + temperature: Optional float between 0-1, lower values make evaluation more deterministic. + If not specified, uses the model's default embedding_model: Optional model to use for embeddings, defaults to text-embedding-3-small """ @@ -1135,7 +1136,7 @@ def __init__( self, model: str | None = None, strictness=3, - temperature=0.5, + temperature=None, embedding_model=None, client: Client | None = None, **kwargs, diff --git a/py/autoevals/test_ragas.py b/py/autoevals/test_ragas.py index 74db4e5..40d022d 100644 --- a/py/autoevals/test_ragas.py +++ b/py/autoevals/test_ragas.py @@ -262,3 +262,18 @@ def mock_responses_api(request): ) assert captured_embedding_model == "text-embedding-3-large" + + +def test_answer_relevancy_does_not_send_a_default_temperature(): + # Claude 4.7+ and 5.x reject a non-default temperature, so the request only carries one the caller asked for. + scorer = AnswerRelevancy(model="claude-opus-5-5", client=object()) + request = ragas_module.extract_question_gen_request( + answer="Paris", context="France", model=scorer.model, **scorer.extra_args + ) + assert "temperature" not in request + + scorer = AnswerRelevancy(model="gpt-4o", temperature=0.2, client=object()) + request = ragas_module.extract_question_gen_request( + answer="Paris", context="France", model=scorer.model, **scorer.extra_args + ) + assert request["temperature"] == 0.2