diff --git a/js/llm.test.ts b/js/llm.test.ts index 5863eec..d5d4fc0 100644 --- a/js/llm.test.ts +++ b/js/llm.test.ts @@ -6,6 +6,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest"; import { Battle, buildClassificationTools, + DEFAULT_MODEL, LLMClassifierFromTemplate, OpenAIClassifier, templateUsesThreadVariables, @@ -65,6 +66,10 @@ afterAll(() => { }); describe("LLM Tests", () => { + test("exports the default scorer model", () => { + expect(DEFAULT_MODEL).toBe("gpt-6-luna"); + }); + test("templateUsesThreadVariables recognizes thread_with_system", () => { expect(templateUsesThreadVariables("{{thread_with_system}}")).toBe(true); expect( diff --git a/js/llm.ts b/js/llm.ts index 1b7f318..1e57232 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -86,7 +86,7 @@ export type LLMArgs = { * The default model to use for LLM-based evaluations. * @deprecated Use `init({ defaultModel: "..." })` to configure the default model instead. */ -export const DEFAULT_MODEL = "gpt-5-mini"; +export const DEFAULT_MODEL = "gpt-6-luna"; const PLAIN_RESPONSE_SCHEMA = { properties: { diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index b0093cb..1a38712 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -88,7 +88,7 @@ ) # Deprecated: Use init(default_model="...") to configure the default model instead. -DEFAULT_MODEL = "gpt-5-mini" +DEFAULT_MODEL = "gpt-6-luna" PLAIN_RESPONSE_SCHEMA = { "properties": {"choice": {"description": "The choice", "title": "Choice", "type": "string"}}, diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index 8e987ef..2bcecef 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -9,7 +9,14 @@ from pydantic import BaseModel from autoevals import init -from autoevals.llm import Battle, Factuality, LLMClassifier, OpenAILLMClassifier, build_classification_tools +from autoevals.llm import ( + DEFAULT_MODEL, + Battle, + Factuality, + LLMClassifier, + OpenAILLMClassifier, + build_classification_tools, +) from autoevals.oai import OpenAIV1Module, get_default_model from autoevals.thread_utils import compute_thread_template_vars, template_uses_thread_variables @@ -19,6 +26,10 @@ class TestModel(BaseModel): num: int +def test_default_scorer_model(): + assert DEFAULT_MODEL == "gpt-6-luna" + + def test_render_messages(): classifier = OpenAILLMClassifier( "test",