diff --git a/SCORERS.md b/SCORERS.md index 5689324..8e96890 100644 --- a/SCORERS.md +++ b/SCORERS.md @@ -25,7 +25,7 @@ Evaluates whether the output is factually consistent with the expected answer. - `input` (string): The input question or prompt - `output` (string, required): The generated answer to evaluate - `expected` (string, required): The ground truth answer -- `model` (string, optional): Model to use (default: configured via `init()` or "gpt-5-mini") +- `model` (string, optional): Model to use (default: configured via `init()` or "gpt-5.6-terra") - `client` (Client, optional): Custom OpenAI client **Score Range:** 0-1 @@ -209,7 +209,7 @@ Evaluates how relevant the retrieved context is to the input question. - `input` (string, required): The question - `output` (string, required): The generated answer - `context` (string[] | string, required): Retrieved context passages -- `model` (string, optional): Model to use (default: "gpt-5-nano") +- `model` (string, optional): Model to use (default: "gpt-5.6-luna") **Score Range:** 0-1 @@ -600,7 +600,7 @@ Note: Interpretation varies by scorer type. Binary scorers (ExactMatch, ValidJSO Many scorers share these common parameters: -- `model` (string): LLM model to use for evaluation (default: configured via `init()` or "gpt-5-mini") +- `model` (string): LLM model to use for evaluation (default: configured via `init()` or "gpt-5.6-terra") - `client` (Client): Custom OpenAI-compatible client - `use_cot` (boolean): Enable chain-of-thought reasoning for LLM scorers (default: true) - `temperature` (number): LLM temperature setting diff --git a/js/init-models.test.ts b/js/init-models.test.ts index 65668ee..5f442df 100644 --- a/js/init-models.test.ts +++ b/js/init-models.test.ts @@ -36,7 +36,7 @@ describe("init with defaultModel parameter", () => { expect(getDefaultEmbeddingModel()).toBe("text-embedding-3-large"); // Completion model should remain at default since we didn't update it - expect(getDefaultModel()).toBe("gpt-5-mini"); + expect(getDefaultModel()).toBe("gpt-5.6-terra"); }); test("object form can set both models", () => { @@ -76,7 +76,7 @@ describe("init with defaultModel parameter", () => { test("falls back to defaults when not set", () => { init(); - expect(getDefaultModel()).toBe("gpt-5-mini"); + expect(getDefaultModel()).toBe("gpt-5.6-terra"); expect(getDefaultEmbeddingModel()).toBe("text-embedding-ada-002"); }); diff --git a/js/oai.test.ts b/js/oai.test.ts index 5adf9ba..b521fa0 100644 --- a/js/oai.test.ts +++ b/js/oai.test.ts @@ -332,8 +332,8 @@ describe("OAI", () => { expect(Object.is(builtClient, otherClient)).toBe(true); }); - test("getDefaultModel returns gpt-5-mini by default", () => { - expect(getDefaultModel()).toBe("gpt-5-mini"); + test("getDefaultModel returns gpt-5.6-terra by default", () => { + expect(getDefaultModel()).toBe("gpt-5.6-terra"); }); test("init sets default model", () => { @@ -346,7 +346,7 @@ describe("OAI", () => { expect(getDefaultModel()).toBe("claude-3-5-sonnet-20241022"); init({ defaultModel: undefined }); - expect(getDefaultModel()).toBe("gpt-5-mini"); + expect(getDefaultModel()).toBe("gpt-5.6-terra"); }); test("init can set both client and default model", () => { diff --git a/js/oai.ts b/js/oai.ts index 6da1dee..5c14290 100644 --- a/js/oai.ts +++ b/js/oai.ts @@ -203,7 +203,7 @@ export interface InitOptions { * * Can be either: * - A string (for backward compatibility): Sets the default completion model only. - * Defaults to "gpt-5-mini" if not set. + * Defaults to "gpt-5.6-terra" if not set. * - An object with `completion` and/or `embedding` properties: Allows setting * default models for different evaluation types. Only the specified models * are updated; others remain unchanged. @@ -237,7 +237,7 @@ export interface InitOptions { | { /** * Default model for LLM-as-a-judge evaluations (completion). - * Defaults to "gpt-5-mini" if not set. + * Defaults to "gpt-5.6-terra" if not set. */ completion?: string; /** @@ -300,10 +300,10 @@ export const init = ({ client, defaultModel }: InitOptions = {}) => { }; /** - * Get the configured default completion model, or "gpt-5-mini" if not set. + * Get the configured default completion model, or "gpt-5.6-terra" if not set. */ export const getDefaultModel = (): string => { - return globalThis.__defaultModel ?? "gpt-5-mini"; + return globalThis.__defaultModel ?? "gpt-5.6-terra"; }; /** diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index 1a38712..c92ac2c 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -3,7 +3,7 @@ This module provides a collection of pre-built LLM scorers for common evaluation tasks. All evaluators accept the following common arguments: -- model: Model to use (defaults to gpt-5-mini) +- model: Model to use (defaults to gpt-5.6-terra) - temperature: Controls randomness (0-1). If not specified, uses the model's default. - max_tokens: Maximum tokens to generate. If not specified, uses the model's default. - client: OpenAI client (defaults to global client from init()) diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index 7a20e29..4516c4b 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -437,7 +437,7 @@ def init( default_model: The default model(s) to use for evaluations when not specified per-call. Can be either: - A string (for backward compatibility): Sets the default completion model only. - Defaults to "gpt-5-mini" if not set. + Defaults to "gpt-5.6-terra" if not set. - A dictionary with "completion" and/or "embedding" keys: Allows setting default models for different evaluation types. Only the specified models are updated; others remain unchanged. @@ -494,8 +494,8 @@ def init( def get_default_model() -> str: - """Get the configured default completion model, or "gpt-5-mini" if not set.""" - return _default_model_var.get(None) or "gpt-5-mini" + """Get the configured default completion model, or "gpt-5.6-terra" if not set.""" + return _default_model_var.get(None) or "gpt-5.6-terra" def get_default_embedding_model() -> str: diff --git a/py/autoevals/ragas.py b/py/autoevals/ragas.py index 4726159..fbcd265 100644 --- a/py/autoevals/ragas.py +++ b/py/autoevals/ragas.py @@ -17,7 +17,7 @@ **Common arguments**: - - `model`: Model to use for evaluation, defaults to the model configured via init(default_model=...) or "gpt-5-mini" + - `model`: Model to use for evaluation, defaults to the model configured via init(default_model=...) or "gpt-5.6-terra" - `client`: Optional Client for API calls. If not provided, uses global client from init() **Example - Direct usage**: @@ -132,7 +132,7 @@ def check_required(name, **kwargs): # Deprecated: Use init(default_model="...") to configure the default model instead. # This was previously "gpt-5-nano" but now defaults to the configured model. -DEFAULT_RAGAS_MODEL = "gpt-5-nano" +DEFAULT_RAGAS_MODEL = "gpt-5.6-luna" def _get_model(model: str | None) -> str: @@ -145,7 +145,7 @@ def _get_model(model: str | None) -> str: return model # Check if user configured a custom default via init(default_model=...) - # If they did (even if it's "gpt-5-mini"), respect it for consistency + # If they did (even if it's "gpt-5.6-terra"), respect it for consistency configured_default = _default_model_var.get(None) if configured_default is not None: return configured_default diff --git a/py/autoevals/test_init_models.py b/py/autoevals/test_init_models.py index b193f28..2c64b12 100644 --- a/py/autoevals/test_init_models.py +++ b/py/autoevals/test_init_models.py @@ -35,7 +35,7 @@ def test_object_form_can_set_embedding_model_only(): assert get_default_embedding_model() == "text-embedding-3-large" # Completion model should remain at default since we didn't update it - assert get_default_model() == "gpt-5-mini" + assert get_default_model() == "gpt-5.6-terra" def test_object_form_can_set_both_models(): @@ -70,7 +70,7 @@ def test_falls_back_to_defaults_when_not_set(): """Test that defaults are used when default_model is not provided.""" init() - assert get_default_model() == "gpt-5-mini" + assert get_default_model() == "gpt-5.6-terra" assert get_default_embedding_model() == "text-embedding-ada-002" diff --git a/py/autoevals/test_oai.py b/py/autoevals/test_oai.py index 1f3447d..5d1f41d 100644 --- a/py/autoevals/test_oai.py +++ b/py/autoevals/test_oai.py @@ -262,11 +262,11 @@ def test_prepare_openai_v0_with_client(mock_openai_v0: OpenAIV0Module): assert prepared_client.complete.__name__ == "acreate" -def test_get_default_model_returns_gpt_4o_by_default(): - """Test that get_default_model returns gpt-5-mini when no default is configured.""" +def test_get_default_model_returns_gpt_5_6_terra_by_default(): + """Test that get_default_model returns gpt-5.6-terra when no default is configured.""" # Reset init to clear any previous default model init(None) - assert get_default_model() == "gpt-5-mini" + assert get_default_model() == "gpt-5.6-terra" def test_init_sets_default_model(): @@ -284,7 +284,7 @@ def test_init_can_reset_default_model(): assert get_default_model() == "claude-3-5-sonnet-20241022" init(None, default_model=None) - assert get_default_model() == "gpt-5-mini" + assert get_default_model() == "gpt-5.6-terra" def test_init_can_set_both_client_and_default_model():