Skip to content

Consolidate model skips in E2E tests #498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lilly-luo merged 1 commit into from
Sep 4, 2026
Merged

Consolidate model skips in E2E tests #498

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 56 additions & 52 deletions tests/test_e2e.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,33 @@ def test_empty_pick_returns_zero_and_writes_nothing(self, tmp_path, monkeypatch,
# Tests are skipped when the binary is not installed or no models are available.
# ---------------------------------------------------------------------------

# Model exclusions live here so a temporary outage can be scoped consistently
# across every harness that exercises the affected model.
E2E_MODEL_SKIP_HARNESSES: dict[str, frozenset[str]] = {
# Codex nano is unreliably slow and exceeds the E2E timeout.
"gpt-5-4-nano": frozenset({"codex"}),
# The CI OpenAI project cannot serve this snapshot in its geography.
"gpt-5-3-codex": frozenset({"codex", "pi"}),
# Bedrock Grok rejects request/tool shapes from these harnesses.
"grok": frozenset({"codex", "copilot", "pi"}),
# These Gemini endpoints hang OpenCode well past its E2E timeout.
"databricks-gemini-3-1-flash-lite": frozenset({"opencode"}),
# Codex-tuned and newer GPT endpoints do not support Copilot's MLflow chat route.
"-codex": frozenset({"copilot"}),
"gpt-5-5": frozenset({"copilot"}),
"gpt-5-6": frozenset({"copilot"}),
# Astra currently fails through these paths in prod-aws-us-east-1.
"astra": frozenset({"copilot", "pi", "web_search"}),
}


def _model_is_skipped(model: str, harness: str) -> bool:
normalized = model.lower()
return any(
fragment in normalized and harness in harnesses
for fragment, harnesses in E2E_MODEL_SKIP_HARNESSES.items()
)


def _require_binary(binary: str):
if not shutil.which(binary):
Expand All @@ -419,31 +446,21 @@ def _require_binary(binary: str):
class TestCodexLaunch:
"""Run codex against every available codex model."""

# Substrings of model IDs that are known-incompatible with the codex CLI on
# Databricks today. Each entry should have a comment explaining why.
CODEX_INCOMPATIBLE_MODEL_FRAGMENTS = (
# nano endpoint is unreliably slow and times out past the 60s budget.
"gpt-5-4-nano",
# Discoverable and correctly configured, but the gateway's upstream OpenAI project can't
# serve this snapshot from the CI region: "The requested model snapshot is not available
# for your project's geography." The gateway relays that as a bare INTERNAL_ERROR
# ("invalid response from an upstream server"), so the launch fails after codex-cli
# exhausts its five reconnects. Nothing ucode writes can fix it.
"gpt-5-3-codex",
# Bedrock Grok rejects the Responses tool schema Codex sends (missing nested `function`).
"grok",
)

def _codex_models(self, e2e_state: dict) -> list[str]:
models = [
model
for model in (e2e_state.get("codex_models") or [])
if not any(frag in model for frag in self.CODEX_INCOMPATIBLE_MODEL_FRAGMENTS)
if not _model_is_skipped(model, "codex")
]
if not models:
pytest.skip("No Codex models available on this workspace")
return models

def test_astra_is_not_skipped(self):
assert self._codex_models({"codex_models": ["databricks-gpt-6-astra"]}) == [
"databricks-gpt-6-astra"
]

def test_launch_codex_per_model(self, tmp_path, monkeypatch, e2e_state, e2e_workspace):
"""Parametrized inline — iterates over all codex models and asserts each works."""
import ucode.config_io as config_io_mod
Expand Down Expand Up @@ -749,21 +766,13 @@ def test_does_not_write_settings_json_for_auth(
class TestOpencodeLaunch:
"""Run OpenCode against the available native and OSS model providers."""

# Models that hang opencode well past 180s on the staging gateway with
# no stderr beyond the initial `> build · <model>` line, while every
# other configured model returns in ~3s. Backend-side latency we can't
# influence from this repo; skip rather than block CI.
SKIP_MODELS: frozenset[str] = frozenset(
{"databricks-gemini-3-1-flash-lite", "databricks-gemini-3-1-flash-lite-image"}
)

def _all_models(self, e2e_state: dict) -> list[tuple[str, str]]:
"""Return [(provider, model_id), ...] for all opencode models."""
opencode_models: dict = e2e_state.get("opencode_models") or {}
out: list[tuple[str, str]] = []
for provider, models in opencode_models.items():
for model in models or []:
if model in self.SKIP_MODELS:
if _model_is_skipped(model, "opencode"):
continue
out.append((provider, model))
return out
Expand Down Expand Up @@ -942,37 +951,26 @@ class TestCopilotLaunch:
"""Run copilot against every Claude/codex model via the MLflow chat-completions gateway.

Gemini is excluded by design — Databricks' Gemini translator rejects the
`stream_options` field Copilot CLI sends. Some codex variants are also
incompatible upstream and are listed in COPILOT_INCOMPATIBLE_MODEL_FRAGMENTS.
`stream_options` field Copilot CLI sends. Other incompatible models are
scoped to Copilot in E2E_MODEL_SKIP_HARNESSES.
"""

# Substrings of model IDs that are known-incompatible with Copilot CLI on
# Databricks today. Each entry should have a comment explaining why.
COPILOT_INCOMPATIBLE_MODEL_FRAGMENTS = (
# Codex-tuned endpoints expose only openai/v1/responses and
# cursor/v1/chat/completions, not mlflow/v1/chat/completions.
"-codex",
# gpt-5.5 rejects function tools + reasoning_effort on /chat/completions
# ("Please use /v1/responses instead").
"gpt-5-5",
# gpt-5.6 models similarly reject /chat/completions with 404.
"gpt-5-6",
# Bedrock Grok rejects the gateway's llm/v1/chat task type.
"grok",
)

def _all_models(self, e2e_state: dict) -> list[tuple[str, str]]:
"""Return [(family, model_id), ...] for every model copilot can talk to."""
out: list[tuple[str, str]] = []
claude_models: dict = e2e_state.get("claude_models") or {}
for family, model_id in _launchable_model_items(claude_models):
out.append((f"claude-{family}", model_id))
for model in e2e_state.get("codex_models") or []:
if any(frag in model for frag in self.COPILOT_INCOMPATIBLE_MODEL_FRAGMENTS):
if _model_is_skipped(model, "copilot"):
continue
out.append(("codex", model))
return out

def test_astra_is_skipped(self):
state = {"codex_models": ["databricks-gpt-6-astra", "databricks-gpt-5-4"]}
assert self._all_models(state) == [("codex", "databricks-gpt-5-4")]

def test_launch_copilot_per_model(
self, tmp_path, monkeypatch, e2e_state, e2e_workspace, e2e_token
):
Expand Down Expand Up @@ -1022,25 +1020,22 @@ class TestPiLaunch:
test exercises each one end-to-end through the validation path.
"""

INCOMPATIBLE_MODEL_FRAGMENTS = (
# The CI project's upstream OpenAI account cannot serve this snapshot in its geography.
"gpt-5-3-codex",
# Bedrock Grok currently rejects Pi's OpenAI request with HTTP 400.
"grok",
)

def _all_models(self, e2e_state: dict) -> list[tuple[str, str]]:
out: list[tuple[str, str]] = []
claude_models: dict = e2e_state.get("claude_models") or {}
for family, model_id in _launchable_model_items(claude_models):
out.append((f"claude-{family}", model_id))
for model in e2e_state.get("codex_models") or []:
if not any(fragment in model for fragment in self.INCOMPATIBLE_MODEL_FRAGMENTS):
if not _model_is_skipped(model, "pi"):
out.append(("codex", model))
for model in e2e_state.get("gemini_models") or []:
out.append(("gemini", model))
return out

def test_astra_is_skipped(self):
state = {"codex_models": ["databricks-gpt-6-astra", "databricks-gpt-5-4"]}
assert self._all_models(state) == [("codex", "databricks-gpt-5-4")]

def test_launch_pi_per_model(self, tmp_path, monkeypatch, e2e_state, e2e_workspace, e2e_token):
import ucode.config_io as config_io_mod
from ucode.agents import pi
Expand Down Expand Up @@ -1109,12 +1104,21 @@ def test_launch_pi_per_model(self, tmp_path, monkeypatch, e2e_state, e2e_workspa


def _first_codex_model(e2e_state: dict) -> str:
models = e2e_state.get("codex_models") or []
models = [
model
for model in (e2e_state.get("codex_models") or [])
if not _model_is_skipped(model, "web_search")
]
if not models:
pytest.skip("No Responses-API (codex) models available on this workspace")
return models[0]


def test_web_search_skips_astra():
state = {"codex_models": ["databricks-gpt-6-astra", "databricks-gpt-5-4"]}
assert _first_codex_model(state) == "databricks-gpt-5-4"


class TestWebSearchResponsesApi:
"""Hit the real Databricks Codex (Responses API) endpoint with native
web_search and assert the model returns non-empty text."""
Expand Down
Loading