diff --git a/backend/druks/agents.py b/backend/druks/agents.py index ec69baaa..779e1e08 100644 --- a/backend/druks/agents.py +++ b/backend/druks/agents.py @@ -10,6 +10,8 @@ from dbos import DBOS, StepOptions from pydantic import BaseModel, ConfigDict, ValidationError +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession from druks.apps.exceptions import AppBotError from druks.apps.loader import resolve_app @@ -22,8 +24,9 @@ from druks.db import db_session from druks.durable.activity import set_run_phase from druks.durable.engine import _step_engine, step_session -from druks.durable.exceptions import WorkflowError -from druks.durable.models import AgentCall, Artifact +from druks.durable.enums import RunState +from druks.durable.exceptions import WaitTimeout, WorkflowError +from druks.durable.models import AgentCall, Artifact, Run from druks.exceptions import ObjectNotFound from druks.files.datastructures import File from druks.files.exceptions import FileUnavailableError @@ -55,6 +58,8 @@ _QUOTA_FALLBACK_WAIT_SECONDS = 30 * 60 _QUOTA_MAX_WAIT_SECONDS = 6 * 60 * 60 _REAP_BEFORE_WAIT_SECONDS = 120 +# The topic a run waiting out a quota window listens on for a changed provider or account. +_QUOTA_TOPIC = "agents.quota" _in_after_agent: ContextVar[bool] = ContextVar("_in_after_agent", default=False) @@ -232,26 +237,40 @@ async def _invoke() -> Any: # re-runs the whole enclosing step from attempt one anyway. await asyncio.sleep(delay * random.uniform(0.9, 1.1)) - async def _quota_retry_wait() -> float: - # Runs as its own step: the body does no IO, and replay reuses the - # recorded wait instead of re-reading the scrape. + async def _quota_retry_wait() -> dict[str, Any]: + # Runs as its own step: replay reuses the recorded wait instead of + # re-reading the scrape. async with step_session() as session: - # The scrape belongs to the subscription account. config = await get_config(session, self.id, workflow.account_id) provider_id = config.model.partition("/")[0] scrape = await UsageScrape.latest_for( session, provider_id, config.charged_account_id ) - if scrape: - now = datetime.now(UTC) - reset = scrape.soonest_reset_after(now) - if reset: - return (reset - now).total_seconds() - # No scrape yet, or nothing ahead of now: a blind half hour. - return _QUOTA_FALLBACK_WAIT_SECONDS + now = datetime.now(UTC) + reset = scrape.soonest_reset_after(now) if scrape else None + # No scrape yet, or nothing ahead of now: a blind half hour. + seconds = (reset - now).total_seconds() if reset else _QUOTA_FALLBACK_WAIT_SECONDS + # Jitter, so one window's reset doesn't wake every run at once. + wait = seconds + random.uniform(60, 120) + return { + "quota_holder": config.quota_holder, + "seconds": seconds, + "wait": wait, + "ends_at": (now + timedelta(seconds=wait)).isoformat(), + } + + async def _quota_wait_left(quota_holder: str, ends_at: str) -> float: + # Another holder retries now; its own limit, if any, fails the call and + # earns that holder its own wait. + async with step_session() as session: + config = await get_config(session, self.id, workflow.account_id) + if config.quota_holder != quota_holder: + return 0 + return (datetime.fromisoformat(ends_at) - datetime.now(UTC)).total_seconds() transient_retry_index = 0 - quota_retried = False + # The quota holders whose window this call has waited out once already. + quota_waited: set[str] = set() while True: try: result = await DBOS.run_step_async( @@ -268,21 +287,21 @@ async def _quota_retry_wait() -> float: # Jitter, so a provider outage doesn't wake every run at once. wait_seconds = delay * random.uniform(0.9, 1.1) elif error.retry is Retry.QUOTA: - # One shot: sleep until the provider's window turns, read - # off the latest usage scrape. - if quota_retried: - raise - quota_retried = True - wait_seconds = await DBOS.run_step_async( + # Wait until the provider's window turns, read off the latest + # usage scrape, once per quota holder. + quota = await DBOS.run_step_async( StepOptions(name=f"{workflow.kind}.agent.{self.id}.retry_wait"), _quota_retry_wait, ) - # A far-off reset (an exhausted weekly window) isn't worth - # sleeping into — fail now. - if wait_seconds > _QUOTA_MAX_WAIT_SECONDS: + # A quota holder waits once, and a far-off reset (an exhausted + # weekly window) isn't worth sleeping into — fail now. + if ( + quota["quota_holder"] in quota_waited + or quota["seconds"] > _QUOTA_MAX_WAIT_SECONDS + ): raise - wait_seconds += random.uniform(60, 120) - resumes_at = datetime.now(UTC) + timedelta(seconds=wait_seconds) + quota_waited.add(quota["quota_holder"]) + resumes_at = datetime.fromisoformat(quota["ends_at"]) await set_run_phase("waiting_for_quota", ends_at=resumes_at) if workflow.conversation_id: await _post_to_chat( @@ -290,6 +309,29 @@ async def _quota_retry_wait() -> float: "chat.report_quota_wait", functools.partial(report_quota_wait, resumes_at=resumes_at), ) + wait_seconds = quota["wait"] + if wait_seconds >= _REAP_BEFORE_WAIT_SECONDS: + await workflow._reap_run() + # Each wait's length comes from a step, so replay takes the same turns. + while wait_seconds > 0: + try: + await workflow.wait( + _QUOTA_TOPIC, label="waiting_for_quota", ttl_seconds=wait_seconds + ) + except WaitTimeout: + break + # A changed provider or account woke the wait. + wait_seconds = await DBOS.run_step_async( + StepOptions(name=f"{workflow.kind}.agent.{self.id}.quota_check"), + functools.partial( + _quota_wait_left, quota["quota_holder"], quota["ends_at"] + ), + ) + if workflow.conversation_id: + await _post_to_chat( + workflow.workflow_id, "chat.report_quota_resume", report_quota_resume + ) + continue else: # Permanent or unclassified — never retried. raise @@ -300,16 +342,21 @@ async def _quota_retry_wait() -> float: # Durable: the end time checkpoints now, so a restart resumes # with the remaining wait, and replay skips it entirely. await DBOS.sleep_async(wait_seconds) - if error.retry is Retry.QUOTA and workflow.conversation_id: - await _post_to_chat( - workflow.workflow_id, "chat.report_quota_resume", report_quota_resume - ) else: # Body-level only, on both passes: in-step calls hit the early return # live and are skipped with their step on replay. workflow.journal.add(result) return result + @classmethod + async def wake_quota_waits(cls, session: AsyncSession) -> None: + """Wake every run waiting out a quota window, so it reads its agent's provider and + account again. Call it after the change commits.""" + running = await session.scalars(select(Run.id).where(Run.state == RunState.RUNNING.value)) + for run_id in running: + if await DBOS.get_event_async(run_id, "waiting_on", timeout_seconds=0) == _QUOTA_TOPIC: + await DBOS.send_async(run_id, {}, topic=_QUOTA_TOPIC) + async def _run( self, *, diff --git a/backend/druks/chat/constants.py b/backend/druks/chat/constants.py index 7e3e2c87..bd8bfb2f 100644 --- a/backend/druks/chat/constants.py +++ b/backend/druks/chat/constants.py @@ -40,8 +40,7 @@ "what it waits for and where to answer it.]" ) QUOTA_RESUMED_MESSAGE = ( - "[Internal: Run {run} continues now that the provider's usage limit reset. Tell the " - "person in one short line.]" + "[Internal: Run {run} continues after its usage-limit wait. Tell the person in one short line.]" ) FAILURE_MESSAGE = ( "[Internal: Run {run} failed: {failure}. Tell the person in one short line, with no " diff --git a/backend/druks/chat/service.py b/backend/druks/chat/service.py index 95f57eac..29db529f 100644 --- a/backend/druks/chat/service.py +++ b/backend/druks/chat/service.py @@ -747,7 +747,7 @@ async def report_quota_wait(session: AsyncSession, run: Run, *, resumes_at: date async def report_quota_resume(session: AsyncSession, run: Run) -> str | None: - """Tell the chat that started a run that it continues after the usage limit reset. + """Tell the chat that started a run that it continues after its usage-limit wait. Returns the conversation to deliver.""" return await report_outcome(session, run, QUOTA_RESUMED_MESSAGE.format(run=run.id)) diff --git a/backend/druks/harnesses/config.py b/backend/druks/harnesses/config.py index 16074cc7..293f8c27 100644 --- a/backend/druks/harnesses/config.py +++ b/backend/druks/harnesses/config.py @@ -70,6 +70,12 @@ def secrets_hash(self) -> str: def charged_account_id(self) -> str | None: return self.subscription.account_id if self.subscription else None + @property + def quota_holder(self) -> str: + """Whom a usage limit binds: the model's provider and the account signed in to it.""" + email = self.subscription.identity.get("email") if self.subscription else None + return f"{self.model}|{self.billing}|{email}" + async def check_config( session: AsyncSession, harness_name: str, model: str, billing: str diff --git a/backend/druks/harnesses/routes.py b/backend/druks/harnesses/routes.py index bc172864..f4619c9e 100644 --- a/backend/druks/harnesses/routes.py +++ b/backend/druks/harnesses/routes.py @@ -7,6 +7,7 @@ from druks.accounts.dependencies import current_session_account, current_session_or_setup from druks.accounts.models import Account from druks.accounts.schemas import AccountResponse +from druks.agents import Agent from druks.api.dependencies import SessionDep from druks.secrets.datastructures import Audience from druks.secrets.enums import SecretKind @@ -169,7 +170,7 @@ async def sign_in_with_token( credentials, expires_at = provider.parse_token(token) except ConnectError as error: raise HTTPException(status_code=422, detail=str(error)) from error - return await VaultSecret.store( + subscription = await VaultSecret.store( session, SecretKind.SUBSCRIPTION, Audience.provider(provider.id), @@ -179,6 +180,10 @@ async def sign_in_with_token( identity={"email": None}, expires_at=expires_at, ) + # A woken run reads the subscription, so it commits first. + await session.commit() + await Agent.wake_quota_waits(session) + return subscription async def _store_connection( @@ -214,6 +219,7 @@ async def _store_connection( # across an await, and after the commit the reply would need another read. response = AccountResponse.model_validate(resolved) await session.commit() + await Agent.wake_quota_waits(session) try: # The flow is already spent, so a failed refresh only logs. await provider.refresh_catalog(session) diff --git a/backend/druks/user_settings/routes.py b/backend/druks/user_settings/routes.py index d3ddba1c..2a9b877d 100644 --- a/backend/druks/user_settings/routes.py +++ b/backend/druks/user_settings/routes.py @@ -3,6 +3,7 @@ from druks.accounts.dependencies import current_account, current_session_account from druks.accounts.models import Account +from druks.agents import Agent from druks.api.dependencies import SessionDep from druks.apps.loader import get_app, iter_apps from druks.apps.registry import agents, bots, channels, workflows @@ -108,6 +109,9 @@ async def update_settings(session: SessionDep, body: UpdateSettingsRequest) -> I await settings.update(**fields) if any(field in fields for field in _EXECUTION_DEFAULTS): await check_agent_configs(session, settings) + # A woken run reads the settings, so they commit first. + await session.commit() + await Agent.wake_quota_waits(session) return settings @@ -217,4 +221,9 @@ async def update_app_settings( # the just-written overrides off this request's session. await apply_schedules(session) + if body.agent_harnesses or body.agent_models or body.agent_billings: + # A woken run reads the overrides, so they commit first. + await session.commit() + await Agent.wake_quota_waits(session) + return await get_app_settings(session) diff --git a/backend/tests/test_agents.py b/backend/tests/test_agents.py index fd891827..d3b398e6 100644 --- a/backend/tests/test_agents.py +++ b/backend/tests/test_agents.py @@ -27,6 +27,7 @@ from druks.services import Resource, Service from druks.usage.models import UsageScrape from druks.user_settings.models import SettingsOverride +from druks.workflows import WaitTimeout from druks.workspaces import Workspace from pydantic import BaseModel, SecretStr from sqlalchemy import select @@ -831,20 +832,20 @@ async def latest_for(_cls, _session, _harness, _account_id): return latest_for -async def test_body_level_quota_waits_for_the_reset_once( - druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps -): - from druks.harnesses.exceptions import HarnessRateLimitError - - now = datetime(2026, 8, 1, 10, tzinfo=UTC) - failures = [HarnessRateLimitError(f"quota {attempt}") for attempt in range(2)] +def _patch_quota(monkeypatch, tmp_path, current_run, failures, *, now): + """An agent whose calls fail with ``failures`` in turn (None succeeds), on a clock stopped + at ``now``, with a scrape whose binding window resets an hour later.""" sandbox = _patch_runtime(monkeypatch, tmp_path, {"ok": True}) sandbox.run_agent = AsyncMock( - side_effect=[make_agent_result(None, agent="dummy", error=failure) for failure in failures] + side_effect=[ + make_agent_result(None if failure else {"ok": True}, agent="dummy", error=failure) + for failure in failures + ] ) _patch_ephemeral(monkeypatch, sandbox) clock = MagicMock() clock.now.return_value = now + clock.fromisoformat = datetime.fromisoformat monkeypatch.setattr(agents, "datetime", clock) monkeypatch.setattr( agents.UsageScrape, @@ -869,21 +870,47 @@ async def test_body_level_quota_waits_for_the_reset_once( ) ), ) - jitter = MagicMock(return_value=90.0) - monkeypatch.setattr(agents.random, "uniform", jitter) + monkeypatch.setattr(agents.random, "uniform", MagicMock(return_value=90.0)) + current_run._reap_run = AsyncMock() + return sandbox + + +async def _switch_claude_account(email: str) -> None: + # The run's account connects another Claude account in place of the limited one. + account = await Account.get_default(db_session()) + await VaultSecret.store( + db_session(), + SecretKind.SUBSCRIPTION, + Audience.provider("anthropic"), + account_id=account.id, + secrets={"claudeAiOauth": {"accessToken": email}}, + identity={"email": email}, + ) + + +async def test_body_level_quota_waits_for_the_reset_once( + druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps +): + from druks.harnesses.exceptions import HarnessRateLimitError + + now = datetime(2026, 8, 1, 10, tzinfo=UTC) + failures = [HarnessRateLimitError(f"quota {attempt}") for attempt in range(2)] + _patch_quota(monkeypatch, tmp_path, current_run, failures, now=now) set_run_phase = AsyncMock() monkeypatch.setattr(agents, "set_run_phase", set_run_phase) post_to_chat = AsyncMock() monkeypatch.setattr(agents, "_post_to_chat", post_to_chat) - current_run._reap_run = AsyncMock() current_run.conversation_id = "conversation-1" - checkpoints, sleep = _inline_agent_steps + current_run.wait = AsyncMock(side_effect=WaitTimeout("agents.quota")) + checkpoints, _sleep = _inline_agent_steps with pytest.raises(HarnessRateLimitError) as excinfo: await DUMMY_AGENT() assert excinfo.value is failures[-1] - sleep.assert_awaited_once_with(60 * 60 + 90.0) + current_run.wait.assert_awaited_once_with( + "agents.quota", label="waiting_for_quota", ttl_seconds=60 * 60 + 90.0 + ) [wait, resume] = [call.args for call in post_to_chat.await_args_list] assert (wait[:2], wait[2].func, wait[2].keywords) == ( ("wf-9", "chat.report_quota_wait"), @@ -891,7 +918,6 @@ async def test_body_level_quota_waits_for_the_reset_once( {"resumes_at": now + timedelta(hours=1, seconds=90)}, ) assert resume == ("wf-9", "chat.report_quota_resume", report_quota_resume) - jitter.assert_called_once_with(60.0, 120.0) set_run_phase.assert_any_await( "waiting_for_quota", ends_at=now + timedelta(hours=1, seconds=90), @@ -901,12 +927,66 @@ async def test_body_level_quota_waits_for_the_reset_once( "test.agent.dummy", "test.agent.dummy.retry_wait", "test.agent.dummy", + "test.agent.dummy.retry_wait", ] calls = await AgentCall.list_for_run(db_session(), "wf-9") assert len(calls) == 2 assert all(call.failure_code == "rate_limited" for call in calls) +async def test_a_quota_wait_woken_without_a_switch_waits_out_its_reset( + druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps +): + from druks.harnesses.exceptions import HarnessRateLimitError + + now = datetime(2026, 8, 1, 10, tzinfo=UTC) + sandbox = _patch_quota( + monkeypatch, tmp_path, current_run, [HarnessRateLimitError("quota"), None], now=now + ) + current_run.wait = AsyncMock(side_effect=[{}, WaitTimeout("agents.quota")]) + + assert await DUMMY_AGENT() == DummyOutput(ok=True) + + assert current_run.wait.await_count == 2 + assert current_run.wait.await_args.kwargs["ttl_seconds"] == 60 * 60 + 90.0 + assert sandbox.run_agent.await_count == 2 + + +async def test_a_switched_account_retries_now_and_waits_out_its_own_limit_once( + druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps +): + from druks.harnesses.exceptions import HarnessRateLimitError + + now = datetime(2026, 8, 1, 10, tzinfo=UTC) + failures = [HarnessRateLimitError(f"quota {attempt}") for attempt in range(3)] + sandbox = _patch_quota(monkeypatch, tmp_path, current_run, failures, now=now) + + async def switch_on_the_first_wait(*_args, **_kwargs): + if current_run.wait.await_count == 1: + await _switch_claude_account("second@example.com") + return {} + raise WaitTimeout("agents.quota") + + current_run.wait = AsyncMock(side_effect=switch_on_the_first_wait) + checkpoints, _sleep = _inline_agent_steps + + with pytest.raises(HarnessRateLimitError) as excinfo: + await DUMMY_AGENT() + + assert excinfo.value is failures[-1] + assert current_run.wait.await_count == 2 + assert sandbox.run_agent.await_count == 3 + assert [options["name"] for options in checkpoints] == [ + "test.agent.dummy", + "test.agent.dummy.retry_wait", + "test.agent.dummy.quota_check", + "test.agent.dummy", + "test.agent.dummy.retry_wait", + "test.agent.dummy", + "test.agent.dummy.retry_wait", + ] + + async def test_body_level_quota_reset_over_six_hours_reraises_without_sleeping( druks_db, tmp_path, monkeypatch, current_run, _inline_agent_steps ): @@ -940,17 +1020,14 @@ async def test_body_level_quota_reset_over_six_hours_reraises_without_sleeping( ) ), ) - jitter = MagicMock() - monkeypatch.setattr(agents.random, "uniform", jitter) current_run._reap_run = AsyncMock() - _, sleep = _inline_agent_steps + current_run.wait = AsyncMock() with pytest.raises(HarnessUsageLimitError) as excinfo: await DUMMY_AGENT() assert excinfo.value is failure - sleep.assert_not_awaited() - jitter.assert_not_called() + current_run.wait.assert_not_awaited() current_run._reap_run.assert_not_awaited() [call] = await AgentCall.list_for_run(db_session(), "wf-9") assert call.failure_code == "usage_limit" diff --git a/backend/tests/test_api_providers.py b/backend/tests/test_api_providers.py index cddbec36..68936f0d 100644 --- a/backend/tests/test_api_providers.py +++ b/backend/tests/test_api_providers.py @@ -1,8 +1,10 @@ from datetime import UTC, datetime, timedelta from pathlib import Path +from unittest.mock import AsyncMock from conftest import connect_provider from druks.accounts.models import Account +from druks.agents import Agent from druks.harnesses import directory from druks.harnesses.models import ProviderCatalog from druks.harnesses.providers import AnthropicProvider @@ -259,6 +261,15 @@ async def test_a_pasted_setup_token_is_the_requesting_accounts_subscription( assert row.identity == {"email": None} +def test_a_pasted_token_wakes_the_runs_waiting_out_a_quota(tmp_path: Path, monkeypatch): + _sign_in_with(monkeypatch, tmp_path, "setup_token") + wake = AsyncMock() + monkeypatch.setattr(Agent, "wake_quota_waits", wake) + with _build_client(tmp_path) as client: + client.post("/api/providers/anthropic/sign-in/token", json={"token": "sk-ant-oat01-x"}) + wake.assert_awaited_once() + + def test_a_token_of_another_shape_is_refused(tmp_path: Path, monkeypatch): _sign_in_with(monkeypatch, tmp_path, "setup_token") with _build_client(tmp_path) as client: diff --git a/backend/tests/test_api_settings.py b/backend/tests/test_api_settings.py index caf8b2e7..37b141bc 100644 --- a/backend/tests/test_api_settings.py +++ b/backend/tests/test_api_settings.py @@ -1,7 +1,9 @@ from pathlib import Path +from unittest.mock import AsyncMock from conftest import settings_client from druks.accounts.models import Account +from druks.agents import Agent from druks.contrib.software_factory.app import SoftwareFactory from druks.db import db_session from druks.user_settings.models import SettingsOverride @@ -74,6 +76,26 @@ def test_patch_settings_judges_the_default_triple_together(tmp_path: Path): assert "does not run OpenAI" in claude.json()["detail"] +def test_a_switched_provider_wakes_the_runs_waiting_out_a_quota(tmp_path: Path, monkeypatch): + wake = AsyncMock() + monkeypatch.setattr(Agent, "wake_quota_waits", wake) + with settings_client(tmp_path) as client: + client.patch( + "/api/settings/apps", json={"agentEfforts": {"software_factory.implement": "low"}} + ) + wake.assert_not_awaited() + client.patch( + "/api/settings/apps", + json={ + "agentHarnesses": {"software_factory.implement": "codex"}, + "agentModels": {"software_factory.implement": "openai/gpt-5.5"}, + }, + ) + wake.assert_awaited_once() + client.patch("/api/settings", json={"defaultModel": "anthropic/claude-sonnet-4-5"}) + assert wake.await_count == 2 + + async def test_accounts_report_the_default_without_a_fallback_setting(druks_db, tmp_path: Path): account = await Account.get_or_create(druks_db, "ops@example.com") with settings_client(tmp_path) as client: diff --git a/backend/tests/test_durable_sdk.py b/backend/tests/test_durable_sdk.py index 92cf759b..4c050723 100644 --- a/backend/tests/test_durable_sdk.py +++ b/backend/tests/test_durable_sdk.py @@ -594,6 +594,41 @@ async def waiting() -> None: workflows._items.pop(FailingWaitFlow.kind) +async def test_a_quota_wake_reaches_only_the_runs_waiting_out_a_quota(runtime): + class QuotaWaitFlow(Workflow): + subject = Widget + + async def run_multistep(self) -> None: + await self.wait("agents.quota", label="waiting_for_quota", ttl_seconds=60) + + class ChecksWaitFlow(Workflow): + subject = Widget + + async def run_multistep(self) -> None: + await self.wait("checks.finished", label="Waiting for checks", ttl_seconds=60) + + try: + quota_id = await QuotaWaitFlow.start(subject=Widget(id=770772)) + checks_id = await ChecksWaitFlow.start(subject=Widget(id=770773)) + + async def waiting() -> None: + while await get_run_phase(quota_id) != "waiting_for_quota" or ( + await get_run_phase(checks_id) != "Waiting for checks" + ): + await asyncio.sleep(0.1) + + await asyncio.wait_for(waiting(), 15) + async with session_scope(runtime.engine) as session: + await Agent.wake_quota_waits(session) + + await _wait_for_run(runtime.engine, quota_id, lambda run: run.state == RunState.FINISHED) + assert (await _get_run(runtime.engine, checks_id)).state == RunState.RUNNING + finally: + await DBOS.cancel_workflow_async(checks_id) + workflows._items.pop(QuotaWaitFlow.kind) + workflows._items.pop(ChecksWaitFlow.kind) + + async def test_duplicate_replies_to_one_round_collapse(runtime): workflow_id = await runtime.DoubleGateFlow.start(subject=Widget(id=515151)) parked = await _wait_for_run( diff --git a/docs/configuration.md b/docs/configuration.md index bcb123c5..837a009a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -860,6 +860,11 @@ while values stay unchanged. An exhausted window waits for its reset unless an agent call finishes on the subscription. Manual refresh keeps a 60-second minimum between polls. +An agent call that hits a usage limit waits for the window's reset, up to six +hours, and then retries once. If you connect another subscription, or change the +agent's harness, model, or billing during the wait, the call retries at once. If +the new account is also limited, the call waits once more, for that account's reset. + The `claude` and `codex` CLIs run on their own vendor's subscription or key. `opencode` and `pi` run on an API key only, for Anthropic or OpenAI. A key for a Models.dev provider stores, but an agent on that provider refuses to run: no