diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index 252f7650d39..6cdb10f4d2f 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -480,3 +480,30 @@ them. Both switches apply immediately; **Save** never changes them. When interce Claude/Anthropic accounts are managed only on **Providers > Anthropic > Accounts**: login and the browser option, the account roster with switch, pause, remove, and reauthentication, the paste-code field, account pool settings, and quota. Claude no longer has an Account sub-tab. Provider-level controls such as the default provider, removal, and the enabled switch stay on Providers. The bookmarks `#claude` and `#claude/code` open this page, and the former `#claude/settings` redirects to `#claude/code`; `#claude/desktop` selects the separate Claude Desktop tab. The old `#claude/account` bookmark redirects to `#providers?provider=anthropic&tab=accounts`. Legacy `#integrations/claude` and `#integrations/claude/desktop` bookmarks still redirect to `#claude/code` and `#claude/desktop`, respectively. Arrow keys move between tabs; Home and End select the first and last tab. + + +## Test one provider model + +In **Providers**, **Test connection** checks discovery connectivity. Use the +separate **Test completion** control to check whether a selected model answers. +Choose a model and accept the confirmation that the request may incur provider +charges. Cancelling the confirmation sends nothing. Changing the provider or +model, or leaving the page, cancels the test and discards a stale result. + +The completion test sends `Reply exactly OK.` once with a 32-token output cap, +a 20-second deadline and a 64 KiB stream limit. It reports TTFT at the first +non-empty text delta, total latency, the upstream-reported model and a match, +mismatch or unknown comparison against the wire model id. Missing or conflicting +metadata is unknown; upstream metadata does not verify model identity. The panel +shows the applicability reason when authentication, adapter or transport settings +cannot be tested. No generated text is displayed, and the diagnostic does not +update routing health, request history or the provider catalog. + +The authenticated management endpoint is +`POST /api/providers/test-completion?name=` with exactly +`{"model":"","confirm":true}`. Dashboard sessions require their usual CSRF +token. Invalid input returns 400, unknown providers return 404 and a concurrent +test for the same provider returns 409. Completed diagnostics and applicability +results return 200, including a structured error when inference fails. Destination +policy and provider egress settings apply; redirects, retries and failover are disabled. +For the CLI workflow, see [Providers and accounts](/reference/cli/providers-accounts/). diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index ba5e4187e0d..946d51fd489 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -24,7 +24,7 @@ both `--adapter` and `--base-url`. | `list` | `--json`, `--jsonl` | List configured providers and the remaining registry entries; `--jsonl` emits one configured provider object per line. | | `add ` | `--adapter `, `--base-url `, `--api-key `, `--responses-path `, `--auth-mode `, `--default-model `, `--set-default`, `--force`, `--json`, `--sync`, `--live` | Save locally by default; `--sync` attempts client sync in either output mode. `--live` writes through the running proxy and cannot combine with `--sync`. `--force` permits overwrite. | | `edit ` | provider field flags, `--headers `, `--model-context-tier `, `--upstream-http-version `, `--fast `, `--context-window `, `--json` | Edit validated live provider fields without replacing key pools. Repeat `--model-context-tier` for multiple Copilot models. `--headers` merges custom request headers; pass `{}` or `-` to clear them. | -| `test ` | `--json` | Probe the real upstream model endpoint. | +| `test ` | `--model `, `--yes`, `--json` | Check discovery; with `--model`, request one confirmed completion test. | | `show ` | `--json` | Show config with API keys masked. | | `remove ` | `--json`, `--live`, `--yes` | Local removal refuses the default and last provider. Live removal requires `--yes` and uses server dependency checks, default reassignment and account/custom-model cleanup. | | `set-default ` | `--json`, `--live` | Select a local default, or update the running proxy with `--live`. | @@ -120,10 +120,37 @@ notices. Null, absent or `not-requested` refresh can exit 0 without a client syn Read back before recovery; do not repeat a persisted write to repair convergence. Errors use safe prose on stderr, including in JSON mode, rather than a JSON error envelope. -`provider test` is separate: it checks upstream model discovery connectivity. A failed connection exits 1; success and a static catalog with no applicable discovery endpoint exit 0. +`provider test ` without `--model` checks upstream model discovery connectivity. A failed connection exits 1; success and a static catalog with no applicable discovery endpoint exit 0. `applicable: false` is an expected static-catalog result, not successful inference or a failed connection. +To send one completion request for a chosen model, use: + +```bash +ocx provider test my-relay --model my-model +ocx provider test my-relay --model my-model --yes --json +``` + +This test may incur provider charges. An interactive terminal asks for confirmation; +`--yes` accepts the charge warning. Non-interactive use and `--json` require +`--yes`. Using `--yes` without `--model` is a usage error. Declining confirmation +sends no request and exits 1. The command uses the running proxy's management API. + +The completion test sends the fixed prompt `Reply exactly OK.`, requests a 32-token output cap, stops after 20 seconds or 64 KiB, and never retries or fails over. +It supports key-auth or local OpenAI Chat and Responses providers, including +per-model wire overrides. Disabled, OAuth, forwarded-auth and unsupported adapter +or transport configurations return an applicability reason without sending a completion. +TLS profiles, HTTP version pins, enabled request pacing and custom executors with +a provider proxy override are currently unsupported. + +Results include time to first text delta (TTFT), total latency, the requested and +wire model ids, sanitized upstream model metadata and a distinct match, mismatch +or unknown comparison. Missing or conflicting metadata is unknown. The comparison +uses the upstream id before masking, against the id actually sent after alias and +suffix handling. Provider-reported metadata cannot verify model identity: +`identityVerified` is always `false`. Generated text is not returned. A failed +completion exits 1; success and a not-applicable result exit 0. + ### Transport settings and request pacing `provider edit` is already live. Use `--upstream-http-version http1.1` to pin that diff --git a/gui/src/components/provider-workspace/ProviderCompletionTest.tsx b/gui/src/components/provider-workspace/ProviderCompletionTest.tsx new file mode 100644 index 00000000000..63a19421bdb --- /dev/null +++ b/gui/src/components/provider-workspace/ProviderCompletionTest.tsx @@ -0,0 +1,110 @@ +/** An explicitly consented, bounded completion probe; it never runs during discovery. */ +import { useEffect, useRef, useState } from "react"; +import { confirmAction } from "../../action-dialogs"; +import { readJsonOrThrow } from "../../fetch-json"; +import { useI18n, useT } from "../../i18n/shared"; +import { Select } from "../../ui"; +import "./provider-completion-test.css"; + +type InapplicableReason = "disabled" | "forward_auth" | "oauth_auth" | "unsupported_adapter" | "unsupported_transport"; +type TransportDetail = "tls_profile" | "upstream_http_version" | "request_pacing" | "custom_executor_egress"; +export type CompletionTestResult = + | { applicable: false; reason: InapplicableReason; adapter?: string; detail?: TransportDetail } + | { applicable: true; ok: boolean; provider: string; requestedModel: string; wireModel: string; + wire: "openai-chat" | "openai-responses"; upstreamModel: string | null; + modelComparison: "match" | "mismatch" | "unknown"; + comparisonNote?: "missing_metadata" | "conflicting_metadata"; identityVerified: false; + ttftMs: number | null; totalLatencyMs: number; outputTokenCap: 32; + termination: "completed" | "failed" | "timeout" | "aborted" | "too_large"; + error: { code: string; status?: number; message: string } | null }; + +type Props = { apiBase: string; provider: string; models: string[] }; + +export default function ProviderCompletionTest({ apiBase, provider, models }: Props) { + const t = useT(); + const [model, setModel] = useState(""); + const selectedModel = models.includes(model) ? model : ""; + return ( +
+

{t("pws.completion.title")}

+