From 83f98f7230e890f35b43e8df6db900e37ac612ba Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 18 Oct 2026 04:35:54 +0900 Subject: [PATCH 1/4] feat(providers): explicit single-model completion test Add a user-confirmed, bounded completion diagnostic for one provider and one model, next to the existing no-completion connectivity probe: POST /api/providers/test-completion, `ocx provider test --model `, and a dashboard control in the provider workspace. The runner talks to the upstream through the provider's normal credential, destination and egress path and reports TTFT, total latency and the raw upstream model metadata read before response.model is rewritten. Comparison is metadata only and never claims identity. Refs #2834 --- .../src/content/docs/guides/web-dashboard.md | 27 ++ .../docs/reference/cli/providers-accounts.md | 31 +- .../ProviderCompletionTest.tsx | 110 +++++++ .../provider-workspace/ProviderDetails.tsx | 1 + .../provider-workspace/ProviderOverview.tsx | 6 +- .../provider-completion-test.css | 22 ++ gui/src/i18n/de.ts | 28 ++ gui/src/i18n/en.ts | 28 ++ gui/src/i18n/fr.ts | 28 ++ gui/src/i18n/ja.ts | 28 ++ gui/src/i18n/ko.ts | 28 ++ gui/src/i18n/pt.ts | 28 ++ gui/src/i18n/ru.ts | 28 ++ gui/src/i18n/tr.ts | 28 ++ gui/src/i18n/vi.ts | 28 ++ gui/src/i18n/zh-TW.ts | 28 ++ gui/src/i18n/zh.ts | 28 ++ gui/tests/provider-completion-test.test.tsx | 208 ++++++++++++++ scripts/test-layout/layout.json | 3 + .../references/01_surface_providers-models.md | 12 +- src/cli/capabilities-provider-models.ts | 9 +- src/cli/provider-runtime.ts | 70 ++++- .../management/provider-completion-test.ts | 270 ++++++++++++++++++ src/server/management/provider-routes.ts | 2 + src/server/management/route-registry.ts | 1 + structure/INDEX.md | 7 +- structure/cli-management.md | 2 +- structure/gui-and-management-api.md | 2 +- structure/manifest.json | 11 + structure/provider-completion-diagnostic.md | 11 + .../cli-capability-provider-workflows.test.ts | 6 +- tests/cli/cli-provider-sync-result.test.ts | 146 ++++++++++ tests/fixtures/test-layout-expected.json | 2 +- tests/helpers/provider-completion-fixture.ts | 22 ++ ...provider-completion-test-authority.test.ts | 161 +++++++++++ .../provider-completion-test-ingress.test.ts | 48 ++++ .../provider-completion-test.test.ts | 253 ++++++++++++++++ 37 files changed, 1732 insertions(+), 19 deletions(-) create mode 100644 gui/src/components/provider-workspace/ProviderCompletionTest.tsx create mode 100644 gui/src/components/provider-workspace/provider-completion-test.css create mode 100644 gui/tests/provider-completion-test.test.tsx create mode 100644 src/server/management/provider-completion-test.ts create mode 100644 structure/provider-completion-diagnostic.md create mode 100644 tests/helpers/provider-completion-fixture.ts create mode 100644 tests/providers/provider-completion-test-authority.test.ts create mode 100644 tests/providers/provider-completion-test-ingress.test.ts create mode 100644 tests/providers/provider-completion-test.test.ts diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index 252f7650d39..6cdb10f4d2f 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -480,3 +480,30 @@ them. Both switches apply immediately; **Save** never changes them. When interce Claude/Anthropic accounts are managed only on **Providers > Anthropic > Accounts**: login and the browser option, the account roster with switch, pause, remove, and reauthentication, the paste-code field, account pool settings, and quota. Claude no longer has an Account sub-tab. Provider-level controls such as the default provider, removal, and the enabled switch stay on Providers. The bookmarks `#claude` and `#claude/code` open this page, and the former `#claude/settings` redirects to `#claude/code`; `#claude/desktop` selects the separate Claude Desktop tab. The old `#claude/account` bookmark redirects to `#providers?provider=anthropic&tab=accounts`. Legacy `#integrations/claude` and `#integrations/claude/desktop` bookmarks still redirect to `#claude/code` and `#claude/desktop`, respectively. Arrow keys move between tabs; Home and End select the first and last tab. + + +## Test one provider model + +In **Providers**, **Test connection** checks discovery connectivity. Use the +separate **Test completion** control to check whether a selected model answers. +Choose a model and accept the confirmation that the request may incur provider +charges. Cancelling the confirmation sends nothing. Changing the provider or +model, or leaving the page, cancels the test and discards a stale result. + +The completion test sends `Reply exactly OK.` once with a 32-token output cap, +a 20-second deadline and a 64 KiB stream limit. It reports TTFT at the first +non-empty text delta, total latency, the upstream-reported model and a match, +mismatch or unknown comparison against the wire model id. Missing or conflicting +metadata is unknown; upstream metadata does not verify model identity. The panel +shows the applicability reason when authentication, adapter or transport settings +cannot be tested. No generated text is displayed, and the diagnostic does not +update routing health, request history or the provider catalog. + +The authenticated management endpoint is +`POST /api/providers/test-completion?name=` with exactly +`{"model":"","confirm":true}`. Dashboard sessions require their usual CSRF +token. Invalid input returns 400, unknown providers return 404 and a concurrent +test for the same provider returns 409. Completed diagnostics and applicability +results return 200, including a structured error when inference fails. Destination +policy and provider egress settings apply; redirects, retries and failover are disabled. +For the CLI workflow, see [Providers and accounts](/reference/cli/providers-accounts/). diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index f4ba4e6f112..afc179875a3 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -24,7 +24,7 @@ both `--adapter` and `--base-url`. | `list` | `--json`, `--jsonl` | List configured providers and the remaining registry entries; `--jsonl` emits one configured provider object per line. | | `add ` | `--adapter `, `--base-url `, `--api-key `, `--responses-path `, `--auth-mode `, `--default-model `, `--set-default`, `--force`, `--json`, `--sync`, `--live` | Save locally by default; `--sync` attempts client sync in either output mode. `--live` writes through the running proxy and cannot combine with `--sync`. `--force` permits overwrite. | | `edit ` | provider field flags, `--headers `, `--model-context-tier `, `--upstream-http-version `, `--fast `, `--context-window `, `--json` | Edit validated live provider fields without replacing key pools. Repeat `--model-context-tier` for multiple Copilot models. `--headers` merges custom request headers; pass `{}` or `-` to clear them. | -| `test ` | `--json` | Probe the real upstream model endpoint. | +| `test ` | `--model `, `--yes`, `--json` | Check discovery; with `--model`, request one confirmed completion test. | | `show ` | `--json` | Show config with API keys masked. | | `remove ` | `--json`, `--live`, `--yes` | Local removal refuses the default and last provider. Live removal requires `--yes` and uses server dependency checks, default reassignment and account/custom-model cleanup. | | `set-default ` | `--json`, `--live` | Select a local default, or update the running proxy with `--live`. | @@ -120,10 +120,37 @@ notices. Null, absent or `not-requested` refresh can exit 0 without a client syn Read back before recovery; do not repeat a persisted write to repair convergence. Errors use safe prose on stderr, including in JSON mode, rather than a JSON error envelope. -`provider test` is separate: it checks upstream model discovery connectivity. A failed connection exits 1; success and a static catalog with no applicable discovery endpoint exit 0. +`provider test ` without `--model` checks upstream model discovery connectivity. A failed connection exits 1; success and a static catalog with no applicable discovery endpoint exit 0. `applicable: false` is an expected static-catalog result, not successful inference or a failed connection. +To send one completion request for a chosen model, use: + +```bash +ocx provider test my-relay --model my-model +ocx provider test my-relay --model my-model --yes --json +``` + +This test may incur provider charges. An interactive terminal asks for confirmation; +`--yes` accepts the charge warning. Non-interactive use and `--json` require +`--yes`. Using `--yes` without `--model` is a usage error. Declining confirmation +sends no request and exits 1. The command uses the running proxy's management API. + +The completion test sends the fixed prompt `Reply exactly OK.`, requests a 32-token output cap, stops after 20 seconds or 64 KiB, and never retries or fails over. +It supports key-auth or local OpenAI Chat and Responses providers, including +per-model wire overrides. Disabled, OAuth, forwarded-auth and unsupported adapter +or transport configurations return an applicability reason without sending a completion. +TLS profiles, HTTP version pins, enabled request pacing and custom executors with +a provider proxy override are currently unsupported. + +Results include time to first text delta (TTFT), total latency, the requested and +wire model ids, sanitized upstream model metadata and a distinct match, mismatch +or unknown comparison. Missing or conflicting metadata is unknown. The comparison +uses the upstream id before masking, against the id actually sent after alias and +suffix handling. Provider-reported metadata cannot verify model identity: +`identityVerified` is always `false`. Generated text is not returned. A failed +completion exits 1; success and a not-applicable result exit 0. + ### Transport settings and request pacing `provider edit` is already live. Use `--upstream-http-version http1.1` to pin that diff --git a/gui/src/components/provider-workspace/ProviderCompletionTest.tsx b/gui/src/components/provider-workspace/ProviderCompletionTest.tsx new file mode 100644 index 00000000000..63a19421bdb --- /dev/null +++ b/gui/src/components/provider-workspace/ProviderCompletionTest.tsx @@ -0,0 +1,110 @@ +/** An explicitly consented, bounded completion probe; it never runs during discovery. */ +import { useEffect, useRef, useState } from "react"; +import { confirmAction } from "../../action-dialogs"; +import { readJsonOrThrow } from "../../fetch-json"; +import { useI18n, useT } from "../../i18n/shared"; +import { Select } from "../../ui"; +import "./provider-completion-test.css"; + +type InapplicableReason = "disabled" | "forward_auth" | "oauth_auth" | "unsupported_adapter" | "unsupported_transport"; +type TransportDetail = "tls_profile" | "upstream_http_version" | "request_pacing" | "custom_executor_egress"; +export type CompletionTestResult = + | { applicable: false; reason: InapplicableReason; adapter?: string; detail?: TransportDetail } + | { applicable: true; ok: boolean; provider: string; requestedModel: string; wireModel: string; + wire: "openai-chat" | "openai-responses"; upstreamModel: string | null; + modelComparison: "match" | "mismatch" | "unknown"; + comparisonNote?: "missing_metadata" | "conflicting_metadata"; identityVerified: false; + ttftMs: number | null; totalLatencyMs: number; outputTokenCap: 32; + termination: "completed" | "failed" | "timeout" | "aborted" | "too_large"; + error: { code: string; status?: number; message: string } | null }; + +type Props = { apiBase: string; provider: string; models: string[] }; + +export default function ProviderCompletionTest({ apiBase, provider, models }: Props) { + const t = useT(); + const [model, setModel] = useState(""); + const selectedModel = models.includes(model) ? model : ""; + return ( +
+

{t("pws.completion.title")}

+