From e38316891f989d4ec125086b18450f45da56e2c6 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 23 Oct 2026 08:02:44 +0900 Subject: [PATCH 1/4] feat(jev): opt-in session pin keeps a JEV combo session on its decided target (#7525) --- docs-site/src/content/docs/guides/combos.md | 48 ++++ .../src/content/docs/reference/cli/agents.md | 5 + .../docs/reference/configuration/routing.md | 1 + scripts/test-layout/layout-jev.json | 5 + scripts/test-layout/layout.json | 2 +- .../references/01_surface_agents-routing.md | 3 +- src/cli/capabilities-agents-routing.ts | 4 +- src/cli/combo-result.ts | 6 + src/cli/combo-stats.ts | 3 +- src/cli/combo.ts | 10 + src/combos/jev-session-pin-config.ts | 51 ++++ src/combos/jev.ts | 4 + src/combos/types.ts | 19 ++ src/server/management/combo-routes.ts | 2 + src/server/responses/core-combo.ts | 21 +- src/server/responses/jev-session-pin.ts | 170 ++++++++++++ src/types/config.ts | 6 + src/usage/jev-stats.ts | 55 +++- structure/config.md | 2 +- structure/gui-and-management-api.md | 2 +- structure/providers/jev-decision.md | 21 ++ tests/cli/cli-combo-session-pin.test.ts | 81 ++++++ tests/fixtures/test-layout-expected.json | 2 +- tests/helpers/responses-core-source.ts | 2 + tests/routing/jev-session-pin.test.ts | 206 ++++++++++++++ tests/server/server-jev-session-pin.test.ts | 252 ++++++++++++++++++ 26 files changed, 966 insertions(+), 17 deletions(-) create mode 100644 scripts/test-layout/layout-jev.json create mode 100644 src/combos/jev-session-pin-config.ts create mode 100644 src/server/responses/jev-session-pin.ts create mode 100644 tests/cli/cli-combo-session-pin.test.ts create mode 100644 tests/routing/jev-session-pin.test.ts create mode 100644 tests/server/server-jev-session-pin.test.ts diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 5118c2c1a90..49f1647e651 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -272,6 +272,54 @@ API `false` disables the switch. Omitted fields survive JEV updates, target repl rename, including saves through the existing editor. Switching away from JEV removes both quota settings. No new editor controls or background quota work are enabled. +### Session pin for JEV + +A JEV Combo decides a target for every request, so consecutive turns of one conversation can move +between providers, and each move starts with a cold provider prompt cache. `sessionPin` is an +opt-in setting per JEV Combo, off by default, that keeps a client session on the target JEV chose +for it while the session stays active: + +```bash +ocx combo set jev-auto --session-pin 20m --json +``` + +```json +{ "strategy": "jev", "sessionPin": { "ttlMs": 1200000 } } +``` + +The first request of a session is decided normally. Once the target JEV applied for it completes a +response, later requests of the same session reuse that target and effort without another decision +call. `ttlMs` (1 minute to 24 hours) is an idle window: every completed turn on the pinned target, +tool-call continuations included, starts it again, and a session that stays quiet longer is decided +afresh. + +A session is identified by the client's own session headers (`thread-id`, `session_id` or +`session-id`, `x-codex-parent-thread-id`, or a promoted `x-session-id`) together with the API key +that admitted the request and the Combo. Requests without a session header are never pinned, +`prompt_cache_key` is never used as a session identity, and two API keys never share a pin. + +JEV decides again, and the new decision replaces the pin, when: + +- the pinned target is not offered for this request: disabled, removed from the Combo, cooling + down after a 429 or another failure, out of cached quota, metered, or otherwise ineligible; +- the pinned effort is no longer offered for that target; +- `decisionQuotaSignals` is on and the target's quota evidence reads nearly exhausted; +- any send of a pinned session fails (that request still fails over as usual); +- the configuration has changed since the pin was written. + +A cancelled request keeps the pin. A fail-open decision (timeout, missing key, invalid answer) +never creates one. Pins live in the proxy's memory, at most 512 sessions, and do not survive a +restart. + +Usage records mark a reused turn with `jevDecision.pinned: true` and `pinReason: "session"`. A +decision that replaced a released pin records why in `pinReason` (`expired`, `config_changed`, +`target_unavailable`, `effort_unavailable` or `quota_nearly_exhausted`). `ocx combo stats` +counts pinned requests apart from decisions, and their model tokens still count per model. + +A pin makes provider cache reuse possible; whether the provider actually serves cached tokens still +depends on the provider, the account and a stable prompt prefix. `--session-pin -` (API `null`) +turns it off, switching away from JEV removes it, and saves from the dashboard editor keep it. + ### Decision method `strategy: "jev"` asks a decision backend to choose the first eligible target and a compatible diff --git a/docs-site/src/content/docs/reference/cli/agents.md b/docs-site/src/content/docs/reference/cli/agents.md index 31c87b8a1f1..583a68fbadd 100644 --- a/docs-site/src/content/docs/reference/cli/agents.md +++ b/docs-site/src/content/docs/reference/cli/agents.md @@ -271,6 +271,11 @@ switching away from JEV removes both settings. This adds quota data to the selec backend, not probes, new account selection or hard exclusions. See [quota defaults, freshness and privacy](/guides/combos/#advisory-quota-signals-for-jev). +`ocx combo set --session-pin ` (JEV only) keeps each client session on the target +JEV chose for it while the session stays active, so the provider prompt cache stays warm. The +duration is the idle window: `90s`, `20m`, `1h` or milliseconds, from 1 minute to 24 hours; `-` +turns it off. See [Session pin for JEV](/guides/combos/#session-pin-for-jev). + `ocx combo test [--combo ] [--decision-provider | --decision-model ] [--decision-timeout ]` sends one synthetic decision probe through a saved combo's method or an unsaved selection and reports the gate, backend, and latency; it may spend one decision call. diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 51503592756..e518b009789 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -178,6 +178,7 @@ namespace, and cannot use reserved bare native families such as `gpt-*`, `o1-*`, | `decisionTimeoutMs?` | `number` | `4000` | `strategy: "jev"` only. Decision deadline before failing open, 1000–120000 ms. | | `decisionQuotaSignals?` | `boolean` | `false` | JEV only. Explicit `true` sends loaded advisory quota evidence to the selected backend; never changes hard eligibility or account selection. | | `decisionQuotaTiers?` | `{ moderate?: number; limited?: number; nearlyExhausted?: number }` | `{ limited: 70, nearlyExhausted: 90 }` | JEV only. Partial overrides merge with defaults; finite 0–100 values must be strictly ascending. `moderate` is absent unless explicit. Does not enable signals by itself. | +| `sessionPin?` | `{ ttlMs: number }` | off | JEV only. Keeps an explicit client session on the target JEV chose for it while the session stays active; `ttlMs` is the idle window, 60000–86400000. `null` clears. See [Session pin for JEV](/guides/combos/#session-pin-for-jev). | Quota settings, privacy and unknown-pool semantics are described in [Advisory quota signals for JEV](/guides/combos/#advisory-quota-signals-for-jev). diff --git a/scripts/test-layout/layout-jev.json b/scripts/test-layout/layout-jev.json new file mode 100644 index 00000000000..dfe200d9834 --- /dev/null +++ b/scripts/test-layout/layout-jev.json @@ -0,0 +1,5 @@ +{ + "jev-session-pin.test.ts": "routing", + "server-jev-session-pin.test.ts": "server", + "cli-combo-session-pin.test.ts": "cli" +} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index e399ab3370c..fc03aa9fb38 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1,7 +1,7 @@ { "version": 1, "root": "tests", - "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json"], + "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-jev.json"], "explicit": { "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai", "owner-registry-isolation.test.ts": "config", diff --git a/skills/ocx/references/01_surface_agents-routing.md b/skills/ocx/references/01_surface_agents-routing.md index a3ebc10254b..dabc5a4b102 100644 --- a/skills/ocx/references/01_surface_agents-routing.md +++ b/skills/ocx/references/01_surface_agents-routing.md @@ -958,7 +958,7 @@ State-changing: no. JSON mode: `payload`. ### `ocx combo set` -Usage: `ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]` +Usage: `ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--session-pin ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]` Create, update or rename a combo. @@ -985,6 +985,7 @@ State-changing: yes. JSON mode: `payload`. | `--decision-timeout` | string | JEV deadline 1000-120000 ms; - clears. | | `--decision-quota-signals` | string | JEV advisory quota on/off; - clears. | | `--decision-quota-tiers` | string | Partial JSON thresholds; - clears. | +| `--session-pin` | string | JEV session pin idle window 1m-24h (90s, 20m, 1h); - clears. | | `--rename-from` | string | Existing combo ID to rename. | | `--json` | boolean | Emit structured JSON. | | `--native-alias` | string | Bare legacy flag means on; explicit on/off preserves presence, including false. | diff --git a/src/cli/capabilities-agents-routing.ts b/src/cli/capabilities-agents-routing.ts index f9d8d0b5e96..a5c84cea479 100644 --- a/src/cli/capabilities-agents-routing.ts +++ b/src/cli/capabilities-agents-routing.ts @@ -385,10 +385,10 @@ export const AGENT_ROUTING_CAPABILITIES: readonly Capability[] = [ }, { command: ["combo","set"], - usage: "ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]", + usage: "ocx combo set [--targets | --targets-file ] [--strategy ] [--sticky <1-100|->] [--effort ] [--effort-mode ] [--alias ] [--native-alias [on|off]] [--allow-metered-targets ] [--display-name ] [--description ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] [--session-pin ] [--rename-from ] [--image-input ] [--reasoning-effort-mode ] [--json]", summary: "Create, update or rename a combo.", routes: [{"method":"GET","path":"/api/combos"},{"method":"PUT","path":"/api/combos"}], - flags: [{"name":"--allow-metered-targets","value":"string","summary":"on grants operator consent to metered targets only on a loopback-only proxy; off clears it."},{"name":"--targets","value":"string","summary":"Ordered provider/model[:weight] list; required for creation."},{"name":"--strategy","value":"string","summary":"Routing strategy."},{"name":"--sticky","value":"string","summary":"Round-robin limit 1-100; - resets."},{"name":"--effort","value":"string","summary":"Default effort; - clears."},{"name":"--effort-mode","value":"string","summary":"fallback or force; - resets to fallback."},{"name":"--alias","value":"string","summary":"Alias; - clears."},{"name":"--display-name","value":"string","summary":"Label; - clears."},{"name":"--description","value":"string","summary":"Catalog row description; - clears."},{"name":"--decision-provider","value":"string","summary":"JEV provider; - clears."},{"name":"--decision-model","value":"string","summary":"JEV route; - clears; exclusive with provider."},{"name":"--decision-timeout","value":"string","summary":"JEV deadline 1000-120000 ms; - clears."},{"name":"--decision-quota-signals","value":"string","summary":"JEV advisory quota on/off; - clears."},{"name":"--decision-quota-tiers","value":"string","summary":"Partial JSON thresholds; - clears."},{"name":"--rename-from","value":"string","summary":"Existing combo ID to rename."},{"name":"--json","value":"boolean","summary":"Emit structured JSON."},{"name":"--native-alias","value":"string","summary":"Bare legacy flag means on; explicit on/off preserves presence, including false."},{"name":"--targets-file","value":"string","summary":"Bounded ordered target JSON array, exclusive with --targets; preserves effort/profile/last-resort metadata."},{"name":"--image-input","value":"string","summary":"auto or disabled; explicit auto clears the previous disabled choice."},{"name":"--reasoning-effort-mode","value":"string","summary":"strict or adaptive, distinct from --effort-mode fallback/force."}], + flags: [{"name":"--allow-metered-targets","value":"string","summary":"on grants operator consent to metered targets only on a loopback-only proxy; off clears it."},{"name":"--targets","value":"string","summary":"Ordered provider/model[:weight] list; required for creation."},{"name":"--strategy","value":"string","summary":"Routing strategy."},{"name":"--sticky","value":"string","summary":"Round-robin limit 1-100; - resets."},{"name":"--effort","value":"string","summary":"Default effort; - clears."},{"name":"--effort-mode","value":"string","summary":"fallback or force; - resets to fallback."},{"name":"--alias","value":"string","summary":"Alias; - clears."},{"name":"--display-name","value":"string","summary":"Label; - clears."},{"name":"--description","value":"string","summary":"Catalog row description; - clears."},{"name":"--decision-provider","value":"string","summary":"JEV provider; - clears."},{"name":"--decision-model","value":"string","summary":"JEV route; - clears; exclusive with provider."},{"name":"--decision-timeout","value":"string","summary":"JEV deadline 1000-120000 ms; - clears."},{"name":"--decision-quota-signals","value":"string","summary":"JEV advisory quota on/off; - clears."},{"name":"--decision-quota-tiers","value":"string","summary":"Partial JSON thresholds; - clears."},{"name":"--session-pin","value":"string","summary":"JEV session pin idle window 1m-24h (90s, 20m, 1h); - clears."},{"name":"--rename-from","value":"string","summary":"Existing combo ID to rename."},{"name":"--json","value":"boolean","summary":"Emit structured JSON."},{"name":"--native-alias","value":"string","summary":"Bare legacy flag means on; explicit on/off preserves presence, including false."},{"name":"--targets-file","value":"string","summary":"Bounded ordered target JSON array, exclusive with --targets; preserves effort/profile/last-resort metadata."},{"name":"--image-input","value":"string","summary":"auto or disabled; explicit auto clears the previous disabled choice."},{"name":"--reasoning-effort-mode","value":"string","summary":"strict or adaptive, distinct from --effort-mode fallback/force."}], mutates: true, json: "payload", details: ["create and update are aliases of set. Existing fields are read before writing; --targets replaces targets.","force overrides valid client effort and may increase cost and latency.","Only the implemented target grammar is supported; full GUI target metadata and routing-profile writes are separate tasks.","New file/flag conflicts refuse before transport. Partial edits preserve target metadata; explicit false/default values override server carry. Turning a native alias off may also require clearing its incompatible alias.","Read/merge/write uses one target and remains an upsert without revision CAS. Saved-but-incomplete catalog outcomes return nonzero with their receipt."], diff --git a/src/cli/combo-result.ts b/src/cli/combo-result.ts index 040d099d38c..b98b9560547 100644 --- a/src/cli/combo-result.ts +++ b/src/cli/combo-result.ts @@ -1,4 +1,5 @@ import { normalizeJevQuotaTiers } from "../combos/jev-quota-config"; +import { normalizeJevSessionPin } from "../combos/jev-session-pin-config"; import { parseComboTargets } from "./combo-input"; import { printCatalogResult } from "./catalog-command-result"; @@ -39,6 +40,11 @@ function publicCombo(value: unknown): Record { if (!tiers) invalid(); result[key] = tiers; continue; + } else if (key === "sessionPin") { + const pin = normalizeJevSessionPin(child); + if (!pin) invalid(); + result[key] = pin; + continue; } else if (key === "nativeAlias" || key === "allowMeteredTargets" || key === "decisionQuotaSignals") { if (typeof child !== "boolean") invalid(); } else invalid(); diff --git a/src/cli/combo-stats.ts b/src/cli/combo-stats.ts index 57d6ca42c74..2ee6d9a55a7 100644 --- a/src/cli/combo-stats.ts +++ b/src/cli/combo-stats.ts @@ -4,7 +4,7 @@ import { CliUsageError, printData, runtimeBaseUrl, runtimeRequest, takeFlag, tak const USAGE = "Usage: ocx combo stats [--range <7d|30d|all>] [--json]"; const SUMMARY_COUNTS = [ - "decisions", "appliedDecisions", "failOpenDecisions", "successfulRequests", "requestsWithModelFallback", + "decisions", "appliedDecisions", "failOpenDecisions", "pinnedRequests", "successfulRequests", "requestsWithModelFallback", "modelAttempts", "measuredModelAttempts", "modelInputTokens", "modelOutputTokens", "modelReasoningTokens", "modelCacheReadTokens", "modelCacheWriteTokens", "modelTotalTokens", "decisionUsageReported", "decisionInputTokens", "decisionOutputTokens", "decisionTotalTokens", @@ -111,6 +111,7 @@ export async function handleComboStatsCommand(argv: string[], deps: RuntimeApiDe `Combo ${id} — JEV observations (${range})`, ...(summary.decisions === 0 ? ["No recorded JEV decisions in this window."] : []), `Decisions: ${summary.decisions}; applied: ${summary.appliedDecisions}; fail-open: ${summary.failOpenDecisions}`, + ...(summary.pinnedRequests > 0 ? [`Session-pinned requests (no decision call): ${summary.pinnedRequests}`] : []), `Successful requests: ${summary.successfulRequests}; requests with model fallback: ${summary.requestsWithModelFallback}`, `Model tokens: ${summary.modelTotalTokens}; measured attempts: ${summary.measuredModelAttempts}/${summary.modelAttempts}`, `Decision tokens: ${summary.decisionTotalTokens}; decisions reporting usage: ${summary.decisionUsageReported}/${summary.decisions}`, diff --git a/src/cli/combo.ts b/src/cli/combo.ts index e93295e202c..8c8714fa649 100644 --- a/src/cli/combo.ts +++ b/src/cli/combo.ts @@ -1,4 +1,5 @@ import { normalizeJevQuotaTiers } from "../combos/jev-quota-config"; +import { parseJevSessionPinDuration } from "../combos/jev-session-pin-config"; import { JEV_DECISION_TIMEOUT_MAX_MS, JEV_DECISION_TIMEOUT_MIN_MS } from "../combos/types"; import { CliUsageError, @@ -29,6 +30,7 @@ const USAGE = `Usage: [--image-input ] [--reasoning-effort-mode ] [--decision-provider | --decision-model ] [--decision-timeout ] [--decision-quota-signals ] [--decision-quota-tiers ] + [--session-pin ] (e.g. 20m; keeps a session on JEV's target while active) (jev only; the provider must be a configured jev-decision row) [--rename-from ] [--json] ocx combo stats [--range <7d|30d|all>] [--json] @@ -141,6 +143,12 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise { try { parsedQuotaTiers = quotaTiers === undefined || quotaTiers === "-" ? null : normalizeJevQuotaTiers(JSON.parse(quotaTiers)); } catch { throw new CliUsageError("invalid --decision-quota-tiers JSON", USAGE); } if (quotaTiers !== undefined && quotaTiers !== "-" && !parsedQuotaTiers) throw new CliUsageError("quota tiers must be finite 0..100 and strictly ascending", USAGE); if (strategy !== "jev" && (quotaSignals !== undefined && quotaSignals !== "-" || quotaTiers !== undefined && quotaTiers !== "-")) throw new CliUsageError("quota settings apply only to jev", USAGE); + const sessionPinRaw = takeOption(args, "--session-pin"); + const sessionPinMs = sessionPinRaw === undefined || sessionPinRaw === "-" ? undefined : parseJevSessionPinDuration(sessionPinRaw); + if (sessionPinRaw !== undefined && sessionPinRaw !== "-" && sessionPinMs === undefined) { + throw new CliUsageError("--session-pin must be a duration from 1m to 24h (for example 90s, 20m, 1h, or milliseconds), or -", USAGE); + } + if (sessionPinMs !== undefined && strategy !== "jev") throw new CliUsageError("--session-pin applies only to the jev strategy", USAGE); const decisionTimeout = takeOption(args, "--decision-timeout"); let decisionTimeoutMs: number | null | undefined; if (decisionTimeout !== undefined) { @@ -184,12 +192,14 @@ async function set(argv: string[], deps: RuntimeApiDeps): Promise { if (decisionTimeoutMs !== undefined) combo.decisionTimeoutMs = decisionTimeoutMs; if (quotaSignals !== undefined) combo.decisionQuotaSignals = quotaSignals === "-" ? null : quotaSignals === "on"; if (quotaTiers !== undefined) combo.decisionQuotaTiers = parsedQuotaTiers; + if (sessionPinRaw !== undefined) combo.sessionPin = sessionPinMs === undefined ? null : { ttlMs: sessionPinMs }; if (strategy !== "jev") { delete combo.decisionProvider; delete combo.decisionModel; delete combo.decisionTimeoutMs; delete combo.decisionQuotaSignals; delete combo.decisionQuotaTiers; + delete combo.sessionPin; } const current = partialCurrent ?? await runtimeRequest<{ combos?: ComboRow[] }>("/api/combos", { redirect: "error" }, deps); const existing = (current.combos ?? []).find(row => row.id === (renameFrom ?? id)); diff --git a/src/combos/jev-session-pin-config.ts b/src/combos/jev-session-pin-config.ts new file mode 100644 index 00000000000..94ec4838877 --- /dev/null +++ b/src/combos/jev-session-pin-config.ts @@ -0,0 +1,51 @@ +/** + * Opt-in JEV session pin settings (#7525). Dependency-free so combo validation, the management + * API and the CLI share one grammar and one range. + */ + +/** Shortest idle window a pin may be configured with. */ +export const JEV_SESSION_PIN_TTL_MIN_MS = 60_000; +/** Longest idle window a pin may be configured with (one day). */ +export const JEV_SESSION_PIN_TTL_MAX_MS = 86_400_000; + +export interface JevSessionPinConfig { + /** Idle window after the last successful pinned turn; a quieter session is decided afresh. */ + ttlMs: number; +} + +/** + * Closed persisted vocabulary: `session` marks a turn that reused its pin; every other value names + * why an existing pin was released before a fresh decision. + */ +export const JEV_SESSION_PIN_REASONS = [ + "session", + "expired", + "config_changed", + "target_unavailable", + "effort_unavailable", + "quota_nearly_exhausted", +] as const; +export type JevSessionPinReason = (typeof JEV_SESSION_PIN_REASONS)[number]; +export type JevSessionPinRelease = Exclude; + +/** The canonical pin setting, or undefined for anything that is not exactly `{ ttlMs }` in range. */ +export function normalizeJevSessionPin(value: unknown): JevSessionPinConfig | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const keys = Object.keys(value); + if (keys.length !== 1 || keys[0] !== "ttlMs") return undefined; + const ttlMs = (value as { ttlMs?: unknown }).ttlMs; + return typeof ttlMs === "number" && Number.isSafeInteger(ttlMs) + && ttlMs >= JEV_SESSION_PIN_TTL_MIN_MS && ttlMs <= JEV_SESSION_PIN_TTL_MAX_MS + ? { ttlMs } + : undefined; +} + +const DURATION_UNITS_MS: Readonly> = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000 }; + +/** Parse `90s`, `20m`, `1h` or bare milliseconds into an in-range TTL, else undefined. */ +export function parseJevSessionPinDuration(raw: string): number | undefined { + const match = /^(\d{1,9})(ms|s|m|h)?$/.exec(raw.trim()); + if (!match) return undefined; + const ttlMs = Number(match[1]) * DURATION_UNITS_MS[match[2] ?? "ms"]!; + return ttlMs >= JEV_SESSION_PIN_TTL_MIN_MS && ttlMs <= JEV_SESSION_PIN_TTL_MAX_MS ? ttlMs : undefined; +} diff --git a/src/combos/jev.ts b/src/combos/jev.ts index 0987c6746a1..aa4dd02eedd 100644 --- a/src/combos/jev.ts +++ b/src/combos/jev.ts @@ -109,6 +109,10 @@ export interface JevDecision { confidence?: number; chosenProbability?: number; usage?: Record; + /** Set when an opt-in session pin (#7525) supplied this decision without asking the backend. */ + pinned?: true; + /** `session` with `pinned`; otherwise why an existing session pin was released first. */ + pinReason?: import("./jev-session-pin-config").JevSessionPinReason; } export interface ResolveJevDecisionOptions { diff --git a/src/combos/types.ts b/src/combos/types.ts index 1cd2c0a71c7..f3256fe43c8 100644 --- a/src/combos/types.ts +++ b/src/combos/types.ts @@ -1,4 +1,10 @@ import { normalizeJevQuotaTiers, type ResolvedJevQuotaTiers } from "./jev-quota-config"; +import { + JEV_SESSION_PIN_TTL_MAX_MS, + JEV_SESSION_PIN_TTL_MIN_MS, + normalizeJevSessionPin, + type JevSessionPinConfig, +} from "./jev-session-pin-config"; import { isCodexReasoningEffort, isDeclaredReasoningEffort } from "../reasoning-effort"; import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models"; import type { OcxComboConfig, OcxComboCooldownWaitPolicy, OcxComboDefaultEffort, OcxComboDefaultEffortMode, OcxComboReasoningEffortMode, OcxComboStrategy, OcxComboTarget, OcxProviderConfig } from "../types"; @@ -79,6 +85,8 @@ export interface NormalizedComboConfig { decisionTimeoutMs?: number; decisionQuotaSignals?: boolean; decisionQuotaTiers?: ResolvedJevQuotaTiers; + /** JEV only, opt-in: keep a session on the target JEV chose for it (#7525). */ + sessionPin?: JevSessionPinConfig; targets: NormalizedComboTarget[]; } @@ -310,6 +318,16 @@ export function comboConfigIssues( if (field === "decisionQuotaSignals" ? typeof body[field] !== "boolean" : !normalizeJevQuotaTiers(body[field])) issues.push({ path: [field], message: field === "decisionQuotaSignals" ? "decisionQuotaSignals must be a boolean" : "decisionQuotaTiers must contain finite 0..100 strictly ascending thresholds" }); } + if (body.sessionPin !== undefined && body.sessionPin !== null) { + if (body.strategy !== "jev") { + issues.push({ path: ["sessionPin"], message: 'sessionPin is only valid with strategy "jev"' }); + } else if (!normalizeJevSessionPin(body.sessionPin)) { + issues.push({ + path: ["sessionPin"], + message: `sessionPin must be { "ttlMs": }`, + }); + } + } if (body.decisionProvider !== undefined && body.decisionProvider !== null) { const decisionProvider = typeof body.decisionProvider === "string" ? body.decisionProvider.trim() : ""; if (!decisionProvider) { @@ -533,6 +551,7 @@ export function normalizeComboConfig(raw: OcxComboConfig): NormalizedComboConfig ...(typeof raw.decisionTimeoutMs === "number" ? { decisionTimeoutMs: raw.decisionTimeoutMs } : {}), ...(raw.strategy === "jev" && raw.decisionQuotaSignals === true ? { decisionQuotaSignals: true } : {}), ...(raw.strategy === "jev" && normalizeJevQuotaTiers(raw.decisionQuotaTiers) ? { decisionQuotaTiers: normalizeJevQuotaTiers(raw.decisionQuotaTiers)! } : {}), + ...(raw.strategy === "jev" && normalizeJevSessionPin(raw.sessionPin) ? { sessionPin: normalizeJevSessionPin(raw.sessionPin)! } : {}), targets: raw.targets.map(target => ({ provider: target.provider.trim(), model: target.model.trim(), diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index 2eae49c2f82..89da58994c8 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -225,6 +225,8 @@ export async function handleComboRoutes(ctx: ManagementContext): Promise choice.candidate), + combo.decisionQuotaSignals === true ? { config, tiers: combo.decisionQuotaTiers } : undefined, + ); let decision: JevDecision; - try { + if (pinReuse?.decision) decision = pinReuse.decision; + else try { decision = await resolveJevComboDecision({ body, candidates: choices.map(choice => choice.candidate), @@ -709,6 +719,7 @@ export async function executeComboResponses( latencyMs: Math.max(0, Date.now() - decisionStartedAt), }; } + if (pinReuse?.released) decision = { ...decision, pinReason: pinReuse.released }; jevDecision = decision; const selected = choices.find(choice => choice.candidate.key === decision.targetKey) ?? first; pick = { ...selected.pick, attempted: [targetKey(selected.pick.target)] }; @@ -729,6 +740,8 @@ export async function executeComboResponses( : {}), ...(decision.usage ? { usage: decision.usage } : {}), ...(decision.quota ? { quota: decision.quota } : {}), + ...(decision.pinned ? { pinned: true } : {}), + ...(decision.pinReason ? { pinReason: decision.pinReason } : {}), }); console.debug("[combo] JEV decision", { backend: decision.backend, @@ -736,6 +749,7 @@ export async function executeComboResponses( effort: decision.effort, gate: decision.gate, latencyMs: decision.latencyMs, + ...(decision.pinReason ? { pinReason: decision.pinReason } : {}), ...(decision.confidence !== undefined ? { confidence: decision.confidence } : {}), ...(decision.chosenProbability !== undefined ? { chosenProbability: decision.chosenProbability } @@ -748,7 +762,7 @@ export async function executeComboResponses( const decision = comboRouteDecisionTrace(config, comboId, pick, requestedModel); const selectedReason = options.memoryModelPhase ? memoryModelRouteReason(options.memoryModelPhase) - : toolLoopPin ? COMBO_TOOL_LOOP_PIN_REASON : undefined; + : toolLoopPin ? COMBO_TOOL_LOOP_PIN_REASON : jevDecision?.pinned ? JEV_SESSION_PIN_ROUTE_REASON : undefined; logCtx.routeDecision = selectedReason ? { ...decision, selected: { ...decision.selected, reason: selectedReason } } : decision; @@ -991,6 +1005,7 @@ export async function executeComboResponses( && provider && provider.disabled !== true && currentCombo?.targets.some(target => targetKey(target) === targetKey(completedTarget))) { rememberComboForLane(sessionLaneIdFromRequest(req.headers), comboId, completedTarget, model, writerGeneration); + jevPin?.complete(targetKey(completedTarget), initialJevDecision, writerGeneration); } options.onResponseComplete?.(model); }, @@ -1187,6 +1202,8 @@ export async function executeComboResponses( } attemptRetained = true; lastFailure = failure.response; + // A real target failure ends the session pin; the next turn is decided afresh (#7525). + jevPin?.forget(); if (!quotaFailure && (lastFailure.status === 429 || lastFailure.status === 402)) { quotaFailure = lastFailure; quotaFailedChildLog = childLog; diff --git a/src/server/responses/jev-session-pin.ts b/src/server/responses/jev-session-pin.ts new file mode 100644 index 00000000000..1856f9d9f36 --- /dev/null +++ b/src/server/responses/jev-session-pin.ts @@ -0,0 +1,170 @@ +/** + * Opt-in JEV session pin (#7525). + * + * A JEV Combo decides a target per request, so consecutive turns of one conversation can land on + * different providers and each switch starts with a cold provider prompt cache. With + * `sessionPin` set, a session keeps the target JEV chose for it while the session stays active. + * + * Identity. A pin is keyed by the admitted caller (the salted context principal of the matched + * data-plane key, or `loopback`), the Combo id and the explicit session lane from + * `sessionLaneIdFromRequest` (thread, session and parent-thread headers). A request without an + * explicit session identity never pins, and a body field such as `prompt_cache_key` is never an + * identity on its own. + * + * Lifetime. A pin is written only after the target JEV applied for this session completes a + * response, and it is refreshed by every later completion on that same target, including tool-loop + * continuations. It is released when the session stays idle past `ttlMs`, when any config + * generation change happens after it was written, when its target is not among the request's + * current JEV choices (disabled, removed, cooling down, quota exhausted, metered or otherwise + * ineligible), when its effort is no longer offered, when opted-in quota evidence reads + * `nearly_exhausted`, and when any target of a pinned session fails. A cancelled send writes + * and releases nothing. Storage is process-local and bounded by `JEV_SESSION_PIN_CAPACITY`. + */ +import type { JevCandidate, JevDecision, JevDecisionBackend } from "../../combos/jev"; +import { jevQuotaSignalForTarget } from "../../combos/jev-quota"; +import { JEV_QUOTA_DEFAULT_TIERS, normalizeJevQuotaTiers, type JevQuotaTiers } from "../../combos/jev-quota-config"; +import type { JevSessionPinRelease } from "../../combos/jev-session-pin-config"; +import type { NormalizedComboConfig } from "../../combos/types"; +import { captureConfigGeneration } from "../../lib/state-store-sweeper"; +import type { OcxComboDefaultEffort, OcxConfig } from "../../types"; +import type { DataPlaneAdmission } from "../auth-cors"; + +export const JEV_SESSION_PIN_ROUTE_REASON = "jev-session-pin"; +export const JEV_SESSION_PIN_CAPACITY = 512; + +interface PinEntry { + targetKey: string; + effort: OcxComboDefaultEffort | null; + backend: JevDecisionBackend; + at: number; + generation: number; +} + +const pins = new Map(); + +function callerNamespace(admission: DataPlaneAdmission | undefined): string { + if (!admission) return "unadmitted"; + if (admission.kind === "loopback") return "loopback"; + if (admission.contextPrincipalId) return `principal:${admission.contextPrincipalId}`; + return admission.kind === "configured" ? `key:${admission.keyId}` : admission.kind; +} + +function write(key: string, entry: PinEntry): void { + // Insertion order is recency order; evicting from the front drops the least recent session. + pins.delete(key); + pins.set(key, entry); + while (pins.size > JEV_SESSION_PIN_CAPACITY) { + const oldest = pins.keys().next().value; + if (oldest === undefined) break; + pins.delete(oldest); + } +} + +export interface JevSessionPinReuse { + /** The pinned decision to apply instead of asking the decision backend. */ + decision?: JevDecision; + /** Set when a pin existed and was released instead of reused. */ + released?: JevSessionPinRelease; +} + +export interface JevSessionPinScope { + /** + * Reuse the session's pin against this request's current JEV choices, or report its release. + * `quota` is passed only when the Combo opted into advisory quota evidence; it is read from + * loaded snapshots exactly as the decision payload would read it. + */ + reuse( + choices: readonly JevCandidate[], + quota?: { config: OcxConfig; tiers?: JevQuotaTiers }, + ): JevSessionPinReuse; + /** + * Record a completed response. With the request's applied JEV decision it (re)writes the pin + * when the completed target is the decided one; without one it only refreshes a pin that + * already names the completed target. + */ + complete(completedTargetKey: string, decision: JevDecision | undefined, writerGeneration: number): void; + /** Drop the session's pin after a real target failure. */ + forget(): void; +} + +/** The pin scope for one Combo request, or undefined when this request can never pin. */ +export function jevSessionPinScope(args: { + combo: Pick; + comboId: string; + admission: DataPlaneAdmission | undefined; + lane: string | undefined; + now?: () => number; +}): JevSessionPinScope | undefined { + const ttlMs = args.combo.sessionPin?.ttlMs; + if (args.combo.strategy !== "jev" || ttlMs === undefined || !args.lane) return undefined; + const key = `${callerNamespace(args.admission)}\u0000${args.comboId}\u0000${args.lane}`; + const now = args.now ?? Date.now; + const release = (reason: JevSessionPinRelease): JevSessionPinReuse => { + pins.delete(key); + return { released: reason }; + }; + return { + reuse(choices, quota) { + const entry = pins.get(key); + if (!entry) return {}; + if (entry.generation !== captureConfigGeneration()) return release("config_changed"); + const at = now(); + if (at < entry.at || at - entry.at >= ttlMs) return release("expired"); + const candidate = choices.find(choice => choice.key === entry.targetKey); + if (!candidate) return release("target_unavailable"); + if (entry.effort !== null && !candidate.reasoningEfforts.includes(entry.effort)) { + return release("effort_unavailable"); + } + if (quota && candidate.quotaIdentity !== null) { + const identity = candidate.quotaIdentity ?? candidate; + const tiers = normalizeJevQuotaTiers(quota.tiers) ?? JEV_QUOTA_DEFAULT_TIERS; + const signal = jevQuotaSignalForTarget(quota.config, identity.provider, identity.model, at, tiers); + if (signal.tier === "nearly_exhausted") return release("quota_nearly_exhausted"); + } + return { + decision: { + backend: entry.backend, + targetKey: entry.targetKey, + effort: entry.effort, + gate: "apply", + latencyMs: 0, + pinned: true, + pinReason: "session", + }, + }; + }, + complete(completedTargetKey, decision, writerGeneration) { + // A config change while the response was in flight makes this completion obsolete. + if (writerGeneration !== captureConfigGeneration()) return; + if (!decision) { + const entry = pins.get(key); + if (entry && entry.targetKey === completedTargetKey && entry.generation === writerGeneration) { + write(key, { ...entry, at: now() }); + } + return; + } + // Only an applied decision pins; a fail-open fallback leaves the next turn to decide again. + if (decision.gate !== "apply" || decision.targetKey !== completedTargetKey) return; + write(key, { + targetKey: decision.targetKey, + effort: decision.effort, + backend: decision.backend, + at: now(), + generation: writerGeneration, + }); + }, + forget() { + pins.delete(key); + }, + }; +} + +/** Test-only reset, alongside the Combo recall and rotation resets. */ +export function clearJevSessionPinsForTests(): void { + pins.clear(); +} + +/** Test-only view of the retained pin count. */ +export function jevSessionPinCountForTests(): number { + return pins.size; +} diff --git a/src/types/config.ts b/src/types/config.ts index 8be911294df..479895aa2c4 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1496,6 +1496,12 @@ export interface OcxComboConfig { decisionQuotaSignals?: boolean | null; /** Partial 0..100 tier overrides; defaults limited 70/nearlyExhausted 90. null clears. */ decisionQuotaTiers?: import("../combos/jev-quota-config").JevQuotaTiers | null; + /** + * `strategy: "jev"` only, default off: keep an explicit client session on the target JEV chose + * for it while the session stays active, so the provider prompt cache stays warm (#7525). + * `ttlMs` is the idle window after the last successful turn (60000..86400000). null clears. + */ + sessionPin?: import("../combos/jev-session-pin-config").JevSessionPinConfig | null; } export type OcxRoutingUnknownEvidenceMode = "allow" | "penalize" | "exclude"; diff --git a/src/usage/jev-stats.ts b/src/usage/jev-stats.ts index fe61f79aa4b..fb06ce7ac8d 100644 --- a/src/usage/jev-stats.ts +++ b/src/usage/jev-stats.ts @@ -1,4 +1,5 @@ import { normalizeJevQuotaSummary, type JevQuotaDecisionSummary } from "../combos/jev-quota-config"; +import { JEV_SESSION_PIN_REASONS, type JevSessionPinReason } from "../combos/jev-session-pin-config"; import type { OcxComboDefaultEffort } from "../types"; import type { PersistedUsageEntry } from "./log"; import { usageDisplayTotalTokens } from "./totals"; @@ -48,6 +49,10 @@ export interface PersistedJevDecisionV1 { outputTokens: number; totalTokens: number; }; + /** The turn reused its session pin (#7525); no decision request was sent for it. */ + pinned?: true; + /** `session` with `pinned`; otherwise why an existing session pin was released. */ + pinReason?: JevSessionPinReason; } export interface JevStatsModelRow { @@ -88,6 +93,8 @@ export interface JevStatsResponse { decisions: number; appliedDecisions: number; failOpenDecisions: number; + /** Requests that reused a session pin; they are not decisions and add only physical attempts. */ + pinnedRequests: number; successfulRequests: number; requestsWithModelFallback: number; modelAttempts: number; @@ -188,6 +195,13 @@ export function normalizePersistedJevDecision(value: unknown): PersistedJevDecis const backend = value.backend === "typesafe" || value.backend === "systemone" || value.backend === "model" ? value.backend : undefined; + const pinReason = typeof value.pinReason === "string" + && (JEV_SESSION_PIN_REASONS as readonly string[]).includes(value.pinReason) + ? value.pinReason as JevSessionPinReason + : undefined; + // A pinned turn always says so with `session`; any other pairing is not a record this writes. + const pinned = value.pinned === true && pinReason === "session"; + if ((value.pinned === true || pinReason === "session") && !pinned) return undefined; return { version: 1, comboId, @@ -203,6 +217,8 @@ export function normalizePersistedJevDecision(value: unknown): PersistedJevDecis ...(confidence !== undefined ? { confidence } : {}), ...(chosenProbability !== undefined ? { chosenProbability } : {}), ...(usage ? { usage } : {}), + ...(pinned ? { pinned: true as const } : {}), + ...(pinReason ? { pinReason } : {}), }; } @@ -210,6 +226,24 @@ interface MutableModelRow extends Omit { effortCounts: Map; } +interface SentAttempt { + attempt: NonNullable[number]; + provider: string; + model: string; + sendCount: number; +} + +/** Attempts that physically sent, with bounded identities; zero-send rows never count. */ +function sentAttempts(entry: PersistedUsageEntry): SentAttempt[] { + return (entry.attempts ?? []).flatMap(attempt => { + const sendCount = boundedCount(attempt.sendCount) ?? 0; + if (sendCount === 0) return []; + const provider = boundedIdentity(attempt.provider, MAX_TARGET_IDENTITY_CHARS); + const model = boundedIdentity(attempt.model, MAX_TARGET_IDENTITY_CHARS); + return provider && model ? [{ attempt, provider, model, sendCount }] : []; + }); +} + interface JevStatsAccumulatorOptions { comboId?: string | null; since?: number | null; @@ -291,6 +325,7 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { private snapshotWindowStart: number | null = null; private snapshotWindowEnd: number | null = null; private decisions = 0; + private pinnedRequests = 0; private appliedDecisions = 0; private successfulRequests = 0; private requestsWithModelFallback = 0; @@ -358,6 +393,12 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { || (this.until !== undefined && entry.timestamp > this.until)) return; const decision = normalizePersistedJevDecision(entry.jevDecision); if (!decision || (this.comboId !== null && decision.comboId !== this.comboId)) return; + if (decision.pinned) { + // A session-pinned turn sent no decision request: it adds physical attempts and tokens only. + this.pinnedRequests = saturatingAdd(this.pinnedRequests, 1); + this.addAttempts(sentAttempts(entry)); + return; + } const backend = decision.backend ?? "unknown"; const backendRow = this.backends.get(backend) @@ -397,17 +438,15 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { saturatingAdd(selected.effortCounts.get(decision.selected.effort) ?? 0, 1), ); - const attempts = (entry.attempts ?? []).flatMap(attempt => { - const sendCount = boundedCount(attempt.sendCount) ?? 0; - if (sendCount === 0) return []; - const provider = boundedIdentity(attempt.provider, MAX_TARGET_IDENTITY_CHARS); - const model = boundedIdentity(attempt.model, MAX_TARGET_IDENTITY_CHARS); - return provider && model ? [{ attempt, provider, model, sendCount }] : []; - }); + const attempts = sentAttempts(entry); if (attempts.some(({ provider, model }) => provider !== decision.selected.provider || model !== decision.selected.model)) { this.requestsWithModelFallback = saturatingAdd(this.requestsWithModelFallback, 1); } + this.addAttempts(attempts); + } + + private addAttempts(attempts: readonly SentAttempt[]): void { for (const { attempt, provider, model: modelId, sendCount } of attempts) { const model = this.rowFor(provider, modelId); model.attempts = saturatingAdd(model.attempts, sendCount); @@ -449,6 +488,7 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { cloned.snapshotWindowStart = this.snapshotWindowStart; cloned.snapshotWindowEnd = this.snapshotWindowEnd; cloned.decisions = this.decisions; + cloned.pinnedRequests = this.pinnedRequests; cloned.appliedDecisions = this.appliedDecisions; cloned.successfulRequests = this.successfulRequests; cloned.requestsWithModelFallback = this.requestsWithModelFallback; @@ -492,6 +532,7 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { decisions: this.decisions, appliedDecisions: this.appliedDecisions, failOpenDecisions: this.decisions - this.appliedDecisions, + pinnedRequests: this.pinnedRequests, successfulRequests: this.successfulRequests, requestsWithModelFallback: this.requestsWithModelFallback, modelAttempts: this.modelAttempts, diff --git a/structure/config.md b/structure/config.md index 10271963372..5eafa2bae03 100644 --- a/structure/config.md +++ b/structure/config.md @@ -2,7 +2,7 @@ Pasted GCP JSON persists only as `gcp-sa:` references in one pool transaction in `src/providers/api-keys.ts`; post-commit deletion follows [Vertex credential markers](providers/google.md#vertex-credential-markers). `src/config/schema/leaf-validators.ts` accepts only boolean `apiKeys[].allowMeteredComboTargets`; malformed entries are dropped rather than promoted. `src/types/config.ts` and combo validation define optional boolean target `metered`, normalized false when omitted, and optional per-combo `allowMeteredTargets` (default false, literal boolean validation shared by config schema and management API). Explicit consent is required under the [combo runtime contract](runtime.md#request-local-target-compatibility). Model-price overrides follow the [pricing contract](pricing.md). Quota activation reuses the existing next-reset fields without adding a polling configuration key. See the [quota activation contract](providers/openai-tiers.md#public-provider-contract). `src/types/config.ts` defines `anthropicAccountPool.routes` as ordered model rules. `src/oauth/anthropic-model-routes.ts` validates bounded names, full case-sensitive globs and stored account IDs; `src/config/diagnostics.ts` rejects malformed candidate writes. Load retains a malformed hand edit so unrelated providers survive, while the enabled Anthropic selector refuses it locally before dispatch. Saved rules remain inert when the pool is disabled; affinity is process-local. Request logs use only the rule’s 1-based `route:#` position, never its configured name. `src/cli/hub-disable.ts` removes interdependent hub/loopback/role/hostname settings through `mutatePersistedConfig` and `validateConfigCandidate` in one commit. Top-level deletion intent uses `src/config/rebase-provenance.ts` and authoritative disk provenance in `src/config/live-reconcile.ts` to preserve removed hub bindings through later live saves, `src/config/live-topology.ts` keeps an independent active-topology baseline and merges desired next-start settings into a separate persistence projection, retaining actual socket identity and live hub/client topology during guarded saves and OAuth disk adoption. Client connections and runtimeRole remain one topology unit through connect/disconnect, including fresh disk-only connections. Disk additions/value updates and deletions all survive repeated saves without changing active topology. `src/config/api-key-reconcile.ts` merges key rows by stable ID under the existing save lock; either side's revocation defeats stale edits while disjoint names and rotations merge; unchanged live key membership/order adopts persisted ordering. `tests/config/hub-disable-live-save.test.ts` exercises disable followed by live key creation, rename, rotation, unrelated settings and repeated saves. Saved API keys have no immutable enrollment marker; explicit exact-ID revocation preserves all unselected keys. -JEV `decisionQuotaSignals` and partial `decisionQuotaTiers` use the shared validator and sparse preserve/clear/strategy-switch contract in [loaded advisory quota](providers/jev-decision.md#loaded-advisory-quota). Native reasoning retention follows [its optional configuration contract](transports/native-reasoning-retention.md#configuration-and-management). Native function-result injection follows [the separate opt-in control contract](transports/streaming-health.md#experimental-native-function-result-injection); this surface does not infer upstream support or alter its defaults. +JEV `decisionQuotaSignals` and partial `decisionQuotaTiers` use the shared validator and sparse preserve/clear/strategy-switch contract in [loaded advisory quota](providers/jev-decision.md#loaded-advisory-quota); the opt-in JEV `sessionPin` follows the same contract under [session pin](providers/jev-decision.md#session-pin). Native reasoning retention follows [its optional configuration contract](transports/native-reasoning-retention.md#configuration-and-management). Native function-result injection follows [the separate opt-in control contract](transports/streaming-health.md#experimental-native-function-result-injection); this surface does not infer upstream support or alter its defaults. Native steering follows [the shared WebSocket contract](transports/streaming-health.md#experimental-native-mid-turn-steering); this surface's defaults remain unchanged. diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 72cb0febe05..c1697cb4865 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -311,7 +311,7 @@ after the save, and a non-retryable skip (no managed catalog) is a clean save. C | File-integration plans | `src/server/management/integration-routes.ts` and `aside-profile-routes.ts` — `POST /api/client-integrations/preview`, `POST /api/client-integrations/restore/preview`, and `POST /api/client-integrations/aside/profiles/{profileId}/preview`. Management-authenticated, declared non-mutating, and they write nothing: no snapshot, no lock, no maintenance, no recovery. They answer `409 integration_preview_unavailable` rather than gathering a model roster, because discovery refreshes credentials and writes the provider cache. Responses carry only declared managed schema paths, closed change kinds and an opaque fingerprint; no value, filesystem location or selected member identity appears. Mutation routes accept `operation` and `planFingerprint` together or not at all, reject a half-bound request and an operation that disagrees with the change, and answer `409 integration_preview_stale` with a freshly computed plan. Binding is an optimistic token, never authorization. [The integration contract](clients/integrations.md) owns the ordering. | | Grok reset coupons | `src/server/management/grok-coupon-routes.ts` — `GET /api/grok/reset-coupons`, `POST /api/grok/reset-coupons/consume`. The dashboard owner is `gui/src/hooks/useGrokResetCoupons.ts` with `gui/src/components/provider-workspace/GrokResetCoupons.tsx`, wired into the xAI OAuth rows of `ProviderAuthPanel`. Redemption truth is the settled ledger `code`, not the HTTP status: a replayed failure returns 200 with `replayed: true`. See [`providers/xai-grok.md`](providers/xai-grok.md). The route returns `ledger_unavailable` before dispatch when the shared ledger cannot be read or locked; stale uncertain attempts are inspected without another redemption or inferred success. A late inspection refusal or a lost claim replays an identity-matching definitive winner; otherwise it returns `operation_state_changed` (late refusal) or `attempt_in_progress` (lost claim) and never replaces an attempted or terminal record. Unknown-outcome responses include the effective operation ID; the CLI mints it before dispatch. Uncertain and pending attempts live in a module-level hold store shared by all hook instances and mirrored to `sessionStorage` (`ocx.grok-coupon-holds.v1`), so they survive modal close/reopen, remount, and reload for the tab session; the pending hold is written before the POST, only a definitive response for the same `operationId` clears it, and only GET refreshes are allowed after uncertainty. Every response that cannot prove settlement under a supplied operation ID, including pre-open refusals, carries that `operationId`. A definitive failure releases the matching speculative pending hold. Supplied token IDs must be non-empty strings before journal admission, and omitted account/token retries recover the operation’s durable identity within retention. Legacy version-one opens are quarantined rather than dispatched. After confirmed redemption, local settlement has five attempts and four 20 ms waits only for a `ConfigMutationLockError` caused by `SQLITE_BUSY`; any other error or changed claim stops without another redemption. | | Claude reset grants | `src/server/management/anthropic-reset-grant-routes.ts` — `GET /api/anthropic/reset-grants`, `POST /api/anthropic/reset-grants/consume` (lazy-loaded). Wire and fail-closed parsing live in `src/providers/anthropic-reset-grants.ts` (the Claude Code 2.1.278 `cedar_ember` contract, sent with `CLAUDE_CLI_USER_AGENT` from `src/providers/claude-cli-identity.ts`); the journal is `src/providers/anthropic-reset-grant-ledger.ts`: a cross-process `BEGIN IMMEDIATE` lock around every synchronous read-modify-write, a 90 s lease, the operation id reused as the upstream `request_id`, same-id retry only inside the vendor's ten-minute window, no settlement inferred from a re-read, and a fail-closed `500 journal_write_failed` when an answer cannot be recorded. Spending requires the `gui-session` principal. A journaled `reset`, including a same-id replay of that code, lifts that account cooldown only when it is a 429 quota refusal recorded before the operation was first journaled (the claim fence, `claimedAt`) and the journal's cleared windows cover every window the refusal named (both when it named none); it then drops the cached quota row and rereads usage with its own request rather than joining an older read. Model-family exclusions stay until the usage reread proves a family is no longer rejected. A failed reread leaves them in place. The HTTP result stays the journal answer if the reread fails. Other result codes do not admit. The dashboard owner is `gui/src/hooks/useAnthropicResetGrants.ts` with `gui/src/components/provider-workspace/AnthropicResetGrants.tsx` on the Anthropic OAuth rows of `ProviderAuthPanel`; after an unknown outcome the dialog only retries the same id. Design and audit record: [`../devlog/_fin/260923_claude_reset_grants/010_plan.md`](../devlog/_fin/260923_claude_reset_grants/010_plan.md). | -| Combos | `src/server/management/combo-routes.ts` — `GET/PUT/DELETE /api/combos` own provider combination and failover definitions. `PUT` keeps a stored field the body omits (`cooldownMs`, `waitForCooldownMs`, `defaultEffortMode`, `reasoningEffortMode`, `imageInput`, `cooldownWaitPolicy`, per-target `lastResort`, JEV `decisionQuotaSignals`/`decisionQuotaTiers` on updates/rename); quota false/null clears sparsely and switching away removes quota policy under [the owning contract](providers/jev-decision.md#loaded-advisory-quota). Explicit values replace it and defaults are stored sparse. | +| Combos | `src/server/management/combo-routes.ts` — `GET/PUT/DELETE /api/combos` own provider combination and failover definitions. `PUT` keeps a stored field the body omits (`cooldownMs`, `waitForCooldownMs`, `defaultEffortMode`, `reasoningEffortMode`, `imageInput`, `cooldownWaitPolicy`, per-target `lastResort`, JEV `decisionQuotaSignals`/`decisionQuotaTiers`/`sessionPin` on updates/rename); quota false/null clears sparsely and switching away removes quota policy under [the owning contract](providers/jev-decision.md#loaded-advisory-quota). Explicit values replace it and defaults are stored sparse. | | Protocol paths | `src/server/management/protocol-routes.ts` (lazy-loaded) — `GET /api/protocols` returns the contract version, the resolved API surfaces and protocol settings, the policy revision and the feature vocabulary; with `?provider=` (one non-empty name of at most 200 characters without control characters, else 400 `invalid_provider`; an unconfigured name is 404 `unknown_provider`, neither echoing the name) it adds a `provider` block from `src/protocols/provider-summary.ts` ([Protocol Paths](data-planes/protocol-paths.md#provider-wire-summary)); `POST /api/protocols/plan` takes `{ model, inbound, features? }` (model at most 200 characters, at most 24 features, any other key refused with 400) and returns a `ProtocolPlanV1` with `basis: "preview"` from `src/protocols/plan-snapshot.ts`. Both are read-only, never log their input, and send nothing upstream. `PATCH /api/protocols/settings` takes `{ messagesEnabled?, unrepresentable?, rollout? }` (strict: unknown keys and wrong types are 400), validates and applies through `src/server/management/protocol-settings-patch.ts`, persists with `saveConfigPreservingClaudeCode`, restores the live config if the save fails (409 on lock contention, 500 otherwise), and answers with the fresh `GET /api/protocols` body; closing Messages also writes `claudeCode.enabled = false` through `commitClaudeCodeBlock` ([Protocol Paths](data-planes/protocol-paths.md#settings)). The CLI drives all three through `ocx api protocols`, `ocx api explain` and `ocx api policy` ([Protocol Paths](data-planes/protocol-paths.md#cli)); none carries a route-registry exemption. | | Workflow budget | `src/server/management/workflow-budget-routes.ts` — `GET /api/workflow-budget` reads the tracked roots or one root, and `POST /api/workflow-budget/clear` clears exactly one. The clear moves the windowed send ring and the child map and nothing else: `active` belongs to turns still in flight, the spend ledger is a token budget an operator did not ask to forgive, and the lifetime send total survives so a clear cannot launder the record. A refusal event carries `spendScope` and `spendLimit` when a token ceiling fired, so the reason is readable without the config open beside it; no scope id is ever attached, because root ids are client thread headers and identity ids are credentials. Both are `deferred-verb` in the route registry — they are owed CLI verbs, and because the ledger is process memory there is no local projection the CLI could read instead. See [`../devlog/_fin/260915_workflow_budget_window/030_wfc_diff_plan.md`](../devlog/_fin/260915_workflow_budget_window/030_wfc_diff_plan.md). | | Codex accounts | `src/codex/auth-api/routes.ts` — `GET/POST/DELETE /api/codex-auth/accounts`, `PUT /api/codex-auth/accounts/alias`, `PUT /api/codex-auth/accounts/pause`, `PUT /api/codex-auth/accounts/pause-exhausted`, `POST /api/codex-auth/accounts/clear-cooldown`, `GET/PUT /api/codex-auth/active`, `PUT /api/codex-auth/auto-switch`, `PUT /api/codex-auth/pool-strategy`, `PUT /api/codex-auth/failover`, `GET /api/codex-auth/quota`, `GET /api/codex-auth/reset-credits` with `POST /api/codex-auth/reset-credits/consume`, and the login flow `POST /api/codex-auth/login`, `POST /api/codex-auth/login/code`, `POST /api/codex-auth/login/cancel`, `GET /api/codex-auth/login-status`. Per-account quota activation uses the existing `GET/PUT /api/settings` surface and `src/codex/quota-auto-refresh.ts`, keeping scheduled spending separate from credential/authentication mutation. Account ids are opaque handles and are serialized so the GUI can address an account; emails are masked and tokens are never serialized. New-account config commits add UI-managed selector bindings in the same config save; deletion deliberately retains existing bindings for fail-closed exact routing and re-add stability. Account mutations request catalog convergence only after config durability and expose only the boolean `catalogRefreshPending` completion projection. `showCodexCredits` is a default-off boolean in settings GET and successful PUT; partial PUT accepts a boolean and rolls back on persistence failure. See [credits exposure](providers/openai-accounts.md#display-only-codex-credits). | diff --git a/structure/providers/jev-decision.md b/structure/providers/jev-decision.md index 82fb6016a68..86c8690f399 100644 --- a/structure/providers/jev-decision.md +++ b/structure/providers/jev-decision.md @@ -196,6 +196,27 @@ checkboxes and `Create JEV Auto` template. Inside that editor, a JEV Combo's `De section (`gui/src/components/combo-workspace-jev-decision.tsx`) chooses TypeSafe, a System One row, or an opencodex model route, with the timeout and a Test probe; there is no separate decisions page. +## Session pin + +`sessionPin: { ttlMs }` is an opt-in, JEV-only Combo field (#7525); `src/combos/jev-session-pin-config.ts` +owns its 60000..86400000 ms range, the CLI duration grammar and the closed `pinReason` vocabulary. +Validation, sparse storage, omitted-field preservation on JEV saves and removal on a strategy switch follow +the quota fields above. Absent, `src/server/responses/core-combo.ts` asks JEV on every request exactly as before. +`src/server/responses/jev-session-pin.ts` keys a process-local pin by the admitted caller (context principal, +key id, or `loopback`), the Combo id and the explicit session lane from `sessionLaneIdFromRequest`; a request +without a lane never pins and no body field is an identity. A pin is written only when the target of an applied +(`gate: "apply"`) decision completes through the Combo callback gate, and every completion on that target +(tool-loop continuations included) renews its idle window. Reuse runs after current eligibility: the pinned +key must be among this request's JEV choices with its effort still offered, and opted-in quota evidence must +not read `nearly_exhausted`; otherwise the pin is released and JEV decides. Any recorded target failure in the +request forgets the pin, cancellation does not, and a config generation change invalidates both stored pins and +in-flight writers. A reused turn sends no decision request and persists `pinned: true` with `pinReason: "session"`; +a decision that replaced a released pin persists the reason. `src/usage/jev-stats.ts` counts pinned turns as +`pinnedRequests`, never as decisions, gates or backend latency, while their physical attempts still count per model. +The store holds at most 512 pins with least-recently-written eviction. Coverage: +`tests/routing/jev-session-pin.test.ts`, `tests/server/server-jev-session-pin.test.ts` and +`tests/cli/cli-combo-session-pin.test.ts`. + JEV setup stays inside those existing shells. A configured `jev-decision` provider Overview exposes **Create JEV Auto**, which navigates to the registered `models/combos/jev-auto` action hash. `gui/src/pages/Combos.tsx` owns that one-shot add intent and normalizes the hash when the modal diff --git a/tests/cli/cli-combo-session-pin.test.ts b/tests/cli/cli-combo-session-pin.test.ts new file mode 100644 index 00000000000..eca05257af6 --- /dev/null +++ b/tests/cli/cli-combo-session-pin.test.ts @@ -0,0 +1,81 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { getConfigPath, loadConfig, saveConfig } from "../../src/config"; +import { handleComboCommand } from "../../src/cli/combo"; +import type { RuntimeApiDeps } from "../../src/cli/runtime-api"; +import { handleComboRoutes } from "../../src/server/management/combo-routes"; +import type { OcxConfig } from "../../src/types"; +import { createTempHome, type TempHome } from "../helpers/temp-home"; + +const targets = [{ provider: "fixture", model: "m" }]; +let home: TempHome; +let config: OcxConfig; +let log: ReturnType; +let error: ReturnType; + +beforeEach(() => { + home = createTempHome("ocx-jev-session-pin-cli-"); + log = spyOn(console, "log").mockImplementation(() => {}); + error = spyOn(console, "error").mockImplementation(() => {}); + config = { + port: 10100, + defaultProvider: "fixture", + providers: { fixture: { adapter: "openai-chat", baseUrl: "https://fixture.invalid/v1", models: ["m"] } }, + combos: { saved: { strategy: "jev", targets } }, + }; + saveConfig(config); +}); +afterEach(() => { log.mockRestore(); error.mockRestore(); home.remove(); }); + +async function api(method: string, body?: unknown): Promise { + const req = new Request("http://127.0.0.1/api/combos", { method, ...(body ? { headers: { "content-type": "application/json" }, body: JSON.stringify(body) } : {}) }); + const response = await handleComboRoutes({ req, url: new URL(req.url), config, deps: {}, version: "fixture", trustedLoopbackIngress: true, guiSessionIssuance: null, + convergeCodexCatalog: async () => ({ status: "committed", changed: false, degraded: false, notices: [] }), syncClaudeAgentDefsBestEffort: async () => {} }); + if (!response) throw new Error("missing combo response"); + return response; +} +const deps: RuntimeApiDeps = { + findLiveProxy: async () => ({ pid: null, port: 14000, source: "runtime" }), + fetchImpl: (async (_input, init) => api(init?.method ?? "GET", init?.body ? JSON.parse(String(init.body)) : undefined)) as typeof fetch, +}; +const put = (combo: unknown, id = "saved", renameFrom?: string) => api("PUT", { id, combo, ...(renameFrom ? { renameFrom } : {}) }); +const stored = () => JSON.parse(readFileSync(getConfigPath(), "utf8")).combos as Record>; + +describe("JEV session pin management (#7525)", () => { + test("the API stores the pin sparsely, keeps it across omitted saves and renames, and drops it on clear or switch", async () => { + expect((await put({ strategy: "jev", targets, sessionPin: { ttlMs: 1_200_000 } })).status).toBe(200); + expect(stored().saved).toMatchObject({ sessionPin: { ttlMs: 1_200_000 } }); + expect((await (await api("GET")).json()).combos[0]).toMatchObject({ sessionPin: { ttlMs: 1_200_000 } }); + // The dashboard does not send the field; its save must not turn the pin off. + expect((await put({ strategy: "jev", targets }, "renamed", "saved")).status).toBe(200); + expect(loadConfig().combos?.renamed?.sessionPin).toEqual({ ttlMs: 1_200_000 }); + expect((await put({ strategy: "jev", targets, sessionPin: null }, "renamed")).status).toBe(200); + expect(stored().renamed).not.toHaveProperty("sessionPin"); + expect((await put({ strategy: "jev", targets, sessionPin: { ttlMs: 60_000 } }, "renamed")).status).toBe(200); + expect((await put({ strategy: "failover", targets }, "renamed")).status).toBe(200); + expect(stored().renamed).not.toHaveProperty("sessionPin"); + }); + + test("the API refuses an out-of-range or non-JEV pin without writing", async () => { + const before = readFileSync(getConfigPath(), "utf8"); + expect((await put({ strategy: "jev", targets, sessionPin: { ttlMs: 1_000 } })).status).toBe(400); + expect((await put({ strategy: "failover", targets, sessionPin: { ttlMs: 60_000 } })).status).toBe(400); + expect(readFileSync(getConfigPath(), "utf8")).toBe(before); + }); + + test("the CLI sets, keeps, clears and rejects the pin", async () => { + expect(await handleComboCommand(["set", "saved", "--session-pin", "20m", "--json"], deps)).toBe(0); + expect(config.combos?.saved?.sessionPin).toEqual({ ttlMs: 1_200_000 }); + expect(await handleComboCommand(["set", "saved", "--decision-timeout", "5000", "--json"], deps)).toBe(0); + expect(config.combos?.saved?.sessionPin).toEqual({ ttlMs: 1_200_000 }); + expect(await handleComboCommand(["set", "saved", "--session-pin", "-", "--json"], deps)).toBe(0); + expect(config.combos?.saved).not.toHaveProperty("sessionPin"); + for (const value of ["30s", "2d", "soon"]) { + expect(await handleComboCommand(["set", "saved", "--session-pin", value, "--json"], deps)).not.toBe(0); + } + expect(await handleComboCommand(["set", "saved", "--strategy", "failover", "--session-pin", "20m", "--json"], deps)).not.toBe(0); + expect(await handleComboCommand(["set", "saved", "--session-pin", "1h", "--json"], deps)).toBe(0); + expect(await handleComboCommand(["set", "saved", "--strategy", "failover", "--json"], deps)).toBe(0); + expect(config.combos?.saved).not.toHaveProperty("sessionPin"); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 0aad0bd1d88..58e7b255283 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -783,7 +783,7 @@ "key-login-preserves-model-costs.test.ts": "oauth", "jev-decision.test.ts": "routing", "jev-service-exchange.test.ts": "routing", "jev-decision-destination.test.ts": "routing", "jev-decision-provider-combo.test.ts": "routing", - "jev-typesafe-golden.test.ts": "routing", "jev-quota-activation.test.ts": "routing", "jev-quota-core-boundary.test.ts": "routing", "jev-quota-signals.test.ts": "routing", "jev-quota-config.test.ts": "routing", "jev-quota-publication.test.ts": "routing", "jev-quota-codex-reset.test.ts": "routing", "jev-quota-anthropic2.test.ts": "routing", "jev-quota-antigravity.test.ts": "routing", "jev-model-backend.test.ts": "routing", + "jev-typesafe-golden.test.ts": "routing", "jev-quota-activation.test.ts": "routing", "jev-quota-core-boundary.test.ts": "routing", "jev-quota-signals.test.ts": "routing", "jev-quota-config.test.ts": "routing", "jev-quota-publication.test.ts": "routing", "jev-quota-codex-reset.test.ts": "routing", "jev-quota-anthropic2.test.ts": "routing", "jev-quota-antigravity.test.ts": "routing", "jev-model-backend.test.ts": "routing", "jev-session-pin.test.ts": "routing", "server-jev-session-pin.test.ts": "server", "cli-combo-session-pin.test.ts": "cli", "jev-provider.test.ts": "providers", "keyring-smoke.test.ts": "ci-workflows", "kilo-client.test.ts": "clients", "kimi-oauth-identity.test.ts": "providers", "kimi-responses-adjacency.test.ts": "providers", "kiro-account-load.test.ts": "providers/kiro", diff --git a/tests/helpers/responses-core-source.ts b/tests/helpers/responses-core-source.ts index bc2530b02e2..2bc9173afe3 100644 --- a/tests/helpers/responses-core-source.ts +++ b/tests/helpers/responses-core-source.ts @@ -35,6 +35,8 @@ export const RESPONSES_CORE_MODULES = [ "core-combo-native.ts", // Reached from core-combo.ts: tool-loop continuations stay on the lane's previous target (#7298). "combo-tool-loop-pin.ts", + // Reached from core-combo.ts: opt-in JEV session pin keeps a session on its decided target (#7525). + "jev-session-pin.ts", // Reached from core-combo.ts: the JEV model backend's internal decision turn. "jev-model-invoke.ts", // Reached from core-combo.ts: JEV effort choices and resolved native quota identity. diff --git a/tests/routing/jev-session-pin.test.ts b/tests/routing/jev-session-pin.test.ts new file mode 100644 index 00000000000..ef41b32a791 --- /dev/null +++ b/tests/routing/jev-session-pin.test.ts @@ -0,0 +1,206 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import type { JevCandidate, JevDecision } from "../../src/combos/jev"; +import { + JEV_SESSION_PIN_TTL_MAX_MS, + JEV_SESSION_PIN_TTL_MIN_MS, + normalizeJevSessionPin, + parseJevSessionPinDuration, +} from "../../src/combos/jev-session-pin-config"; +import { comboConfigIssues, normalizeComboConfig } from "../../src/combos/types"; +import { captureConfigGeneration, reconcileStateGeneration } from "../../src/lib/state-store-sweeper"; +import type { DataPlaneAdmission } from "../../src/server/auth-cors"; +import { + clearJevSessionPinsForTests, + JEV_SESSION_PIN_CAPACITY, + jevSessionPinCountForTests, + jevSessionPinScope, +} from "../../src/server/responses/jev-session-pin"; +import { createJevStatsAccumulator, normalizePersistedJevDecision } from "../../src/usage/jev-stats"; +import type { PersistedUsageEntry } from "../../src/usage/log"; +import type { OcxProviderConfig } from "../../src/types"; + +const TTL = 120_000; +const combo = { strategy: "jev" as const, sessionPin: { ttlMs: TTL } }; +const LOOPBACK: DataPlaneAdmission = { kind: "loopback", source: "loopback" }; +const sol: JevCandidate = { key: "sol/gpt-5.6-sol", provider: "sol", model: "gpt-5.6-sol", reasoningEfforts: ["low", "high"] }; +const luna: JevCandidate = { key: "luna/gpt-5.6-luna", provider: "luna", model: "gpt-5.6-luna", reasoningEfforts: ["low"] }; +const applied: JevDecision = { backend: "typesafe", targetKey: sol.key, effort: "high", gate: "apply", latencyMs: 12 }; + +let clock = 1_000_000; +const now = () => clock; + +function scope(overrides: Partial[0]> = {}) { + return jevSessionPinScope({ combo, comboId: "auto", admission: LOOPBACK, lane: "lane-a", now, ...overrides })!; +} + +function bumpConfigGeneration(): void { + const empty = new Set(); + reconcileStateGeneration({ + generation: 0, + providerNames: empty, + comboIds: empty, + comboTargets: empty, + codexAccountIds: empty, + oauthAccountKeys: empty, + configRoots: empty, + }); +} + +beforeEach(() => { + clock = 1_000_000; + clearJevSessionPinsForTests(); +}); +afterEach(() => clearJevSessionPinsForTests()); + +describe("JEV session pin settings", () => { + test("accept exactly { ttlMs } inside the documented range", () => { + expect(normalizeJevSessionPin({ ttlMs: 1_200_000 })).toEqual({ ttlMs: 1_200_000 }); + for (const value of [null, [], {}, { ttlMs: JEV_SESSION_PIN_TTL_MIN_MS - 1 }, { ttlMs: JEV_SESSION_PIN_TTL_MAX_MS + 1 }, + { ttlMs: 90_000.5 }, { ttlMs: "20m" }, { ttlMs: 90_000, turns: 3 }]) { + expect(normalizeJevSessionPin(value)).toBeUndefined(); + } + }); + + test("CLI durations use ms, s, m or h and stay in range", () => { + expect(parseJevSessionPinDuration("20m")).toBe(1_200_000); + expect(parseJevSessionPinDuration("90s")).toBe(90_000); + expect(parseJevSessionPinDuration("1h")).toBe(3_600_000); + expect(parseJevSessionPinDuration("60000")).toBe(60_000); + for (const raw of ["59s", "25h", "20 m", "-1m", "1.5h", "m", ""]) expect(parseJevSessionPinDuration(raw)).toBeUndefined(); + }); + + test("only a JEV Combo may carry the pin, null clears it and the default stays off", () => { + const providers: Record = { fixture: { adapter: "openai-chat", baseUrl: "https://fixture.invalid/v1", models: ["m"] } }; + const targets = [{ provider: "fixture", model: "m" }]; + expect(comboConfigIssues("auto", { strategy: "jev", targets, sessionPin: { ttlMs: TTL } }, providers)).toEqual([]); + expect(comboConfigIssues("auto", { strategy: "jev", targets, sessionPin: null }, providers)).toEqual([]); + expect(comboConfigIssues("auto", { strategy: "failover", targets, sessionPin: { ttlMs: TTL } }, providers)[0]?.path).toEqual(["sessionPin"]); + expect(comboConfigIssues("auto", { strategy: "jev", targets, sessionPin: { ttlMs: 10 } }, providers)[0]?.path).toEqual(["sessionPin"]); + expect(normalizeComboConfig({ strategy: "jev", targets })).not.toHaveProperty("sessionPin"); + expect(normalizeComboConfig({ strategy: "jev", targets, sessionPin: { ttlMs: TTL } }).sessionPin).toEqual({ ttlMs: TTL }); + }); +}); + +describe("JEV session pin scope", () => { + test("never exists without the opt-in, a JEV strategy or an explicit session lane", () => { + expect(jevSessionPinScope({ combo: { strategy: "jev" }, comboId: "auto", admission: LOOPBACK, lane: "lane-a" })).toBeUndefined(); + expect(jevSessionPinScope({ combo: { ...combo, strategy: "failover" }, comboId: "auto", admission: LOOPBACK, lane: "lane-a" })).toBeUndefined(); + expect(jevSessionPinScope({ combo, comboId: "auto", admission: LOOPBACK, lane: undefined })).toBeUndefined(); + }); + + test("an applied decision pins after it completes, and the pin replays its target and effort", () => { + const pin = scope(); + expect(pin.reuse([sol, luna])).toEqual({}); + pin.complete(sol.key, applied, captureConfigGeneration()); + expect(pin.reuse([luna, sol])).toEqual({ + decision: { backend: "typesafe", targetKey: sol.key, effort: "high", gate: "apply", latencyMs: 0, pinned: true, pinReason: "session" }, + }); + }); + + test("fail-open decisions and completions on another target never pin", () => { + const pin = scope(); + pin.complete(sol.key, { ...applied, gate: "timeout" }, captureConfigGeneration()); + pin.complete(luna.key, applied, captureConfigGeneration()); + pin.complete(sol.key, undefined, captureConfigGeneration()); + expect(pin.reuse([sol, luna])).toEqual({}); + expect(jevSessionPinCountForTests()).toBe(0); + }); + + test("callers, Combos and sessions never share a pin", () => { + scope().complete(sol.key, applied, captureConfigGeneration()); + const alice: DataPlaneAdmission = { kind: "configured", keyId: "alice", source: "bearer", contextPrincipalId: "p-alice" }; + const bob: DataPlaneAdmission = { kind: "configured", keyId: "bob", source: "bearer", contextPrincipalId: "p-bob" }; + expect(scope({ admission: alice }).reuse([sol])).toEqual({}); + scope({ admission: alice }).complete(sol.key, applied, captureConfigGeneration()); + expect(scope({ admission: bob }).reuse([sol])).toEqual({}); + expect(scope({ comboId: "other" }).reuse([sol])).toEqual({}); + expect(scope({ lane: "lane-b" }).reuse([sol])).toEqual({}); + expect(scope({ admission: alice }).reuse([sol]).decision?.pinned).toBe(true); + expect(scope().reuse([sol]).decision?.pinned).toBe(true); + }); + + test("the idle window expires the pin and each completion on the pinned target renews it", () => { + const pin = scope(); + pin.complete(sol.key, applied, captureConfigGeneration()); + clock += TTL - 1; + // A tool-loop continuation or any other completion on the same target keeps the session warm. + pin.complete(sol.key, undefined, captureConfigGeneration()); + clock += TTL - 1; + expect(pin.reuse([sol]).decision?.pinned).toBe(true); + clock += TTL; + expect(pin.reuse([sol])).toEqual({ released: "expired" }); + expect(pin.reuse([sol])).toEqual({}); + }); + + test("a target that left the current choices, or lost the pinned effort, releases the pin", () => { + const pin = scope(); + pin.complete(sol.key, applied, captureConfigGeneration()); + expect(pin.reuse([luna])).toEqual({ released: "target_unavailable" }); + pin.complete(sol.key, applied, captureConfigGeneration()); + expect(pin.reuse([{ ...sol, reasoningEfforts: ["low"] }])).toEqual({ released: "effort_unavailable" }); + expect(jevSessionPinCountForTests()).toBe(0); + }); + + test("a config change releases the pin and discards completions that started before it", () => { + const pin = scope(); + const before = captureConfigGeneration(); + pin.complete(sol.key, applied, before); + bumpConfigGeneration(); + expect(pin.reuse([sol])).toEqual({ released: "config_changed" }); + pin.complete(sol.key, applied, before); + expect(pin.reuse([sol])).toEqual({}); + }); + + test("a failure forgets the pin", () => { + const pin = scope(); + pin.complete(sol.key, applied, captureConfigGeneration()); + pin.forget(); + expect(pin.reuse([sol])).toEqual({}); + }); + + test("the store is bounded and evicts the least recently written session", () => { + for (let lane = 0; lane <= JEV_SESSION_PIN_CAPACITY; lane += 1) { + scope({ lane: `lane-${lane}` }).complete(sol.key, applied, captureConfigGeneration()); + } + expect(jevSessionPinCountForTests()).toBe(JEV_SESSION_PIN_CAPACITY); + expect(scope({ lane: "lane-0" }).reuse([sol])).toEqual({}); + expect(scope({ lane: `lane-${JEV_SESSION_PIN_CAPACITY}` }).reuse([sol]).decision?.pinned).toBe(true); + }); +}); + +describe("persisted session pin evidence", () => { + const base = { + version: 1, + comboId: "auto", + selected: { provider: "sol", model: "gpt-5.6-sol", effort: "high" }, + gate: "apply", + latencyMs: 0, + backend: "typesafe", + }; + + test("pinned turns and release reasons survive normalization; inconsistent pairs do not", () => { + expect(normalizePersistedJevDecision({ ...base, pinned: true, pinReason: "session" })).toMatchObject({ pinned: true, pinReason: "session" }); + expect(normalizePersistedJevDecision({ ...base, pinReason: "expired" })).toMatchObject({ pinReason: "expired" }); + expect(normalizePersistedJevDecision({ ...base, pinReason: "expired" })).not.toHaveProperty("pinned"); + expect(normalizePersistedJevDecision({ ...base, pinReason: "bogus" })).not.toHaveProperty("pinReason"); + expect(normalizePersistedJevDecision({ ...base, pinned: true })).toBeUndefined(); + expect(normalizePersistedJevDecision({ ...base, pinReason: "session" })).toBeUndefined(); + expect(normalizePersistedJevDecision({ ...base, pinned: true, pinReason: "expired" })).toBeUndefined(); + }); + + test("stats count pinned turns apart from decisions while keeping their physical tokens", () => { + const entry = (jevDecision: Record, timestamp: number) => ({ + timestamp, + status: 200, + jevDecision, + attempts: [{ provider: "sol", model: "gpt-5.6-sol", sendCount: 1, usageStatus: "reported", usage: { inputTokens: 100, outputTokens: 5, cacheReadInputTokens: 80 } }], + }) as unknown as PersistedUsageEntry; + const accumulator = createJevStatsAccumulator({ comboId: "auto" }); + accumulator.add(entry({ ...base, latencyMs: 40 }, 1)); + accumulator.add(entry({ ...base, pinned: true, pinReason: "session" }, 2)); + const stats = accumulator.clone().summarize("all", 3); + expect(stats.summary).toMatchObject({ decisions: 1, appliedDecisions: 1, pinnedRequests: 1, modelAttempts: 2, modelCacheReadTokens: 160, averageLatencyMs: 40 }); + expect(stats.gates).toEqual([{ gate: "apply", decisions: 1 }]); + expect(stats.models).toEqual([expect.objectContaining({ provider: "sol", picks: 1, attempts: 2, inputTokens: 200 })]); + }); +}); diff --git a/tests/server/server-jev-session-pin.test.ts b/tests/server/server-jev-session-pin.test.ts new file mode 100644 index 00000000000..fa2879307a8 --- /dev/null +++ b/tests/server/server-jev-session-pin.test.ts @@ -0,0 +1,252 @@ +/** + * #7525: an opt-in JEV session pin keeps one client session on the target JEV chose for it, so the + * provider prompt cache stays warm, while default-off Combos keep deciding on every request. + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { clearComboSelectionState, clearComboTargetCooldowns, coolComboTarget } from "../../src/combos"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; +import { resolveProviderApiKey } from "../../src/providers/api-key-resolve"; +import { publishDecisionKeyQuota } from "../../src/providers/quota-decision-snapshot"; +import type { DataPlaneAdmission } from "../../src/server/auth-cors"; +import type { RequestLogContext } from "../../src/server/request-log"; +import { executeComboResponses } from "../../src/server/responses/core-combo"; +import type { ResponsesDispatchers } from "../../src/server/responses/core-options"; +import { clearComboRecallForTests } from "../../src/server/responses/combo-session-recall"; +import { clearJevSessionPinsForTests, JEV_SESSION_PIN_ROUTE_REASON } from "../../src/server/responses/jev-session-pin"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { activateJevQuotaEvidence, deactivateJevQuotaEvidence } from "../helpers/jev-quota-activation"; +import { createTempHome } from "../helpers/temp-home"; + +const JEV_URL = "https://api.typesafe.ai/v1/systemone"; +const SOL = { provider: "sol", model: "gpt-5.6-sol" } as const; +const LUNA = { provider: "luna", model: "gpt-5.6-luna" } as const; + +function resetState(): void { + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearComboRecallForTests(); + clearJevSessionPinsForTests(); +} +beforeEach(resetState); +afterEach(resetState); + +function modelProvider(model: string): OcxProviderConfig { + return { + adapter: "openai-chat", + baseUrl: `https://${model}.example.test/v1`, + authMode: "key", + apiKey: `key-${model}`, + liveModels: false, + models: [model], + modelReasoningEfforts: { [model]: ["low", "medium", "high"] }, + }; +} + +/** A JEV Combo whose decision service records every call and always answers `choice`. */ +function makeConfig(choice: string, decisions: string[], sessionPin: boolean): OcxConfig { + const jevFetch = (async () => { + decisions.push(choice); + return Response.json({ answers: { route: { choice, confidence: 0.8 } } }); + }) as typeof fetch; + return { + port: 0, + defaultProvider: "sol", + providers: { + jev: { adapter: "jev-decision", baseUrl: JEV_URL, authMode: "key", liveModels: false, apiKey: "typesafe-test-key", fetch: jevFetch }, + sol: modelProvider(SOL.model), + luna: modelProvider(LUNA.model), + }, + combos: { + auto: { + alias: "jev-auto", + strategy: "jev", + targets: [{ ...LUNA }, { ...SOL }], + ...(sessionPin ? { sessionPin: { ttlMs: 20 * 60_000 } } : {}), + }, + }, + }; +} + +type Child = (body: Record, complete: () => void) => Response | Promise; + +/** Children complete like a real stream: the completion callback runs before the response returns. */ +function dispatchers(child: Child): ResponsesDispatchers { + return { + async handleResponses(request, _config, _logCtx, options) { + const body = await request.json() as Record; + return child(body, () => options?.onResponseComplete?.(String(body.model))); + }, + async handleComboResponses() { + throw new Error("nested combo dispatch is not expected"); + }, + }; +} + +const served: Child = (body, complete) => { + complete(); + return Response.json({ id: `resp-${String(body.model)}`, object: "response", status: "completed", model: body.model, output: [] }); +}; + +async function turn(config: OcxConfig, options: { + session?: string | null; + admission?: DataPlaneAdmission; + child?: Child; + signal?: AbortSignal; + log?: RequestLogContext; + models?: string[]; +} = {}): Promise { + const body = { model: "jev-auto", input: "Implement the next step.", stream: false }; + const headers: Record = { "content-type": "application/json" }; + if (options.session !== null) headers.session_id = options.session ?? "session-a"; + const request = new Request("http://127.0.0.1/v1/responses", { method: "POST", headers, body: JSON.stringify(body) }); + const budget = createTranslatorBudget(); + const child = options.child ?? served; + try { + return await executeComboResponses(request, body, "auto", config, options.log ?? { model: "", provider: "" }, { + translatorBudget: budget, + ...(options.admission ? { admission: options.admission } : {}), + ...(options.signal ? { abortSignal: options.signal } : {}), + }, dispatchers((childBody, complete) => { + options.models?.push(String(childBody.model)); + return child(childBody, complete); + })); + } finally { + budget.dispose(); + } +} + +describe("JEV session pin (#7525)", () => { + test("default off: every turn of one session asks JEV again and records no pin", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, false); + const log: RequestLogContext = { model: "", provider: "" }; + expect((await turn(config)).status).toBe(200); + expect((await turn(config, { log })).status).toBe(200); + expect(decisions).toHaveLength(2); + expect(log.jevDecision).not.toHaveProperty("pinned"); + expect(log.jevDecision).not.toHaveProperty("pinReason"); + }); + + test("a pinned session reuses JEV's target and effort without another decision call", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + const bodies: Record[] = []; + const child: Child = (body, complete) => { bodies.push(body); return served(body, complete); }; + expect((await turn(config, { child })).status).toBe(200); + const log: RequestLogContext = { model: "", provider: "" }; + expect((await turn(config, { child, log })).status).toBe(200); + expect(decisions).toHaveLength(1); + expect(bodies.map(body => [body.model, (body.reasoning as { effort?: string } | undefined)?.effort])) + .toEqual([["sol/gpt-5.6-sol", "high"], ["sol/gpt-5.6-sol", "high"]]); + expect(log.jevDecision).toEqual({ + version: 1, + comboId: "auto", + backend: "typesafe", + selected: { provider: "sol", model: SOL.model, effort: "high" }, + gate: "apply", + latencyMs: 0, + pinned: true, + pinReason: "session", + }); + expect(log.routeDecision?.selected.reason).toBe(JEV_SESSION_PIN_ROUTE_REASON); + }); + + test("requests without an explicit session, other sessions and other callers are decided afresh", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + const alice: DataPlaneAdmission = { kind: "configured", keyId: "alice", source: "bearer", contextPrincipalId: "p-alice" }; + const bob: DataPlaneAdmission = { kind: "configured", keyId: "bob", source: "bearer", contextPrincipalId: "p-bob" }; + await turn(config, { session: null }); + await turn(config, { session: null }); + expect(decisions).toHaveLength(2); + await turn(config, { admission: alice }); + await turn(config, { admission: bob }); + await turn(config, { admission: alice, session: "session-b" }); + expect(decisions).toHaveLength(5); + await turn(config, { admission: alice }); + expect(decisions).toHaveLength(5); + }); + + test("a failed pinned send hops over and the next turn is decided afresh", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + await turn(config); + const models: string[] = []; + const failing: Child = (body, complete) => body.model === "sol/gpt-5.6-sol" + ? Response.json({ error: { type: "server_error", message: "busy" } }, { status: 500 }) + : served(body, complete); + expect((await turn(config, { child: failing, models })).status).toBe(200); + expect(models).toEqual(["sol/gpt-5.6-sol", "luna/gpt-5.6-luna"]); + expect(decisions).toHaveLength(1); + const log: RequestLogContext = { model: "", provider: "" }; + await turn(config, { log }); + expect(decisions).toHaveLength(2); + expect(log.jevDecision).not.toHaveProperty("pinned"); + }); + + test("a cancelled send neither writes nor drops the pin", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + const cancelled = () => { + const controller = new AbortController(); + const child: Child = () => { + controller.abort(new Error("client went away")); + return Response.json({ error: { message: "aborted" } }, { status: 499 }); + }; + return { child, signal: controller.signal }; + }; + expect((await turn(config, cancelled())).status).toBe(499); + await turn(config); + expect(decisions).toHaveLength(2); + expect((await turn(config, cancelled())).status).toBe(499); + const log: RequestLogContext = { model: "", provider: "" }; + await turn(config, { log }); + expect(decisions).toHaveLength(2); + expect(log.jevDecision).toMatchObject({ pinned: true }); + }); + + test("a fail-open decision never pins the session", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + config.providers.jev!.fetch = (async () => { + decisions.push("http"); + return new Response("unavailable", { status: 503 }); + }) as typeof fetch; + await turn(config); + await turn(config); + expect(decisions).toEqual(["http", "http"]); + }); + + test("a cooling pinned target releases the pin and records why", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + await turn(config); + coolComboTarget("auto", SOL, { cooldownMs: 60_000 }); + const log: RequestLogContext = { model: "", provider: "" }; + await turn(config, { log }); + expect(decisions).toHaveLength(2); + expect(log.jevDecision).toMatchObject({ pinReason: "target_unavailable" }); + expect(log.jevDecision).not.toHaveProperty("pinned"); + }); + + test("opted-in quota evidence reading nearly exhausted releases the pin", async () => { + const home = createTempHome("ocx-jev-session-pin-quota-"); + try { + activateJevQuotaEvidence(); + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + config.combos!.auto!.decisionQuotaSignals = true; + await turn(config); + const provider = config.providers.sol!; + resolveProviderApiKey(provider.apiKey); + publishDecisionKeyQuota("sol", provider, [{ window: "weekly", percent: 95, observedAt: Date.now() }]); + const log: RequestLogContext = { model: "", provider: "" }; + await turn(config, { log }); + expect(decisions).toHaveLength(2); + expect(log.jevDecision).toMatchObject({ pinReason: "quota_nearly_exhausted" }); + } finally { + deactivateJevQuotaEvidence(); + home.remove(); + } + }); +}); From d7319065a34c468c1812e42b4bb34ef5d7aec33e Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 23 Oct 2026 08:28:43 +0900 Subject: [PATCH 2/4] fix(jev): end the session pin on every failure path and keep pinned turns in request totals --- docs-site/src/content/docs/guides/combos.md | 8 +++--- src/cli/combo-stats.ts | 4 ++- src/server/responses/core-combo.ts | 5 ++++ src/usage/jev-stats.ts | 23 ++++++++--------- structure/providers/jev-decision.md | 5 +++- tests/routing/jev-session-pin.test.ts | 17 +++++++++---- tests/server/server-jev-session-pin.test.ts | 28 ++++++++++++++++++--- 7 files changed, 64 insertions(+), 26 deletions(-) diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 49f1647e651..4b79e556099 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -314,7 +314,8 @@ restart. Usage records mark a reused turn with `jevDecision.pinned: true` and `pinReason: "session"`. A decision that replaced a released pin records why in `pinReason` (`expired`, `config_changed`, `target_unavailable`, `effort_unavailable` or `quota_nearly_exhausted`). `ocx combo stats` -counts pinned requests apart from decisions, and their model tokens still count per model. +counts pinned requests apart from decisions; they still count toward successful requests, model +fallbacks and per-model tokens. A pin makes provider cache reuse possible; whether the provider actually serves cached tokens still depends on the provider, the account and a stable prompt prefix. `--session-pin -` (API `null`) @@ -594,8 +595,9 @@ are ignored. Operator notes are evidence for the decision, not commands, and can target allowlist or reasoning-effort limits. Only put information there that may be disclosed to the selected decision backend. -Each logical model call is decided on its own; there is no per-conversation pin. Consecutive turns of -one session can therefore land on different targets, and every switch starts a cold provider prompt +By default, each logical model call is decided on its own. Unless you enable the opt-in +[session pin](#session-pin-for-jev), consecutive turns of one session can therefore land on +different targets, and every switch starts a cold provider prompt cache, so a mix of very different targets can cost more input tokens than it saves. Keep the allowlist to targets you are content to alternate between. Targets marked `lastResort` are withheld from JEV under `cooldownWaitPolicy: "before-last-resort"` while any normal target is offered, and diff --git a/src/cli/combo-stats.ts b/src/cli/combo-stats.ts index 2ee6d9a55a7..04dedc65c08 100644 --- a/src/cli/combo-stats.ts +++ b/src/cli/combo-stats.ts @@ -4,7 +4,7 @@ import { CliUsageError, printData, runtimeBaseUrl, runtimeRequest, takeFlag, tak const USAGE = "Usage: ocx combo stats [--range <7d|30d|all>] [--json]"; const SUMMARY_COUNTS = [ - "decisions", "appliedDecisions", "failOpenDecisions", "pinnedRequests", "successfulRequests", "requestsWithModelFallback", + "decisions", "appliedDecisions", "failOpenDecisions", "successfulRequests", "requestsWithModelFallback", "modelAttempts", "measuredModelAttempts", "modelInputTokens", "modelOutputTokens", "modelReasoningTokens", "modelCacheReadTokens", "modelCacheWriteTokens", "modelTotalTokens", "decisionUsageReported", "decisionInputTokens", "decisionOutputTokens", "decisionTotalTokens", @@ -53,6 +53,8 @@ function projectStats(value: unknown, id: string, range: string) { const rawSummary = record(dto.summary); const summary = { ...counts(rawSummary, SUMMARY_COUNTS), + // Absent means zero: a proxy started before session pins (#7525) does not report the field. + pinnedRequests: Object.hasOwn(rawSummary, "pinnedRequests") ? number(rawSummary.pinnedRequests) : 0, averageLatencyMs: nullable(rawSummary.averageLatencyMs), averageConfidence: nullable(rawSummary.averageConfidence, 1), averageChosenProbability: nullable(rawSummary.averageChosenProbability, 1), diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index a5528d05c6b..bc4452a279e 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -1018,6 +1018,8 @@ export async function executeComboResponses( if (childLog.terminalIncompleteReason !== undefined) logCtx.terminalIncompleteReason = childLog.terminalIncompleteReason; if (childLog.terminalErrorCode !== undefined) logCtx.terminalErrorCode = childLog.terminalErrorCode; if (childLog.upstreamError !== undefined) logCtx.upstreamError = childLog.upstreamError; + // A committed stream that ends failed is a target failure too; incomplete is an answer. + if (status === "failed") jevPin?.forget(); options.onNativePassthroughTerminal?.(status); }, }); @@ -1091,6 +1093,7 @@ export async function executeComboResponses( retainCancelledAttempt(); return clientCancelledResponse(); } + jevPin?.forget(); finishRequestAttempt(attempt, 502, Date.now() - started, childLog.usage); throw error; } @@ -1115,6 +1118,7 @@ export async function executeComboResponses( retainCancelledAttempt(); return clientCancelledResponse(); } + jevPin?.forget(); finishRequestAttempt(attempt, 502, Date.now() - started, childLog.usage); throw error; } @@ -1171,6 +1175,7 @@ export async function executeComboResponses( retainCancelledAttempt(); return clientCancelledResponse(); } + jevPin?.forget(); finishRequestAttempt(attempt, 502, Date.now() - started, childLog.usage); throw error; } diff --git a/src/usage/jev-stats.ts b/src/usage/jev-stats.ts index fb06ce7ac8d..c41c84f9c90 100644 --- a/src/usage/jev-stats.ts +++ b/src/usage/jev-stats.ts @@ -393,10 +393,19 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { || (this.until !== undefined && entry.timestamp > this.until)) return; const decision = normalizePersistedJevDecision(entry.jevDecision); if (!decision || (this.comboId !== null && decision.comboId !== this.comboId)) return; + const attempts = sentAttempts(entry); + // Request-level outcomes count every JEV-routed request, pinned or decided. + if (entry.status >= 200 && entry.status < 400) { + this.successfulRequests = saturatingAdd(this.successfulRequests, 1); + } + if (attempts.some(({ provider, model }) => provider !== decision.selected.provider + || model !== decision.selected.model)) { + this.requestsWithModelFallback = saturatingAdd(this.requestsWithModelFallback, 1); + } + this.addAttempts(attempts); if (decision.pinned) { - // A session-pinned turn sent no decision request: it adds physical attempts and tokens only. + // A session-pinned turn sent no decision request, so it stays out of every decision total. this.pinnedRequests = saturatingAdd(this.pinnedRequests, 1); - this.addAttempts(sentAttempts(entry)); return; } @@ -410,9 +419,6 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { this.averageLatencyMs = nextMean(this.averageLatencyMs, this.decisions, decision.latencyMs); this.decisions = saturatingAdd(this.decisions, 1); if (decision.gate === "apply") this.appliedDecisions = saturatingAdd(this.appliedDecisions, 1); - if (entry.status >= 200 && entry.status < 400) { - this.successfulRequests = saturatingAdd(this.successfulRequests, 1); - } this.gates.set(decision.gate, saturatingAdd(this.gates.get(decision.gate) ?? 0, 1)); if (decision.confidence !== undefined) { this.confidenceTotal = saturatingAdd(this.confidenceTotal, decision.confidence); @@ -437,13 +443,6 @@ class StreamingJevStatsAccumulator implements JevStatsAccumulator { decision.selected.effort, saturatingAdd(selected.effortCounts.get(decision.selected.effort) ?? 0, 1), ); - - const attempts = sentAttempts(entry); - if (attempts.some(({ provider, model }) => provider !== decision.selected.provider - || model !== decision.selected.model)) { - this.requestsWithModelFallback = saturatingAdd(this.requestsWithModelFallback, 1); - } - this.addAttempts(attempts); } private addAttempts(attempts: readonly SentAttempt[]): void { diff --git a/structure/providers/jev-decision.md b/structure/providers/jev-decision.md index 86c8690f399..67d25b42d68 100644 --- a/structure/providers/jev-decision.md +++ b/structure/providers/jev-decision.md @@ -212,7 +212,10 @@ not read `nearly_exhausted`; otherwise the pin is released and JEV decides. Any request forgets the pin, cancellation does not, and a config generation change invalidates both stored pins and in-flight writers. A reused turn sends no decision request and persists `pinned: true` with `pinReason: "session"`; a decision that replaced a released pin persists the reason. `src/usage/jev-stats.ts` counts pinned turns as -`pinnedRequests`, never as decisions, gates or backend latency, while their physical attempts still count per model. +`pinnedRequests`, never as decisions, gates or backend latency; request success, model fallback and physical +attempts count every JEV-routed request, pinned or decided. The CLI projection reads an absent `pinnedRequests` as 0. +Besides a classified failure response, a non-cancellation dispatch, preflight or failure-read exception and a +committed stream that ends `failed` also forget the pin; `incomplete` is an answer and keeps it. The store holds at most 512 pins with least-recently-written eviction. Coverage: `tests/routing/jev-session-pin.test.ts`, `tests/server/server-jev-session-pin.test.ts` and `tests/cli/cli-combo-session-pin.test.ts`. diff --git a/tests/routing/jev-session-pin.test.ts b/tests/routing/jev-session-pin.test.ts index ef41b32a791..9e6b93c22b5 100644 --- a/tests/routing/jev-session-pin.test.ts +++ b/tests/routing/jev-session-pin.test.ts @@ -188,19 +188,26 @@ describe("persisted session pin evidence", () => { expect(normalizePersistedJevDecision({ ...base, pinned: true, pinReason: "expired" })).toBeUndefined(); }); - test("stats count pinned turns apart from decisions while keeping their physical tokens", () => { - const entry = (jevDecision: Record, timestamp: number) => ({ + test("stats count pinned turns apart from decisions while keeping their physical tokens and request outcomes", () => { + const entry = (jevDecision: Record, timestamp: number, provider = "sol") => ({ timestamp, status: 200, jevDecision, - attempts: [{ provider: "sol", model: "gpt-5.6-sol", sendCount: 1, usageStatus: "reported", usage: { inputTokens: 100, outputTokens: 5, cacheReadInputTokens: 80 } }], + attempts: [{ provider, model: "gpt-5.6-sol", sendCount: 1, usageStatus: "reported", usage: { inputTokens: 100, outputTokens: 5, cacheReadInputTokens: 80 } }], }) as unknown as PersistedUsageEntry; const accumulator = createJevStatsAccumulator({ comboId: "auto" }); accumulator.add(entry({ ...base, latencyMs: 40 }, 1)); accumulator.add(entry({ ...base, pinned: true, pinReason: "session" }, 2)); + accumulator.add(entry({ ...base, pinned: true, pinReason: "session" }, 3, "luna")); const stats = accumulator.clone().summarize("all", 3); - expect(stats.summary).toMatchObject({ decisions: 1, appliedDecisions: 1, pinnedRequests: 1, modelAttempts: 2, modelCacheReadTokens: 160, averageLatencyMs: 40 }); + expect(stats.summary).toMatchObject({ + decisions: 1, appliedDecisions: 1, pinnedRequests: 2, successfulRequests: 3, requestsWithModelFallback: 1, + modelAttempts: 3, modelCacheReadTokens: 240, averageLatencyMs: 40, + }); expect(stats.gates).toEqual([{ gate: "apply", decisions: 1 }]); - expect(stats.models).toEqual([expect.objectContaining({ provider: "sol", picks: 1, attempts: 2, inputTokens: 200 })]); + expect(stats.models).toEqual([ + expect.objectContaining({ provider: "sol", picks: 1, attempts: 2, inputTokens: 200 }), + expect.objectContaining({ provider: "luna", picks: 0, attempts: 1 }), + ]); }); }); diff --git a/tests/server/server-jev-session-pin.test.ts b/tests/server/server-jev-session-pin.test.ts index fa2879307a8..13110eb5d23 100644 --- a/tests/server/server-jev-session-pin.test.ts +++ b/tests/server/server-jev-session-pin.test.ts @@ -67,14 +67,15 @@ function makeConfig(choice: string, decisions: string[], sessionPin: boolean): O }; } -type Child = (body: Record, complete: () => void) => Response | Promise; +type ChildOptions = Parameters[3]; +type Child = (body: Record, complete: () => void, options: ChildOptions) => Response | Promise; /** Children complete like a real stream: the completion callback runs before the response returns. */ function dispatchers(child: Child): ResponsesDispatchers { return { async handleResponses(request, _config, _logCtx, options) { const body = await request.json() as Record; - return child(body, () => options?.onResponseComplete?.(String(body.model))); + return child(body, () => options?.onResponseComplete?.(String(body.model)), options); }, async handleComboResponses() { throw new Error("nested combo dispatch is not expected"); @@ -106,9 +107,9 @@ async function turn(config: OcxConfig, options: { translatorBudget: budget, ...(options.admission ? { admission: options.admission } : {}), ...(options.signal ? { abortSignal: options.signal } : {}), - }, dispatchers((childBody, complete) => { + }, dispatchers((childBody, complete, childOptions) => { options.models?.push(String(childBody.model)); - return child(childBody, complete); + return child(childBody, complete, childOptions); })); } finally { budget.dispose(); @@ -184,6 +185,25 @@ describe("JEV session pin (#7525)", () => { expect(log.jevDecision).not.toHaveProperty("pinned"); }); + test("a pinned stream that ends failed, or a dispatch that throws, also ends the pin", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + await turn(config); + const failedStream: Child = (body, _complete, options) => { + options?.onNativePassthroughTerminal?.("failed"); + return Response.json({ id: "resp-failed", object: "response", status: "completed", model: body.model, output: [] }); + }; + await turn(config, { child: failedStream }); + expect(decisions).toHaveLength(1); + await turn(config); + expect(decisions).toHaveLength(2); + const thrown: Child = () => { throw new Error("transport exploded"); }; + await expect(turn(config, { child: thrown })).rejects.toThrow("transport exploded"); + expect(decisions).toHaveLength(2); + await turn(config); + expect(decisions).toHaveLength(3); + }); + test("a cancelled send neither writes nor drops the pin", async () => { const decisions: string[] = []; const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); From 1c44e42383aaed7d1b85b07c9cec0c7c1abdf570 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 23 Oct 2026 08:39:18 +0900 Subject: [PATCH 3/4] fix(jev): fence session pins against in-flight writers after a failure --- src/server/responses/jev-session-pin.ts | 28 +++++++++++++++++++-- structure/providers/jev-decision.md | 3 ++- tests/routing/jev-session-pin.test.ts | 14 +++++++++++ tests/server/server-jev-session-pin.test.ts | 17 +++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/server/responses/jev-session-pin.ts b/src/server/responses/jev-session-pin.ts index 1856f9d9f36..d6178535740 100644 --- a/src/server/responses/jev-session-pin.ts +++ b/src/server/responses/jev-session-pin.ts @@ -17,8 +17,9 @@ * generation change happens after it was written, when its target is not among the request's * current JEV choices (disabled, removed, cooling down, quota exhausted, metered or otherwise * ineligible), when its effort is no longer offered, when opted-in quota evidence reads - * `nearly_exhausted`, and when any target of a pinned session fails. A cancelled send writes - * and releases nothing. Storage is process-local and bounded by `JEV_SESSION_PIN_CAPACITY`. + * `nearly_exhausted`, and when any target of a pinned session fails; a request that was already in + * flight when that failure was recorded cannot write the pin back. A cancelled send writes and + * releases nothing. Storage is process-local and bounded by `JEV_SESSION_PIN_CAPACITY`. */ import type { JevCandidate, JevDecision, JevDecisionBackend } from "../../combos/jev"; import { jevQuotaSignalForTarget } from "../../combos/jev-quota"; @@ -41,6 +42,14 @@ interface PinEntry { } const pins = new Map(); +/** + * Failure fence per session key. `forget()` stamps the key with a fresh sequence number; a request + * that started before that stamp may no longer write the pin back, so a concurrent completion on + * the same session cannot restore a target another request just saw fail. Bounded like the pins: + * an evicted stamp reads as 0, which only re-admits writers that started before any failure. + */ +const failureFences = new Map(); +let failureSequence = 0; function callerNamespace(admission: DataPlaneAdmission | undefined): string { if (!admission) return "unadmitted"; @@ -60,6 +69,16 @@ function write(key: string, entry: PinEntry): void { } } +function fenceFailure(key: string): void { + failureFences.delete(key); + failureFences.set(key, ++failureSequence); + while (failureFences.size > JEV_SESSION_PIN_CAPACITY) { + const oldest = failureFences.keys().next().value; + if (oldest === undefined) break; + failureFences.delete(oldest); + } +} + export interface JevSessionPinReuse { /** The pinned decision to apply instead of asking the decision backend. */ decision?: JevDecision; @@ -99,6 +118,7 @@ export function jevSessionPinScope(args: { if (args.combo.strategy !== "jev" || ttlMs === undefined || !args.lane) return undefined; const key = `${callerNamespace(args.admission)}\u0000${args.comboId}\u0000${args.lane}`; const now = args.now ?? Date.now; + const fence = failureFences.get(key) ?? 0; const release = (reason: JevSessionPinRelease): JevSessionPinReuse => { pins.delete(key); return { released: reason }; @@ -136,6 +156,8 @@ export function jevSessionPinScope(args: { complete(completedTargetKey, decision, writerGeneration) { // A config change while the response was in flight makes this completion obsolete. if (writerGeneration !== captureConfigGeneration()) return; + // So does a failure on this session reported after this request started. + if ((failureFences.get(key) ?? 0) !== fence) return; if (!decision) { const entry = pins.get(key); if (entry && entry.targetKey === completedTargetKey && entry.generation === writerGeneration) { @@ -155,6 +177,7 @@ export function jevSessionPinScope(args: { }, forget() { pins.delete(key); + fenceFailure(key); }, }; } @@ -162,6 +185,7 @@ export function jevSessionPinScope(args: { /** Test-only reset, alongside the Combo recall and rotation resets. */ export function clearJevSessionPinsForTests(): void { pins.clear(); + failureFences.clear(); } /** Test-only view of the retained pin count. */ diff --git a/structure/providers/jev-decision.md b/structure/providers/jev-decision.md index 67d25b42d68..c6794e1a9c9 100644 --- a/structure/providers/jev-decision.md +++ b/structure/providers/jev-decision.md @@ -215,7 +215,8 @@ a decision that replaced a released pin persists the reason. `src/usage/jev-stat `pinnedRequests`, never as decisions, gates or backend latency; request success, model fallback and physical attempts count every JEV-routed request, pinned or decided. The CLI projection reads an absent `pinnedRequests` as 0. Besides a classified failure response, a non-cancellation dispatch, preflight or failure-read exception and a -committed stream that ends `failed` also forget the pin; `incomplete` is an answer and keeps it. +committed stream that ends `failed` also forget the pin; `incomplete` is an answer and keeps it. Each forget stamps a +per-session failure fence, so a request already in flight on that session cannot write the pin back. The store holds at most 512 pins with least-recently-written eviction. Coverage: `tests/routing/jev-session-pin.test.ts`, `tests/server/server-jev-session-pin.test.ts` and `tests/cli/cli-combo-session-pin.test.ts`. diff --git a/tests/routing/jev-session-pin.test.ts b/tests/routing/jev-session-pin.test.ts index 9e6b93c22b5..bbb54340313 100644 --- a/tests/routing/jev-session-pin.test.ts +++ b/tests/routing/jev-session-pin.test.ts @@ -158,6 +158,20 @@ describe("JEV session pin scope", () => { expect(pin.reuse([sol])).toEqual({}); }); + test("a request already in flight when another one fails cannot restore the pin", () => { + scope().complete(sol.key, applied, captureConfigGeneration()); + const inFlight = scope(); + const failing = scope(); + failing.forget(); + inFlight.complete(sol.key, { ...applied, pinned: true, pinReason: "session" }, captureConfigGeneration()); + inFlight.complete(sol.key, applied, captureConfigGeneration()); + expect(scope().reuse([sol])).toEqual({}); + // A request that starts after the failure decides afresh and may pin again. + const next = scope(); + next.complete(sol.key, applied, captureConfigGeneration()); + expect(scope().reuse([sol]).decision?.pinned).toBe(true); + }); + test("the store is bounded and evicts the least recently written session", () => { for (let lane = 0; lane <= JEV_SESSION_PIN_CAPACITY; lane += 1) { scope({ lane: `lane-${lane}` }).complete(sol.key, applied, captureConfigGeneration()); diff --git a/tests/server/server-jev-session-pin.test.ts b/tests/server/server-jev-session-pin.test.ts index 13110eb5d23..0191e656a36 100644 --- a/tests/server/server-jev-session-pin.test.ts +++ b/tests/server/server-jev-session-pin.test.ts @@ -204,6 +204,23 @@ describe("JEV session pin (#7525)", () => { expect(decisions).toHaveLength(3); }); + test("a stream that reports failed after the Combo committed it ends the pin", async () => { + const decisions: string[] = []; + const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); + await turn(config); + let committedOptions: ChildOptions; + const lateFailure: Child = (body, _complete, options) => { + committedOptions = options; + return Response.json({ id: "resp-late", object: "response", status: "completed", model: body.model, output: [] }); + }; + expect((await turn(config, { child: lateFailure })).status).toBe(200); + expect(decisions).toHaveLength(1); + // The callback gate is committed once the Combo returned the response; the stream fails later. + committedOptions?.onNativePassthroughTerminal?.("failed"); + await turn(config); + expect(decisions).toHaveLength(2); + }); + test("a cancelled send neither writes nor drops the pin", async () => { const decisions: string[] = []; const config = makeConfig("sol/gpt-5.6-sol:high", decisions, true); From fd7f316195b22e7e30acbc5ab61f2425ecdf3300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A3=BC=EB=8B=88?= Date: Fri, 23 Oct 2026 08:50:46 +0900 Subject: [PATCH 4/4] fix(jev): refuse stale pin writers once a failure fence was evicted --- src/server/responses/jev-session-pin.ts | 15 ++++++++++++--- tests/routing/jev-session-pin.test.ts | 12 ++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/server/responses/jev-session-pin.ts b/src/server/responses/jev-session-pin.ts index d6178535740..e0df28b1f16 100644 --- a/src/server/responses/jev-session-pin.ts +++ b/src/server/responses/jev-session-pin.ts @@ -45,11 +45,14 @@ const pins = new Map(); /** * Failure fence per session key. `forget()` stamps the key with a fresh sequence number; a request * that started before that stamp may no longer write the pin back, so a concurrent completion on - * the same session cannot restore a target another request just saw fail. Bounded like the pins: - * an evicted stamp reads as 0, which only re-admits writers that started before any failure. + * the same session cannot restore a target another request just saw fail. Bounded like the pins; + * because an evicted stamp is unknowable, a writer whose key has no stamp is also refused once any + * stamp was evicted after it started. */ const failureFences = new Map(); let failureSequence = 0; +/** `failureSequence` at the most recent fence eviction. */ +let fenceEvictedAt = 0; function callerNamespace(admission: DataPlaneAdmission | undefined): string { if (!admission) return "unadmitted"; @@ -76,6 +79,7 @@ function fenceFailure(key: string): void { const oldest = failureFences.keys().next().value; if (oldest === undefined) break; failureFences.delete(oldest); + fenceEvictedAt = failureSequence; } } @@ -119,6 +123,7 @@ export function jevSessionPinScope(args: { const key = `${callerNamespace(args.admission)}\u0000${args.comboId}\u0000${args.lane}`; const now = args.now ?? Date.now; const fence = failureFences.get(key) ?? 0; + const startedAtSequence = failureSequence; const release = (reason: JevSessionPinRelease): JevSessionPinReuse => { pins.delete(key); return { released: reason }; @@ -157,7 +162,9 @@ export function jevSessionPinScope(args: { // A config change while the response was in flight makes this completion obsolete. if (writerGeneration !== captureConfigGeneration()) return; // So does a failure on this session reported after this request started. - if ((failureFences.get(key) ?? 0) !== fence) return; + const currentFence = failureFences.get(key); + if ((currentFence ?? 0) !== fence) return; + if (currentFence === undefined && fenceEvictedAt > startedAtSequence) return; if (!decision) { const entry = pins.get(key); if (entry && entry.targetKey === completedTargetKey && entry.generation === writerGeneration) { @@ -186,6 +193,8 @@ export function jevSessionPinScope(args: { export function clearJevSessionPinsForTests(): void { pins.clear(); failureFences.clear(); + failureSequence = 0; + fenceEvictedAt = 0; } /** Test-only view of the retained pin count. */ diff --git a/tests/routing/jev-session-pin.test.ts b/tests/routing/jev-session-pin.test.ts index bbb54340313..cfd09a99bbe 100644 --- a/tests/routing/jev-session-pin.test.ts +++ b/tests/routing/jev-session-pin.test.ts @@ -172,6 +172,18 @@ describe("JEV session pin scope", () => { expect(scope().reuse([sol]).decision?.pinned).toBe(true); }); + test("an evicted failure fence still refuses a writer that started before it", () => { + scope().complete(sol.key, applied, captureConfigGeneration()); + const inFlight = scope(); + scope().forget(); + for (let lane = 0; lane <= JEV_SESSION_PIN_CAPACITY; lane += 1) scope({ lane: `other-${lane}` }).forget(); + inFlight.complete(sol.key, applied, captureConfigGeneration()); + expect(scope().reuse([sol])).toEqual({}); + const next = scope(); + next.complete(sol.key, applied, captureConfigGeneration()); + expect(scope().reuse([sol]).decision?.pinned).toBe(true); + }); + test("the store is bounded and evicts the least recently written session", () => { for (let lane = 0; lane <= JEV_SESSION_PIN_CAPACITY; lane += 1) { scope({ lane: `lane-${lane}` }).complete(sol.key, applied, captureConfigGeneration());