diff --git a/docs-site/src/content/docs/guides/codex-integration.md b/docs-site/src/content/docs/guides/codex-integration.md index 82cf7aaf446..c38a4685ddd 100644 --- a/docs-site/src/content/docs/guides/codex-integration.md +++ b/docs-site/src/content/docs/guides/codex-integration.md @@ -1154,7 +1154,8 @@ Pool quota responses can authorize a pause only with valid usage and a captured that is still current. A response without that proof may refresh displayed usage without pausing an account. Check this server’s authenticated `GET /api/codex-auth/low-quota-events` history for `logged` alerts or save failures. Manual resume remains in force for the current quota episode. The -default alert reaches only the log and API; it does not produce a desktop or OS notification. +default alert reaches only the log and API. Add `actions.desktop: true` to also show a macOS or +Linux notification on the machine running the proxy; Windows stays log-only. When the ChatGPT 5-hour quota is exhausted, Codex may offer a reserve fallback model (`gpt-reserve` / Luna Reserve). While that state is active, the Codex model picker can make diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 3c8ef94a6a4..a74c0f2a1f4 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -29,7 +29,7 @@ alias would produce an unparseable synthetic Fast selector. Codex's own model ca account keeps its separate 98% hard lock. The optional shape is: ```json -{"codexPool":{"lowQuotaProtection":{"enabled":true,"threshold":80,"actions":{"pause":true,"notify":true},"windows":{"short":true,"weekly":true}}}} +{"codexPool":{"lowQuotaProtection":{"enabled":true,"threshold":80,"actions":{"pause":true,"notify":true,"desktop":true},"windows":{"short":true,"weekly":true}}}} ``` Absence or `enabled: false` disables it. `threshold` is a finite inclusive percentage from 1 @@ -52,9 +52,20 @@ With `notify`, the server writes a local log line with window and percentage but and records a bounded event. Authenticated `GET /api/codex-auth/low-quota-events` exposes only that server’s events, including account id, window, usage percentage, known reset time, timestamp, and status. The default status is `logged`: the alert reached the log and event history only. -There is no desktop or OS notification. Each account/window logs once per server-local episode. -If an injected notification sink fails, its failure is recorded and a later eligible observation -can retry it; only a successful sink is marked `delivered`. +Each account/window logs once per server-local episode. + +Optional `actions.desktop: true` (it requires `notify: true`) also hands each notice to the +operating-system notifier of the machine running the proxy: `/usr/bin/osascript` on macOS, or +`notify-send` from `/usr/bin`, `/usr/local/bin` or `/bin` on Linux. The notification names the account +only by its pseudonymous log label (for example `p1a2b3c`), never its email, alias or id, and +includes the window, usage, threshold and known reset time. Windows and hosts without a supported +notifier keep the alert `logged`. `delivered` means the OS notifier accepted the notice; it cannot +see whether the system's notification settings then hide it. A failed or timed-out notifier is +recorded as `failed`; an eligible observation at least five minutes later can retry it, without +repeating the log line. Turning `desktop` on or off +applies to the next observation without a restart. A notice still in flight is cancelled at shutdown, and +when the next quota observation detects that its episode ended (reset boundary, recovery, +account removal or policy change); until then the notifier is bounded by a five-second timeout. ## Model resolution order diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 89cc545d264..c5176ae8445 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -776,7 +776,7 @@ manager. Its routes are: | `PUT, PATCH /api/codex-auth/pool-strategy` | Update Codex account-pool selection strategy | 400 invalid strategy/config | | `PUT /api/codex-auth/failover` | Set the account failover threshold | 400 invalid threshold | | `GET /api/codex-auth/quota` | Read cached quota state by account | — | -| `GET /api/codex-auth/low-quota-events?limit=20` | Read only this server’s last 0–100 low-quota log/notice and pause-save events (default 20); includes account id and status (`logged` for the default log-only alert; `delivered` for a successful injected notice sink; `succeeded` for a completed pause save) | 400 invalid limit; management authentication required | +| `GET /api/codex-auth/low-quota-events?limit=20` | Read only this server’s last 0–100 low-quota log/notice and pause-save events (default 20); includes account id and status (`logged` for the default log-only alert or a host without a supported notifier; `delivered` when the opted-in OS notifier accepted the notice; `failed` when it failed or timed out; `cancelled` when its episode ended first; `succeeded` for a completed pause save) | 400 invalid limit; management authentication required | | `GET /api/codex-auth/reset-credits` | Inspect reset-credit eligibility for an account | 400 missing account id; upstream status passthrough; 500 lookup failure | | `POST /api/codex-auth/reset-credits/consume` | Consume an eligible reset credit. Optional `operationId` (UUIDv4) makes the redemption idempotent: the same id replays one durable outcome instead of spending a second credit. | 400 missing account id or invalid `operationId`; 409 `identity_mismatch` when the id belongs to another account; upstream status passthrough; 503 `server_busy`, `capacity`, or `unavailable`; 500 consume failure | | `POST /api/codex-auth/login` | Start Codex login or reauthentication | 400 invalid request; conflict/busy login states | diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 1a803455f1a..7caa599b932 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -109,6 +109,7 @@ "cursor-local-installer.test.ts": "providers/cursor", "codex-quota-query-backoff.test.ts": "codex-integration", "pnpm-command-isolation.test.ts": "update", "provider-antigravity-quota-retry.test.ts": "providers", "project-config-warning-snapshot.test.ts": "codex-integration", "codex-quota-auto-refresh-generation.test.ts": "codex-integration", "codex-account-clear-paused.test.ts": "codex-integration", + "low-quota-desktop-notifier.test.ts": "codex-integration", "low-quota-protection.test.ts": "codex-integration", "bun-runtime-preflight.test.ts": "lib", "responses-compaction-recovery.test.ts": "responses", "responses-canonical-nonstream.test.ts": "responses", "compaction-recovery-settings.test.ts": "config", "responses-compaction-recovery-policy.test.ts": "responses", "plugin-loader.test.ts": "lib", "plugin-upstream-hooks.test.ts": "lib", diff --git a/src/codex/low-quota-desktop-notifier.ts b/src/codex/low-quota-desktop-notifier.ts new file mode 100644 index 00000000000..071c83c5ccf --- /dev/null +++ b/src/codex/low-quota-desktop-notifier.ts @@ -0,0 +1,125 @@ +import { existsSync } from "node:fs"; +import { isTestHomeGuardArmed } from "../lib/test-home-guard"; + +/** What a low-quota notice may tell the operating system. Never an email, raw id or credential. */ +export type LowQuotaNotice = { + /** Pseudonymous `p` log label of the pool account. */ + accountLabel: string; + window: "short" | "weekly"; + percentUsed: number; + threshold: number; + /** Epoch milliseconds of the window reset, when known. */ + resetAt: number | null; +}; + +/** Resolves when the OS notifier accepted the notice; rejects on failure, timeout or abort. */ +export type LowQuotaNoticeSink = (notice: LowQuotaNotice, signal: AbortSignal) => Promise; + +type SpawnedNotifier = { exited: Promise; kill(): void }; +export type LowQuotaNotifierIo = { + platform?: NodeJS.Platform; + /** Returns true when the absolute path is an existing executable candidate. */ + exists?: (path: string) => boolean; + spawn?: (argv: string[]) => SpawnedNotifier; + timeoutMs?: number; + /** Formats the reset time for display; defaults to the host locale. */ + formatTime?: (epochMs: number) => string; +}; + +const OSASCRIPT = "/usr/bin/osascript"; +/** Literal system locations only: a PATH lookup would let any earlier PATH entry run as the proxy. */ +const NOTIFY_SEND_CANDIDATES = ["/usr/bin/notify-send", "/usr/local/bin/notify-send", "/bin/notify-send"]; +const DEFAULT_TIMEOUT_MS = 5_000; +const TITLE = "OpenCodex quota"; + +export function formatLowQuotaNotice(notice: LowQuotaNotice, formatTime: (epochMs: number) => string = defaultFormatTime): { title: string; body: string } { + const window = notice.window === "short" ? "5-hour" : "weekly"; + const reset = notice.resetAt === null ? "reset time unknown" : `resets ${formatTime(notice.resetAt)}`; + return { + title: TITLE, + body: `Account ${notice.accountLabel}: ${window} quota at ${Math.round(notice.percentUsed)}% (threshold ${notice.threshold}%), ${reset}.`, + }; +} + +function defaultFormatTime(epochMs: number): string { + return new Date(epochMs).toLocaleString(); +} + +/** notify-send bodies may be parsed as markup by the notification server. */ +function escapeMarkup(text: string): string { + return text.replace(/&/g, "&").replace(//g, ">"); +} + +/** + * Builds the fixed argv for the host notifier. The notice text is always a separate argv item: + * the AppleScript source is constant and reads its arguments, and notify-send stops option + * parsing at `--`, so no account or quota text is ever interpreted as script or flags. + */ +export function lowQuotaNotifierArgv( + platform: NodeJS.Platform, + exists: (path: string) => boolean, + title: string, + body: string, +): string[] | undefined { + if (platform === "darwin") { + if (!exists(OSASCRIPT)) return undefined; + return [OSASCRIPT, "-e", "on run argv", "-e", "display notification (item 2 of argv) with title (item 1 of argv)", "-e", "end run", title, body]; + } + if (platform === "linux") { + const notifySend = NOTIFY_SEND_CANDIDATES.find(exists); + if (!notifySend) return undefined; + return [notifySend, "--app-name=OpenCodex", "--", title, escapeMarkup(body)]; + } + return undefined; +} + +function defaultSpawn(argv: string[]): SpawnedNotifier { + const proc = Bun.spawn(argv, { stdin: "ignore", stdout: "ignore", stderr: "ignore", windowsHide: true }); + return { exited: proc.exited, kill: () => { try { proc.kill(); } catch { /* already gone */ } } }; +} + +/** + * Returns a sink that hands notices to this host's notification service, or undefined when the + * platform has none we support (Windows and others stay log-only). An armed test process never + * reaches a real notifier unless a spawn is injected. + */ +export function resolveLowQuotaDesktopSink(io: LowQuotaNotifierIo = {}): LowQuotaNoticeSink | undefined { + if (!io.spawn && isTestHomeGuardArmed()) return undefined; + const platform = io.platform ?? process.platform; + const exists = io.exists ?? existsSync; + const spawn = io.spawn ?? defaultSpawn; + const timeoutMs = io.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const probe = lowQuotaNotifierArgv(platform, exists, TITLE, ""); + if (!probe) return undefined; + return (notice, signal) => { + if (signal.aborted) return Promise.reject(new Error("low-quota notice cancelled")); + const { title, body } = formatLowQuotaNotice(notice, io.formatTime); + const argv = lowQuotaNotifierArgv(platform, exists, title, body); + if (!argv) return Promise.reject(new Error("low-quota notifier unavailable")); + let child: SpawnedNotifier; + try { + child = spawn(argv); + } catch (error) { + return Promise.reject(error); + } + return new Promise((resolve, reject) => { + let settled = false; + const finish = (error?: Error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + signal.removeEventListener("abort", onAbort); + if (error) reject(error); + else resolve(); + }; + const onAbort = () => { child.kill(); finish(new Error("low-quota notice cancelled")); }; + const timer = setTimeout(() => { child.kill(); finish(new Error("low-quota notifier timed out")); }, timeoutMs); + (timer as { unref?: () => void }).unref?.(); + signal.addEventListener("abort", onAbort, { once: true }); + child.exited.then( + code => finish(code === 0 ? undefined : new Error(`low-quota notifier exited ${code}`)), + error => finish(error instanceof Error ? error : new Error("low-quota notifier failed")), + ); + }); + }; +} diff --git a/src/codex/low-quota-protection.ts b/src/codex/low-quota-protection.ts index 356efe10d33..9a82e3960d0 100644 --- a/src/codex/low-quota-protection.ts +++ b/src/codex/low-quota-protection.ts @@ -4,15 +4,28 @@ import { isCodexAccountPaused, setCodexAccountPaused } from "./account-pause"; import { createLowQuotaEventLedger, type LowQuotaEvent } from "./low-quota-events"; import { registerLowQuotaObserver } from "./low-quota-observer"; import { resetAtToMs } from "./quota-types"; +import { codexAccountLogLabel, fallbackCodexAccountLogLabel } from "./account-label"; +import type { LowQuotaNotice, LowQuotaNoticeSink } from "./low-quota-desktop-notifier"; -type Window = "short" | "weekly"; -type Notice = { window: Window; percentUsed: number; threshold: number }; type Dependencies = { persist?: (config: OcxConfig) => void | Promise; - notify?: (notice: Notice) => void | Promise; + /** Injected sink; takes precedence over desktop delivery. */ + notify?: (notice: LowQuotaNotice, signal: AbortSignal) => void | Promise; + /** Resolved per notice, and only while the live policy opts into actions.desktop. */ + desktopSink?: () => LowQuotaNoticeSink | undefined; }; type EventBase = Omit; -type Episode = { reset: string; notice: "in-flight" | "logged" | "delivered" | "failed" | undefined; pausedByUs: boolean; noticeBase?: EventBase }; +type Episode = { + reset: string; + notice: "in-flight" | "logged" | "delivered" | "failed" | undefined; + pausedByUs: boolean; + noticeBase?: EventBase; + noticeAbort?: AbortController; + /** A failed notice may retry only after this time, so a broken notifier is not respawned per request. */ + noticeRetryAt?: number; + /** The episode's log line was written; retries do not repeat it. */ + warned?: boolean; +}; export type LowQuotaRegistration = (() => void) & { hasPendingSave(): boolean; flush(): Promise; @@ -20,6 +33,7 @@ export type LowQuotaRegistration = (() => void) & { }; const RETRY_DELAYS_MS = [100, 250]; +const NOTICE_RETRY_MS = 5 * 60_000; const FLUSH_DEADLINE_MS = 500; /** Each server owns its own policy, episode state and deferred writer. */ @@ -44,6 +58,12 @@ export function registerCodexLowQuotaProtection(config: OcxConfig, deps: Depende if (delivery === "pause-save") pauseStatus.set(base, status); ledger.publish({ ...base, delivery, status, timestamp: Date.now() }); } + /** Publishes one cancellation for an in-flight notice and stops its delivery. */ + function cancelNotice(episode: Episode): void { + if (episode.notice === "in-flight" && episode.noticeBase) event(episode.noticeBase, "notice", "cancelled"); + episode.noticeAbort?.abort(); + episode.noticeAbort = undefined; + } function cancelTimer(): void { if (saveTimer) clearTimeout(saveTimer); saveTimer = null; @@ -132,9 +152,7 @@ export function registerCodexLowQuotaProtection(config: OcxConfig, deps: Depende if (pauseStatus.get(base) === "pending") event(base, "pause-save", "cancelled"); pendingPauseEvents.delete(key); } - for (const episode of episodes.values()) { - if (episode.notice === "in-flight" && episode.noticeBase) event(episode.noticeBase, "notice", "cancelled"); - } + for (const episode of episodes.values()) cancelNotice(episode); dirty = false; autoPausedAccounts.clear(); unregister(); @@ -175,9 +193,7 @@ export function registerCodexLowQuotaProtection(config: OcxConfig, deps: Depende const policy = config.codexPool?.lowQuotaProtection; const nextKey = JSON.stringify(policy); if (policyKey !== nextKey) { - for (const episode of episodes.values()) { - if (episode.notice === "in-flight" && episode.noticeBase) event(episode.noticeBase, "notice", "cancelled"); - } + for (const episode of episodes.values()) cancelNotice(episode); episodes.clear(); autoPausedAccounts.clear(); policyKey = nextKey; @@ -187,7 +203,7 @@ export function registerCodexLowQuotaProtection(config: OcxConfig, deps: Depende for (const id of autoPausedAccounts) if (!liveIds.has(id)) autoPausedAccounts.delete(id); for (const [key, episode] of episodes) { if (liveIds.has(key.split("\u0000")[0]!)) continue; - if (episode.notice === "in-flight" && episode.noticeBase) event(episode.noticeBase, "notice", "cancelled"); + cancelNotice(episode); episodes.delete(key); } if (!liveIds.has(accountId)) return; @@ -203,7 +219,7 @@ export function registerCodexLowQuotaProtection(config: OcxConfig, deps: Depende const key = `${accountId}\u0000${window}`; if (percentUsed < policy.threshold) { const prior = episodes.get(key); - if (prior?.notice === "in-flight" && prior.noticeBase) event(prior.noticeBase, "notice", "cancelled"); + if (prior) cancelNotice(prior); episodes.delete(key); continue; } @@ -211,6 +227,7 @@ export function registerCodexLowQuotaProtection(config: OcxConfig, deps: Depende const reset = resetAt === null ? "unknown" : String(resetAt); let episode = episodes.get(key); if (!episode || episode.reset !== reset) { + if (episode) cancelNotice(episode); episode = { reset, notice: undefined, pausedByUs: false }; episodes.set(key, episode); } @@ -234,32 +251,41 @@ export function registerCodexLowQuotaProtection(config: OcxConfig, deps: Depende scheduleSave(); } // A manual resume leaves pausedByUs set until recovery or a new reset episode. - if (policy.actions.notify && episode.notice !== "in-flight" && episode.notice !== "logged" && episode.notice !== "delivered") { - console.warn(`[codex-low-quota] ${window === "short" ? "5-hour" : "weekly"} quota reached ${percentUsed}% used (threshold ${policy.threshold}%)`); - if (!deps.notify) { + if (policy.actions.notify && episode.notice !== "in-flight" && episode.notice !== "logged" && episode.notice !== "delivered" + && !(episode.notice === "failed" && (episode.noticeRetryAt ?? 0) > Date.now())) { + if (!episode.warned) { + episode.warned = true; + console.warn(`[codex-low-quota] ${window === "short" ? "5-hour" : "weekly"} quota reached ${percentUsed}% used (threshold ${policy.threshold}%)`); + } + const sink = deps.notify ?? (policy.actions.desktop === true ? deps.desktopSink?.() : undefined); + if (!sink) { event(base, "notice", "logged"); episode.notice = "logged"; continue; } - episode.notice = "in-flight"; - episode.noticeBase = base; + const account = config.codexAccounts?.find(candidate => candidate.id === accountId); + const notice: LowQuotaNotice = { + accountLabel: account ? codexAccountLogLabel(account) : fallbackCodexAccountLogLabel(accountId), + window, percentUsed, threshold: policy.threshold, resetAt, + }; + const abort = new AbortController(); + const current = episode; + current.notice = "in-flight"; + current.noticeBase = base; + current.noticeAbort = abort; event(base, "notice", "pending"); + const settle = (status: "delivered" | "failed") => { + if (closed || abort.signal.aborted || episodes.get(key) !== current) return; + event(base, "notice", status); + current.notice = status; + current.noticeRetryAt = status === "failed" ? Date.now() + NOTICE_RETRY_MS : undefined; + current.noticeBase = undefined; + current.noticeAbort = undefined; + }; try { - void Promise.resolve(deps.notify({ window, percentUsed, threshold: policy.threshold })).then(() => { - if (closed || episodes.get(key) !== episode) return; - event(base, "notice", "delivered"); - episode.notice = "delivered"; - episode.noticeBase = undefined; - }, () => { - if (closed || episodes.get(key) !== episode) return; - event(base, "notice", "failed"); - episode.notice = "failed"; - episode.noticeBase = undefined; - }); + void Promise.resolve(sink(notice, abort.signal)).then(() => settle("delivered"), () => settle("failed")); } catch { - event(base, "notice", "failed"); - episode.notice = "failed"; - episode.noticeBase = undefined; + settle("failed"); } } } diff --git a/src/config/schema/leaf-validators.ts b/src/config/schema/leaf-validators.ts index 7f002082b0a..a24fee2840e 100644 --- a/src/config/schema/leaf-validators.ts +++ b/src/config/schema/leaf-validators.ts @@ -1023,12 +1023,16 @@ export const codexPoolSchema = z.object({ actions: z.object({ pause: z.boolean(), notify: z.boolean(), + desktop: z.boolean().optional(), }).strict(), windows: z.object({ short: z.boolean(), weekly: z.boolean(), }).strict(), }).strict().superRefine((policy, ctx) => { + if (policy.actions.desktop === true && !policy.actions.notify) { + ctx.addIssue({ code: "custom", path: ["actions", "desktop"], message: "desktop low-quota notification requires notify" }); + } if (!policy.enabled) return; if (!policy.actions.pause && !policy.actions.notify) { ctx.addIssue({ code: "custom", path: ["actions"], message: "enabled low-quota protection needs an action" }); diff --git a/src/server/background-lifecycle.ts b/src/server/background-lifecycle.ts index eaf56a3afc8..8a81af070fa 100644 --- a/src/server/background-lifecycle.ts +++ b/src/server/background-lifecycle.ts @@ -1,5 +1,6 @@ import type { OcxConfig, StorageCleanupPolicy } from "../types"; import { registerCodexLowQuotaProtection, type LowQuotaRegistration } from "../codex/low-quota-protection"; +import { resolveLowQuotaDesktopSink } from "../codex/low-quota-desktop-notifier"; import type { LowQuotaEvent } from "../codex/low-quota-events"; import { startStateStoreSweeper } from "../lib/state-store-sweeper"; import { @@ -209,7 +210,7 @@ export function acquireServerBackgroundLifecycle( fanoutRevoked: false, applyPolicy, resources: acquireServerResourceOwner(), - lowQuota: registerCodexLowQuotaProtection(config), + lowQuota: registerCodexLowQuotaProtection(config, { desktopSink: () => resolveLowQuotaDesktopSink() }), }; try { if (!processLoops) { diff --git a/src/types/config.ts b/src/types/config.ts index d79173a4373..b649692d45a 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1769,6 +1769,8 @@ export interface CodexLowQuotaProtectionConfig { actions: { pause: boolean; notify: boolean; + /** Opt-in: also hand each notice to this host's OS notifier (macOS, Linux). Requires notify. */ + desktop?: boolean; }; windows: { short: boolean; diff --git a/structure/config.md b/structure/config.md index 76680c0ba06..238a59f6f0c 100644 --- a/structure/config.md +++ b/structure/config.md @@ -516,7 +516,7 @@ described in [Responses transport](transports/responses.md), not upstream policy ## Codex Pool low-quota protection -`codexPool.lowQuotaProtection` is opt-in, requires a 1–100 threshold and a selected action/window when enabled, covers pool accounts only and is independent of proactive switching and the main account’s per-window hard lock (`codexMainAccountHardLockThresholds`, short/long integers 80–100 defaulting to 90/98, owned by [OpenAI accounts](providers/openai-accounts.md)). `src/codex/low-quota-protection.ts` pauses in live `pausedCodexAccountIds` before selection, then coalesces a deferred config save with bounded retry and shutdown flush. Fresh accepted observations reach `src/codex/low-quota-observer.ts`; credits-only and expired windows do not act. Manual resume suppresses repause across currently qualifying window episodes; a new reset boundary or below-threshold reading re-arms the policy, but never automatically resumes an account. A timed-out in-flight save remains pending until its eventual success or failure; queued work is cancelled at owner close. An unsuccessful save does not survive restart. The default alert is log-and-API only and records `logged`, not notification delivery. +`codexPool.lowQuotaProtection` is opt-in, requires a 1–100 threshold and a selected action/window when enabled, covers pool accounts only and is independent of proactive switching and the main account’s per-window hard lock (`codexMainAccountHardLockThresholds`, short/long integers 80–100 defaulting to 90/98, owned by [OpenAI accounts](providers/openai-accounts.md)). `src/codex/low-quota-protection.ts` pauses in live `pausedCodexAccountIds` before selection, then coalesces a deferred config save with bounded retry and shutdown flush. Fresh accepted observations reach `src/codex/low-quota-observer.ts`; credits-only and expired windows do not act. Manual resume suppresses repause across currently qualifying window episodes; a new reset boundary or below-threshold reading re-arms the policy, but never automatically resumes an account. A timed-out in-flight save remains pending until its eventual success or failure; queued work is cancelled at owner close. An unsuccessful save does not survive restart. The default alert is log-and-API only and records `logged`, not notification delivery. Optional `actions.desktop` (valid only with `notify`, checked even while disabled) is read from the live policy per notice; `src/codex/low-quota-desktop-notifier.ts` resolves a macOS `osascript` or Linux `notify-send` sink with fixed argv, a timeout and an abort signal, returns none on other platforms or in an armed test process, and never receives email, alias, id or credentials. Shutdown, and every episode end detected by the next observation (reset boundary, recovery, account removal, policy change), aborts an in-flight notice and records one `cancelled`; config mutations alone do not cancel, so a notifier is otherwise bounded by its five-second timeout. A `failed` notice retries no sooner than five minutes later on the next eligible observation, and each episode writes its log line once. ## Management-backed CLI commands need a management plane diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index e71dd5d9d43..b481a8a7895 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -315,7 +315,7 @@ after the save, and a non-retryable skip (no managed catalog) is a clean save. C | Protocol paths | `src/server/management/protocol-routes.ts` (lazy-loaded) — `GET /api/protocols` returns the contract version, the resolved API surfaces and protocol settings, the policy revision and the feature vocabulary; with `?provider=` (one non-empty name of at most 200 characters without control characters, else 400 `invalid_provider`; an unconfigured name is 404 `unknown_provider`, neither echoing the name) it adds a `provider` block from `src/protocols/provider-summary.ts` ([Protocol Paths](data-planes/protocol-paths.md#provider-wire-summary)); `POST /api/protocols/plan` takes `{ model, inbound, features? }` (model at most 200 characters, at most 24 features, any other key refused with 400) and returns a `ProtocolPlanV1` with `basis: "preview"` from `src/protocols/plan-snapshot.ts`. Both are read-only, never log their input, and send nothing upstream. `PATCH /api/protocols/settings` takes `{ messagesEnabled?, unrepresentable?, rollout? }` (strict: unknown keys and wrong types are 400), validates and applies through `src/server/management/protocol-settings-patch.ts`, persists with `saveConfigPreservingClaudeCode`, restores the live config if the save fails (409 on lock contention, 500 otherwise), and answers with the fresh `GET /api/protocols` body; closing Messages also writes `claudeCode.enabled = false` through `commitClaudeCodeBlock` ([Protocol Paths](data-planes/protocol-paths.md#settings)). The CLI drives all three through `ocx api protocols`, `ocx api explain` and `ocx api policy` ([Protocol Paths](data-planes/protocol-paths.md#cli)); none carries a route-registry exemption. | | Workflow budget | `src/server/management/workflow-budget-routes.ts` — `GET /api/workflow-budget` reads the tracked roots or one root, and `POST /api/workflow-budget/clear` clears exactly one. The clear moves the windowed send ring and the child map and nothing else: `active` belongs to turns still in flight, the spend ledger is a token budget an operator did not ask to forgive, and the lifetime send total survives so a clear cannot launder the record. A refusal event carries `spendScope` and `spendLimit` when a token ceiling fired, so the reason is readable without the config open beside it; no scope id is ever attached, because root ids are client thread headers and identity ids are credentials. Both are `deferred-verb` in the route registry — they are owed CLI verbs, and because the ledger is process memory there is no local projection the CLI could read instead. See [`../devlog/_fin/260915_workflow_budget_window/030_wfc_diff_plan.md`](../devlog/_fin/260915_workflow_budget_window/030_wfc_diff_plan.md). | | Codex accounts | `src/codex/auth-api/routes.ts` — `GET/POST/DELETE /api/codex-auth/accounts`, `PUT /api/codex-auth/accounts/alias`, `PUT /api/codex-auth/accounts/pause`, `PUT /api/codex-auth/accounts/pause-exhausted`, `POST /api/codex-auth/accounts/clear-cooldown`, `GET/PUT /api/codex-auth/active`, `PUT /api/codex-auth/auto-switch`, `PUT /api/codex-auth/pool-strategy`, `PUT /api/codex-auth/failover`, `GET /api/codex-auth/quota`, `GET /api/codex-auth/reset-credits` with `POST /api/codex-auth/reset-credits/consume`, and the login flow `POST /api/codex-auth/login`, `POST /api/codex-auth/login/code`, `POST /api/codex-auth/login/cancel`, `GET /api/codex-auth/login-status`. Per-account quota activation uses the existing `GET/PUT /api/settings` surface and `src/codex/quota-auto-refresh.ts`, keeping scheduled spending separate from credential/authentication mutation. Account ids are opaque handles and are serialized so the GUI can address an account; emails are masked and tokens are never serialized. New-account config commits add UI-managed selector bindings in the same config save; deletion deliberately retains existing bindings for fail-closed exact routing and re-add stability. Account mutations request catalog convergence only after config durability and expose only the boolean `catalogRefreshPending` completion projection. `showCodexCredits` is a default-off boolean in settings GET and successful PUT; partial PUT accepts a boolean and rolls back on persistence failure. See [credits exposure](providers/openai-accounts.md#display-only-codex-credits). | -| Low-quota history | `src/server/management/low-quota-routes.ts` lazily serves authenticated `GET /api/codex-auth/low-quota-events` from the bounded ledger owned by the server handling the request. The response includes only that server’s account ids and notice/pause-save status; logs omit identifiers. The default notice records `logged`; a notice records `delivered` only after an injected sink succeeds, while a pause-save records `succeeded` after persistence succeeds. A timed-out in-flight save remains `pending` until it settles. Invalid limits return 400. No OS notification is emitted. | +| Low-quota history | `src/server/management/low-quota-routes.ts` lazily serves authenticated `GET /api/codex-auth/low-quota-events` from the bounded ledger owned by the server handling the request. The response includes only that server’s account ids and notice/pause-save status; logs omit identifiers. The default notice records `logged`; a notice records `delivered` only after the injected or opted-in OS notifier sink succeeds, `failed` when it fails, and `cancelled` when its episode ends first, while a pause-save records `succeeded` after persistence succeeds. A timed-out in-flight save remains `pending` until it settles. Invalid limits return 400. The route itself emits no OS notification. | | Sidebar | `src/server/management/sidebar-routes.ts` — `GET/POST /api/github/star`, `GET /api/update/badge`, and `POST /api/update/desktop-snapshot`. The snapshot POST accepts the raw admin-token principal or the dedicated `local-desktop-snapshot-capability`; GUI sessions and requests carrying `Origin` cannot publish desktop state. A capability's bounded raw body is verified against its authenticated digest before JSON parsing or storage. Badge state is cosmetic and a failed poll degrades silently. | | Logs | `src/server/management/logs-usage-routes.ts` — `GET /api/logs`, `GET /api/claude/inbound-debug`, and `GET /api/debug/injection-logs` join the debug streams described above. | diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index e07ab88928f..1160ef8e6b0 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -399,7 +399,7 @@ The credits module beside `src/codex/quota-types.ts` retains validated WHAM cred ## Low-quota protection -`src/codex/quota.ts` sends accepted usage observations through `src/codex/low-quota-observer.ts`; credits-only updates never replay carried usage into protection, and pool WHAM/header observations reach the policy only after raw percentages pass validation and the captured credential writer is still live; clamped display bars and responses from retired credentials cannot authorize a pause. The pool-account-only [configuration policy](../config.md#codex-pool-low-quota-protection) pauses live selection immediately, defers a bounded config save, and deduplicates account/window notices. Manual resume is respected across every qualifying window already active for that account until recovery or a new reset episode. Native main keeps its separate per-window hard lock. Each server registration owns a bounded status ledger; its authenticated management route exposes only its own account ids. The default alert is a log line plus a `logged` event, with no OS notification. A pool response that could not capture a writer at dispatch (credential already replaced, or the store unreadable) fails closed the same way: its snapshot still refreshes the display cache, but it never authorizes a policy pause. Writer-free legacy and login paths are explicitly distinct from that pool contract and keep their existing observation behavior. +`src/codex/quota.ts` sends accepted usage observations through `src/codex/low-quota-observer.ts`; credits-only updates never replay carried usage into protection, and pool WHAM/header observations reach the policy only after raw percentages pass validation and the captured credential writer is still live; clamped display bars and responses from retired credentials cannot authorize a pause. The pool-account-only [configuration policy](../config.md#codex-pool-low-quota-protection) pauses live selection immediately, defers a bounded config save, and deduplicates account/window notices. Manual resume is respected across every qualifying window already active for that account until recovery or a new reset episode. Native main keeps its separate per-window hard lock. Each server registration owns a bounded status ledger; its authenticated management route exposes only its own account ids. The default alert is a log line plus a `logged` event; the opt-in `actions.desktop` sink in `src/codex/low-quota-desktop-notifier.ts` hands the notice, labelled only by the account's pseudonymous log label, to the host's OS notifier. A pool response that could not capture a writer at dispatch (credential already replaced, or the store unreadable) fails closed the same way: its snapshot still refreshes the display cache, but it never authorizes a policy pause. Writer-free legacy and login paths are explicitly distinct from that pool contract and keep their existing observation behavior. ## Bounded pool quota observations diff --git a/structure/runtime.md b/structure/runtime.md index 35542264e3a..e5eaa4c3741 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -168,7 +168,7 @@ their own files. `src/grok/inject.ts` holds the shared canonical config-write lo Cross-home discovery in `src/cli/cross-home-owner.ts` checks concrete loopback address availability through `src/cli/loopback-family.ts` before probing each family. A synchronous ephemeral bind is immediately closed; only address/family-unavailable errors exclude that address. This prevents a disabled IPv6 loopback's off-host `::1` connect timeout from blocking startup. Other bind errors retain discovery, and a timeout on an available or unknown family still produces an indeterminate owner verdict. Live owners on either available family retain identity and attestation checks. Startup catalog sync and native restore apply the [retired-native policy](catalog.md#shared-catalog). Foreground exit cleanup in `src/cli/index.ts` delegates restoration through the existing sibling/recycle teardown gates; approved external URLs follow [custom-provider consent](codex-provider-routing.md), while unapproved provider configuration and history remain unchanged. Guarded tests isolate the owner registry used by cross-home discovery beneath their selected home, including children without a registry override; production retains the shared locator. See [test state isolation](ops/test-sandbox-cleanup.md). -`src/server/background-lifecycle.ts` registers low-quota protection for each live server, flushes pending saves before releasing its owner, binds its own event ledger to management requests, and unregisters it on failed startup or stop; with no pending save, owner release remains synchronous. The protection writer loads config persistence only when a threshold schedules a save; see [configuration](config.md#codex-pool-low-quota-protection). Codex quota processing has shared and Reserve scopes; retired model evidence is suppressed as +`src/server/background-lifecycle.ts` registers low-quota protection for each live server with the OS desktop sink resolver, flushes pending saves before releasing its owner, binds its own event ledger to management requests, and unregisters it on failed startup or stop; with no pending save, owner release remains synchronous. The protection writer loads config persistence only when a threshold schedules a save; see [configuration](config.md#codex-pool-low-quota-protection). Codex quota processing has shared and Reserve scopes; retired model evidence is suppressed as described in [OpenAI quota ownership](providers/openai-tiers.md#public-provider-contract). `ocx start` refuses a duplicate PID, starts the proxy, writes `~/.opencodex/ocx.pid` and diff --git a/tests/codex-integration/low-quota-desktop-notifier.test.ts b/tests/codex-integration/low-quota-desktop-notifier.test.ts new file mode 100644 index 00000000000..89199bf2350 --- /dev/null +++ b/tests/codex-integration/low-quota-desktop-notifier.test.ts @@ -0,0 +1,161 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + formatLowQuotaNotice, + lowQuotaNotifierArgv, + resolveLowQuotaDesktopSink, + type LowQuotaNotice, + type LowQuotaNotifierIo, +} from "../../src/codex/low-quota-desktop-notifier"; + +const NOTICE: LowQuotaNotice = { accountLabel: "pabc123", window: "weekly", percentUsed: 85, threshold: 80, resetAt: 1_800_000_000_000 }; +const always = () => true; + +type FakeChild = { argv: string[]; killed: number; exit(code: number | null): void; fail(error: Error): void }; + +function fakeSpawn(): { io: Pick; children: FakeChild[] } { + const children: FakeChild[] = []; + return { + children, + io: { + spawn(argv) { + let resolveExit!: (code: number | null) => void; + let rejectExit!: (error: Error) => void; + const exited = new Promise((resolve, reject) => { resolveExit = resolve; rejectExit = reject; }); + const child: FakeChild = { argv, killed: 0, exit: code => resolveExit(code), fail: error => rejectExit(error) }; + children.push(child); + return { exited, kill: () => { child.killed++; } }; + }, + }, + }; +} + +const previousGuard = process.env.OCX_TEST_HOME_GUARD; +afterEach(() => { + if (previousGuard === undefined) delete process.env.OCX_TEST_HOME_GUARD; + else process.env.OCX_TEST_HOME_GUARD = previousGuard; +}); + +describe("low-quota desktop notifier", () => { + test("formats a pseudonymous, bounded message with reset context", () => { + expect(formatLowQuotaNotice(NOTICE, () => "Mon 10:00")).toEqual({ + title: "OpenCodex quota", + body: "Account pabc123: weekly quota at 85% (threshold 80%), resets Mon 10:00.", + }); + expect(formatLowQuotaNotice({ ...NOTICE, window: "short", resetAt: null }).body) + .toBe("Account pabc123: 5-hour quota at 85% (threshold 80%), reset time unknown."); + }); + + test("macOS passes notice text as osascript argv, never as script source", () => { + const hostile = 'x" & do shell script "touch /tmp/pwned" & "'; + const argv = lowQuotaNotifierArgv("darwin", always, "OpenCodex quota", hostile)!; + expect(argv).toEqual([ + "/usr/bin/osascript", "-e", "on run argv", "-e", + "display notification (item 2 of argv) with title (item 1 of argv)", "-e", "end run", + "OpenCodex quota", hostile, + ]); + expect(argv.slice(0, 7).some(part => part.includes("pwned"))).toBe(false); + }); + + test("Linux uses a literal notify-send path, stops option parsing and escapes markup", () => { + const seen: string[] = []; + const exists = (path: string) => { seen.push(path); return path === "/usr/local/bin/notify-send"; }; + expect(lowQuotaNotifierArgv("linux", exists, "--title", "a & --urgency=critical")).toEqual([ + "/usr/local/bin/notify-send", "--app-name=OpenCodex", "--", "--title", "<b>a</b> & --urgency=critical", + ]); + expect(seen.every(path => path.startsWith("/"))).toBe(true); + expect(lowQuotaNotifierArgv("linux", () => false, "t", "b")).toBeUndefined(); + }); + + test("unsupported platforms and missing notifiers resolve no sink", () => { + const { io } = fakeSpawn(); + expect(resolveLowQuotaDesktopSink({ ...io, platform: "win32", exists: always })).toBeUndefined(); + expect(resolveLowQuotaDesktopSink({ ...io, platform: "linux", exists: () => false })).toBeUndefined(); + expect(resolveLowQuotaDesktopSink({ ...io, platform: "darwin", exists: () => false })).toBeUndefined(); + }); + + test("an armed test process never reaches a real notifier without an injected spawn", () => { + process.env.OCX_TEST_HOME_GUARD = "1"; + expect(resolveLowQuotaDesktopSink({ platform: "darwin", exists: always })).toBeUndefined(); + const { io } = fakeSpawn(); + expect(resolveLowQuotaDesktopSink({ ...io, platform: "darwin", exists: always })).toBeDefined(); + }); + + test("resolves on exit 0 and rejects on non-zero exit or spawn error", async () => { + const { io, children } = fakeSpawn(); + const sink = resolveLowQuotaDesktopSink({ ...io, platform: "darwin", exists: always })!; + const ok = sink(NOTICE, new AbortController().signal); + children[0]!.exit(0); + await expect(ok).resolves.toBeUndefined(); + expect(children[0]!.argv.at(-1)).toContain("Account pabc123"); + + const bad = sink(NOTICE, new AbortController().signal); + children[1]!.exit(1); + await expect(bad).rejects.toThrow("exited 1"); + + const broken = sink(NOTICE, new AbortController().signal); + children[2]!.fail(new Error("ENOENT")); + await expect(broken).rejects.toThrow("ENOENT"); + + const throwing = resolveLowQuotaDesktopSink({ platform: "darwin", exists: always, spawn: () => { throw new Error("spawn failed"); } })!; + await expect(throwing(NOTICE, new AbortController().signal)).rejects.toThrow("spawn failed"); + }); + + test("times out by killing the child and clearing its listener", async () => { + const { io, children } = fakeSpawn(); + const sink = resolveLowQuotaDesktopSink({ ...io, platform: "darwin", exists: always, timeoutMs: 5 })!; + const controller = new AbortController(); + const removed: string[] = []; + const remove = controller.signal.removeEventListener.bind(controller.signal); + controller.signal.removeEventListener = ((type: string, listener: EventListenerOrEventListenerObject) => { + removed.push(type); + remove(type, listener); + }) as typeof controller.signal.removeEventListener; + await expect(sink(NOTICE, controller.signal)).rejects.toThrow("timed out"); + expect(children[0]!.killed).toBe(1); + expect(removed).toEqual(["abort"]); + controller.abort(); + expect(children[0]!.killed).toBe(1); + children[0]!.exit(0); + }); + + test("abort kills an in-flight child once and a pre-aborted signal never spawns", async () => { + const { io, children } = fakeSpawn(); + const sink = resolveLowQuotaDesktopSink({ ...io, platform: "darwin", exists: always, timeoutMs: 60_000 })!; + const controller = new AbortController(); + const pending = sink(NOTICE, controller.signal); + controller.abort(); + await expect(pending).rejects.toThrow("cancelled"); + expect(children[0]!.killed).toBe(1); + children[0]!.exit(0); + await Promise.resolve(); + expect(children[0]!.killed).toBe(1); + + const aborted = new AbortController(); + aborted.abort(); + await expect(sink(NOTICE, aborted.signal)).rejects.toThrow("cancelled"); + expect(children).toHaveLength(1); + }); +}); + +describe("low-quota desktop notifier cleanup", () => { + const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + for (const outcome of ["exit 0", "exit 1", "spawn error", "abort"] as const) { + test("after " + outcome + " no timer or abort listener can kill the child again", async () => { + const { io, children } = fakeSpawn(); + const sink = resolveLowQuotaDesktopSink({ ...io, platform: "darwin", exists: always, timeoutMs: 10 })!; + const controller = new AbortController(); + const pending = sink(NOTICE, controller.signal).then(() => "resolved", () => "rejected"); + const child = children[0]!; + if (outcome === "exit 0") child.exit(0); + else if (outcome === "exit 1") child.exit(1); + else if (outcome === "spawn error") child.fail(new Error("EPIPE")); + else controller.abort(); + expect(await pending).toBe(outcome === "exit 0" ? "resolved" : "rejected"); + const killsAtSettle = child.killed; + expect(killsAtSettle).toBe(outcome === "abort" ? 1 : 0); + controller.abort(); + await sleep(25); + expect(child.killed).toBe(killsAtSettle); + }); + } +}); diff --git a/tests/codex-integration/low-quota-protection.test.ts b/tests/codex-integration/low-quota-protection.test.ts index c09c98bfd46..894ce230d49 100644 --- a/tests/codex-integration/low-quota-protection.test.ts +++ b/tests/codex-integration/low-quota-protection.test.ts @@ -21,10 +21,12 @@ import { applyAccountQuotaFromUpstreamHeaders, clearAccountQuota, getAccountQuot import type { CodexAccount } from "../../src/types/accounts"; import type { CodexLowQuotaProtectionConfig, OcxConfig } from "../../src/types/config"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { resolveLowQuotaDesktopSink, type LowQuotaNotice, type LowQuotaNoticeSink } from "../../src/codex/low-quota-desktop-notifier"; +import { fallbackCodexAccountLogLabel } from "../../src/codex/account-label"; const ACCOUNT_A = "low-quota-a"; const ACCOUNT_B = "low-quota-b"; -type Notice = { window: "short" | "weekly"; percentUsed: number; threshold: number }; +type Notice = LowQuotaNotice; let home = ""; let previousHome: string | undefined; @@ -56,7 +58,8 @@ function configWith(policy?: CodexLowQuotaProtectionConfig): OcxConfig { function register(config: OcxConfig, deps: { persist?: (next: OcxConfig) => void; - notify?: (notice: Notice) => void | Promise; + notify?: (notice: Notice, signal: AbortSignal) => void | Promise; + desktopSink?: () => LowQuotaNoticeSink | undefined; } = {}): LowQuotaRegistration { const cleanup = registerCodexLowQuotaProtection(config, deps); cleanups.push(cleanup); @@ -127,7 +130,7 @@ describe("low quota protection", () => { observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 10, weeklyResetAt: secondReset }); observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 90, weeklyResetAt: secondReset }); expect(notices).toHaveLength(5); - expect(notices.at(-1)).toEqual({ window: "weekly", percentUsed: 90, threshold: 80 }); + expect(notices.at(-1)).toMatchObject({ window: "weekly", percentUsed: 90, threshold: 80 }); observeCodexLowQuota(ACCOUNT_A, { shortPercent: 95, shortResetAt: firstReset }); observeCodexLowQuota(ACCOUNT_B, { weeklyPercent: 95, weeklyResetAt: firstReset }); expect(notices).toHaveLength(5); @@ -171,7 +174,7 @@ describe("low quota protection", () => { setAccountQuotaFromParsed(ACCOUNT_A, accepted, undefined, undefined, accepted); await registration.flush(); expect(persisted[0]?.pausedCodexAccountIds).toEqual([ACCOUNT_A]); - expect(notices).toEqual([{ window: "weekly", percentUsed: 99, threshold: 80 }]); + expect(notices).toEqual([{ accountLabel: fallbackCodexAccountLogLabel(ACCOUNT_A), window: "weekly", percentUsed: 99, threshold: 80, resetAt: null }]); }); test("stops acting after unregister", () => { @@ -485,13 +488,19 @@ describe("low quota protection", () => { await registration.flush(); }); - test("notice failure retries on a later observation and only then reports delivery", async () => { + test("notice failure retries on an observation after the backoff and only then reports delivery", async () => { const config = configWith(protection({ actions: { pause: false, notify: true } })); let calls = 0; const registration = register(config, { notify: async () => { if (++calls === 1) throw new Error("sink failed"); } }); + const start = Date.now(); + const clock = spyOn(Date, "now").mockReturnValue(start); + cleanups.push(() => clock.mockRestore()); observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 80 }); await Promise.resolve(); expect(registration.listEvents(1)[0]?.status).toBe("failed"); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 81 }); + expect(calls).toBe(1); + clock.mockReturnValue(start + 5 * 60_000 + 1); observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 82 }); await Promise.resolve(); expect(registration.listEvents(1)[0]?.status).toBe("delivered"); @@ -633,3 +642,238 @@ describe("low quota protection", () => { expect(registration.listEvents(1)[0]).toMatchObject({ accountId: ACCOUNT_A, delivery: "pause-save", status: "cancelled" }); }); }); + +type DeferredCall = { notice: Notice; signal: AbortSignal; resolve(): void; reject(error: Error): void }; + +function deferredDesktop(): { sink: () => LowQuotaNoticeSink; calls: DeferredCall[]; resolves: () => number } { + const calls: DeferredCall[] = []; + let resolved = 0; + const sink: LowQuotaNoticeSink = (notice, signal) => new Promise((resolve, reject) => { + calls.push({ notice, signal, resolve, reject }); + }); + return { sink: () => { resolved++; return sink; }, calls, resolves: () => resolved }; +} + +function desktopPolicy(desktop: boolean | undefined, overrides: Partial = {}): CodexLowQuotaProtectionConfig { + return protection({ actions: { pause: false, notify: true, ...(desktop === undefined ? {} : { desktop }) }, ...overrides }); +} + +function noticeStatuses(registration: LowQuotaRegistration): string[] { + return registration.listEvents(100).filter(event => event.delivery === "notice").map(event => event.status).reverse(); +} + +const settle = () => new Promise(resolve => setTimeout(resolve, 0)); + +describe("low quota desktop delivery", () => { + test("without the desktop opt-in the alert stays logged and no notifier is resolved", () => { + const desktop = deferredDesktop(); + for (const flag of [undefined, false]) { + const config = configWith(desktopPolicy(flag)); + const registration = register(config, { desktopSink: desktop.sink }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + expect(noticeStatuses(registration)).toEqual(["logged"]); + registration(); + } + expect(desktop.resolves()).toBe(0); + }); + + test("the desktop opt-in delivers one pseudonymous notice per episode", async () => { + const config = configWith(desktopPolicy(true)); + config.codexAccounts = [{ ...account(ACCOUNT_A), alias: "secret alias", logLabel: "p0a1b2c" }]; + const desktop = deferredDesktop(); + const registration = register(config, { desktopSink: desktop.sink }); + const resetAt = Date.now() + 60_000; + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85, weeklyResetAt: resetAt }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 90, weeklyResetAt: resetAt }); + expect(desktop.calls).toHaveLength(1); + expect(desktop.calls[0]!.notice).toEqual({ accountLabel: "p0a1b2c", window: "weekly", percentUsed: 85, threshold: 80, resetAt }); + const text = JSON.stringify(desktop.calls[0]!.notice); + for (const secret of ["secret alias", "@example.test", ACCOUNT_A]) expect(text).not.toContain(secret); + desktop.calls[0]!.resolve(); + await settle(); + expect(noticeStatuses(registration)).toEqual(["pending", "delivered"]); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 95, weeklyResetAt: resetAt }); + expect(desktop.calls).toHaveLength(1); + }); + + test("an account without a stored log label gets the fallback pseudonym", () => { + const config = configWith(desktopPolicy(true)); + const desktop = deferredDesktop(); + register(config, { desktopSink: desktop.sink }); + observeCodexLowQuota(ACCOUNT_B, { shortPercent: 81 }); + expect(desktop.calls[0]!.notice.accountLabel).toBe(fallbackCodexAccountLogLabel(ACCOUNT_B)); + }); + + test("a failed desktop notice is recorded, logged once, and retried only after the backoff", async () => { + const config = configWith(desktopPolicy(true)); + const desktop = deferredDesktop(); + const registration = register(config, { desktopSink: desktop.sink }); + const start = Date.now(); + const clock = spyOn(Date, "now").mockReturnValue(start); + cleanups.push(() => clock.mockRestore()); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + cleanups.push(() => warn.mockRestore()); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + desktop.calls[0]!.reject(new Error("notifier exited 1")); + await settle(); + expect(noticeStatuses(registration)).toEqual(["pending", "failed"]); + for (const percent of [86, 87, 88]) observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: percent }); + expect(desktop.calls).toHaveLength(1); + clock.mockReturnValue(start + 5 * 60_000 - 1); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 89 }); + expect(desktop.calls).toHaveLength(1); + clock.mockReturnValue(start + 5 * 60_000 + 1); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 90 }); + expect(desktop.calls).toHaveLength(2); + desktop.calls[1]!.resolve(); + await settle(); + expect(noticeStatuses(registration)).toEqual(["pending", "failed", "pending", "delivered"]); + expect(warn.mock.calls.filter(call => String(call[0]).startsWith("[codex-low-quota] weekly quota reached"))).toHaveLength(1); + }); + + test("an unsupported host keeps the opted-in alert logged", () => { + const config = configWith(desktopPolicy(true)); + let resolved = 0; + const registration = register(config, { desktopSink: () => { resolved++; return undefined; } }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + expect(resolved).toBe(1); + expect(noticeStatuses(registration)).toEqual(["logged"]); + }); + + const cancellations: Array<{ name: string; trigger: (config: OcxConfig, registration: LowQuotaRegistration) => void | Promise }> = [ + { name: "a new reset boundary", trigger: () => { observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85, weeklyResetAt: Date.now() + 120_000 }); } }, + { name: "a below-threshold recovery", trigger: () => { observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 10, weeklyResetAt: Date.now() + 60_000 }); } }, + { name: "account removal", trigger: config => { + config.codexAccounts = [account(ACCOUNT_B)]; + observeCodexLowQuota(ACCOUNT_B, { weeklyPercent: 5 }); + } }, + { name: "disabling the policy", trigger: config => { + config.codexPool = { lowQuotaProtection: desktopPolicy(true, { enabled: false }) }; + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + } }, + { name: "turning desktop delivery off", trigger: config => { + config.codexPool = { lowQuotaProtection: desktopPolicy(false) }; + observeCodexLowQuota(ACCOUNT_B, { weeklyPercent: 5 }); + } }, + { name: "shutdown through flush", trigger: async (_config, registration) => { await registration.flush(); } }, + ]; + + for (const { name, trigger } of cancellations) { + test("an in-flight desktop notice is cancelled once by " + name, async () => { + const config = configWith(desktopPolicy(true)); + const desktop = deferredDesktop(); + const registration = register(config, { desktopSink: desktop.sink }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85, weeklyResetAt: Date.now() + 60_000 }); + const first = desktop.calls[0]!; + expect(first.signal.aborted).toBe(false); + await trigger(config, registration); + expect(first.signal.aborted).toBe(true); + first.resolve(); + await settle(); + const statuses = noticeStatuses(registration); + expect(statuses.filter(status => status === "cancelled")).toHaveLength(1); + expect(statuses.slice(0, 2)).toEqual(["pending", "cancelled"]); + expect(statuses).not.toContain("delivered"); + }); + } + + test("turning desktop delivery on and off takes effect without re-registration", async () => { + const config = configWith(desktopPolicy(false)); + const desktop = deferredDesktop(); + const registration = register(config, { desktopSink: desktop.sink }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + expect(desktop.resolves()).toBe(0); + config.codexPool = { lowQuotaProtection: desktopPolicy(true) }; + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 86 }); + expect(desktop.resolves()).toBe(1); + desktop.calls[0]!.resolve(); + await settle(); + config.codexPool = { lowQuotaProtection: desktopPolicy(false) }; + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 87 }); + expect(desktop.resolves()).toBe(1); + expect(noticeStatuses(registration)).toEqual(["logged", "pending", "delivered", "logged"]); + }); + + test("desktop delivery requires notify even while the policy is disabled", () => { + for (const enabled of [true, false]) { + const policy = protection({ enabled, actions: { pause: true, notify: false, desktop: true } }); + const result = validateConfigCandidate({ ...configWith(), codexPool: { lowQuotaProtection: policy } }); + expect(result.ok, String(enabled)).toBe(false); + } + for (const desktop of [true, false]) { + expect(validateConfigCandidate(configWith(protection({ actions: { pause: true, notify: true, desktop } }))).ok).toBe(true); + } + const typo = { ...protection(), actions: { pause: true, notify: true, desktop: "yes" } }; + expect(validateConfigCandidate({ ...configWith(), codexPool: { lowQuotaProtection: typo } } as unknown as OcxConfig).ok).toBe(false); + }); +}); + +describe("low quota desktop delivery through the platform adapter", () => { + type Child = { killed: number; exit(code: number): void }; + function adapter(timeoutMs = 60_000) { + const children: Child[] = []; + const sink = resolveLowQuotaDesktopSink({ + platform: "darwin", exists: () => true, timeoutMs, + spawn: () => { + let done!: (code: number) => void; + const exited = new Promise(resolve => { done = resolve; }); + const child: Child = { killed: 0, exit: code => done(code) }; + children.push(child); + return { exited, kill: () => { child.killed++; } }; + }, + })!; + return { sink: () => sink, children }; + } + + test("a timed-out notifier is recorded as failed", async () => { + const config = configWith(desktopPolicy(true)); + const host = adapter(5); + const registration = register(config, { desktopSink: host.sink }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(host.children[0]!.killed).toBe(1); + expect(noticeStatuses(registration)).toEqual(["pending", "failed"]); + }); + + test("a reset boundary kills the old child once and the replacement is delivered", async () => { + const config = configWith(desktopPolicy(true)); + const host = adapter(); + const registration = register(config, { desktopSink: host.sink }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85, weeklyResetAt: Date.now() + 60_000 }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 86, weeklyResetAt: Date.now() + 120_000 }); + expect(host.children.map(child => child.killed)).toEqual([1, 0]); + host.children[0]!.exit(0); + host.children[1]!.exit(0); + await settle(); + expect(noticeStatuses(registration)).toEqual(["pending", "cancelled", "pending", "delivered"]); + }); + + test("shutdown kills an in-flight child once and records no late delivery", async () => { + const config = configWith(desktopPolicy(true)); + const host = adapter(); + const registration = register(config, { desktopSink: host.sink }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + await registration.flush(); + expect(host.children[0]!.killed).toBe(1); + host.children[0]!.exit(0); + await settle(); + expect(noticeStatuses(registration)).toEqual(["pending", "cancelled"]); + }); + + test("an injected notify sink takes precedence over desktop delivery", async () => { + const config = configWith(desktopPolicy(true)); + const host = adapter(); + let resolved = 0; + const notices: Notice[] = []; + const registration = register(config, { + notify: notice => { notices.push(notice); }, + desktopSink: () => { resolved++; return host.sink(); }, + }); + observeCodexLowQuota(ACCOUNT_A, { weeklyPercent: 85 }); + await settle(); + expect(notices).toHaveLength(1); + expect(resolved).toBe(0); + expect(host.children).toHaveLength(0); + expect(noticeStatuses(registration)).toEqual(["pending", "delivered"]); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index a1a2c037a5b..d980d707589 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -167,7 +167,7 @@ "dsh-account-adapter.test.ts": "adapters", "project-config-warning-snapshot.test.ts": "codex-integration", "codex-quota-auto-refresh-generation.test.ts": "codex-integration", - "low-quota-protection.test.ts": "codex-integration", "provider-antigravity-quota-retry.test.ts": "providers", + "low-quota-desktop-notifier.test.ts": "codex-integration", "low-quota-protection.test.ts": "codex-integration", "provider-antigravity-quota-retry.test.ts": "providers", "responses-compaction-recovery.test.ts": "responses", "responses-canonical-nonstream.test.ts": "responses", "compaction-recovery-settings.test.ts": "config", "responses-compaction-recovery-policy.test.ts": "responses", "bun-runtime-preflight.test.ts": "lib",