feat(codex): keep native-main credentials in memory with one writer and a reliable watcher

devlog/_plan/261014_native_main_memory_credentials/10_native_main_memory_design.md

@@ -0,0 +1,75 @@
+# Native-main in-memory credential store
+
+## Scope and base
+
+This unit implements the in-memory alternative to the hard-link publication guard for native-main `CODEX_HOME/auth.json`. The inspected base is `a63267c13`; the auth implementation files named in the supplied behavior report have no diff from `3e58c2ce3` to this base.
+
+The supplied behavior report is useful for the main refresh and profile-switch paths, but its reader inventory is not exhaustive for this checkout. The source search also found the account-pause reader, Orca import's main-identity check, and direct profile-manager envelope reads. The implementation and tests cover those paths.
+
+## `auth.json` reader and writer inventory at the base
+
+### Effective Codex home credential (`<CODEX_HOME>/auth.json`)
+
+| Source path | Operation at the base | New boundary |
+| --- | --- | --- |
+| `src/codex/main-account.ts` | `readMainAuthJsonCredential()` reads/decodes the file for status, refresh, and reauth. Refresh and reauth each call `atomicWriteFile()`. | Read through the native-main memory store; refresh and reauth publish through its writer. |
+| `src/codex/auth-collision.ts` | `readCodexTokensResult()` uses `readFileSync()` or `readBounded()`; `readCodexTokens()` and `getMainChatgptAccountId()` are memory-facing callers. | Parse the memory snapshot; preserve the result API and failure statuses. |
+| `src/codex/auth-api/main-account-probe.ts` | Reads through `readCodexTokensResult()` both for the initial probe and its bounded current-credential check. | Continue through the memory-backed reader. |
+| `src/codex/auth-api/account-pause-group.ts` | Reads via `readCodexTokensResult()` and separately calls `readBounded()` to retain the auth envelope. | Parse both credential and envelope from the memory snapshot. |
+| `src/codex/account-lifecycle.ts` | `initializeMainAccountPolicyBinding()` reads a supplied auth path through `readCodexTokensResult(..., { bounded: true })`. | Initialize/observe through the store, then bind from memory. |
+| `src/codex/native-profile-manager.ts` | Reads the live envelope for registration, switch, recovery, verification, and doctor. Writes the selected envelope, switch rollback, and recovery rollback with its async atomic writer. | Read live envelopes from memory; send all live auth writes through the central writer while keeping the encrypted journal, boundary hooks, verification, and rollback. Staging `auth.json` files remain separate. |
+| `src/codex/native-profile-store.ts` | Its generic `readNativeEnvelopeResult(path)`/`readNativeEnvelope(path)` helpers perform bounded reads; the manager uses them on the live path. | The manager's live-path default uses the memory snapshot. Generic readers remain for non-live/staged paths and independent store tests. |
+| `src/codex/orca-import.ts` | `prepare()` bounded-reads `<getCodexHome()>/auth.json` to reserve the main identity during an explicit offline import. | Read the memory snapshot for the effective native-main path. Orca-managed homes remain independent source files. |
+| `src/codex/home.ts` | Checks whether `auth.json` exists to decide whether the local Codex home is in use; it does not read credential bytes. | Path discovery remains a filesystem-presence check, outside credential consumption. |
+| `src/cli/doctor.ts` | Reports whether `<CODEX_HOME>/auth.json` exists; it does not read credential bytes. | Diagnostic existence check remains unchanged. |
+| `src/codex/native-main-auth-temp.ts` | Removes only owned stale `auth.json.ocx.*.tmp` residues; it does not write or read the canonical credential. | Temporary-file cleanup remains separate from the canonical single writer. |
+
+### Other `auth.json` paths found by the whole-`src/` search
+
+| Source path | Operation at the base | Path and relationship |
+| --- | --- | --- |
+| `src/oauth/store.ts` | Reads and writes provider OAuth credentials and keeps the one-time migration backup. | `<OPENCODEX_HOME>/auth.json`, a separate provider store. |
+| `src/server/configured-credentials.ts` | Reads provider OAuth credentials when rebuilding configured-credential state. | Same OpenCodex provider store. |
+| `src/providers/devin-provider-merge-migration.ts` | Snapshots the store before moving the retired Devin provider slot and rekeys that slot through the OAuth store API. | Same OpenCodex provider store. |
+| `src/config.ts` | Re-pins the existing secret-file permissions. | Same OpenCodex provider store; it does not read credential bytes. |
+| `src/oauth/local-token-detect.ts` | Reads the installed Grok CLI credential. | A separate Grok CLI credential store. |
+| `src/oauth/meta-muse.ts` | Reads a pointer with no credential secret. | A separate Muse configuration pointer. |
+| `src/oauth/command-code.ts` | Reads CommandCode's local login record. | A separate CommandCode login record. |
+| `src/codex/orca-auth-source.ts`, `src/codex/orca-import.ts` | Reads explicitly managed Orca account-home credentials. The import also read the effective main file directly at base. | UUID-owned Orca homes; the effective-main import check now uses memory. |
+| `src/codex/native-profile-manager.ts` | Reads a staged login and securely removes its staged auth file. | Instance-local plaintext staging home, not the live CODEX_HOME file. |
+| `src/codex/home.ts`, `src/cli/doctor.ts` | Test CODEX_HOME auth path presence; doctor also checks the OpenCodex provider-store path and parent-directory writeability. | Discovery and diagnostics only; neither reads token bytes. |
+| `src/codex/native-main-auth-temp.ts` | Removes only owned stale `auth.json.ocx.*.tmp` residues. | Same-directory temporary write artifacts, not the canonical credential. |
+
+`src/server/responses/core-auth.ts` and several other files mention auth.json only in comments; they do not open or write it.
+
+## Requirements and implementation map
+
+| # | Owner requirement | Implementation and proof |
+| --- | --- | --- |
+| 1 | One writer for effective native-main auth, for refresh, device reauth, profile switch, rollback, and recovery. | `src/codex/native-main-credentials.ts` owns temp-file/rename publication; callers use its APIs. A source guard rejects other effective-path readers/writers. |
+| 2 | OpenCodex readers use memory, never file bytes. | The store owns the raw memory snapshot; `main-account.ts`, `auth-collision.ts`, account-pause, lifecycle binding, profile manager, and Orca import read it. A source guard covers the effective path. |
+| 3 | Reliable observing: directory watcher, short stat-first timer with conditional hashing, plus pre-refresh and pre-write observation; no handle keeps the process alive and shutdown closes both. | The store watches the CODEX_HOME directory, checks on the configured interval, hashes only changed stat signatures, and shares one observer across triggers. Tests simulate a dropped event and verify the timer path. |
+| 4 | Adopt valid Codex content; preserve memory through empty, partial, malformed, or unreadable states and never overwrite a hiccup. | The observer accepts usable token envelopes or an API-key envelope, adopts any valid external change regardless of token expiry or account fields, and retains its last good snapshot on hiccups. |
+| 5 | Treat deletion as logout only after grace; return a restored valid file within grace; do not refresh/write during grace; permanently clear memory after confirmed deletion. | `NATIVE_MAIN_AUTH_DELETE_GRACE_MS` is a named ten-minute duration to cover the interactive browser-login gap. Store state and tests cover grace, restore, expiry, and no write-back. |
+| 6 | Preserve safe publication: temp file + rename, re-observe immediately before rename, and adopt a valid external file on conflict. | The central writer wraps existing atomic writers with a final shared observation and compares an immutable expected file identity/digest and staged-candidate generation before rename; any observed valid external change supersedes the pending candidate. |
+| 7 | Publish promptly after refresh and retry non-conflict write failures without debounce; retain a successful grant in memory. | Refresh calls the writer in the same claimed operation; transient I/O failures retain the grant and schedule an unref'd retry starting at 50 ms, exponentially backed off to one second. Invalid, unreadable, and grace-period missing files retain the rotated grant without replacing the hiccup. |
+| 8 | Keep the CODEX_HOME exclusive claim around cross-process refresh and write coordination. | Refresh and reauth retain their existing outer claim; native-profile API and startup transactions hold the claim around the central writer; the retry path reacquires the same claim. |
+
+## Tests-first baseline
+
+Before implementation, the new memory-store regression file was run against unchanged base `a63267c13`:
+
+```text
+bun test tests/codex-integration/native-main-credential-memory.test.ts
+0 pass / 12 fail
+```
+
+The failures showed the intended gaps: empty and half-written files lost the last good credential; delete-and-restore had no in-memory grace state; watcher/timer/pre-write observation and confirmed-deletion APIs were absent; a refused refresh publication still raised `MainAuthJsonChangedDuringRefreshError`; and the source guard found direct file reads in `main-account.ts`, `auth-collision.ts`, `account-pause-group.ts`, and `native-profile-manager.ts`. These were the base-red failures; implementation-era and final validation results are recorded in the change report.
+
+## Deletion grace choice
+
+The grace is ten minutes. The relevant upstream behavior deletes `auth.json` before the interactive browser login finishes; a grace measured in seconds would cover a rewrite gap but not a user completing the browser step. Ten minutes covers a normal browser handoff while still making an abandoned login converge to logout in a bounded period. Memory remains usable during that interval, but all refresh and write attempts are blocked. Once confirmed, the cache and pending refresh publication are cleared, so neither an in-memory retry nor a later caller can recreate the deleted file.
+
+## Remaining accepted risk
+
+A Codex in-place write can still land between the final observation and OpenCodex's rename. The observer narrows that interval and preserves valid changes seen before the rename, but only a lock shared with Codex can close it. The upstream Codex writer does not currently take that lock. The hard-link guard prototype remains out of scope.

docs-site/src/content/docs/reference/configuration/server.md

@@ -44,6 +44,7 @@ runs helper features around provider requests.
 | `codexMainAccountHardLockThresholds?` | `{ short?: number; long?: number }` | `{ short: 90, long: 98 }` | Independent 5h and weekly/monthly thresholds. Integers 80–100, with `short <= long`. Invalid persisted fields fall back to defaults; the settings API rejects invalid writes before mutation. |
 | `codexMainAccountHardLockFallback?` | `{ enabled?: boolean; model?: string; reasoningEffort?: string }` | off | Extended hard-lock settings: while the lock blocks and no Codex account can serve the request, qualifying native-Codex-model requests reroute to `model` with `reasoningEffort` instead of failing. `enabled: false` pauses the reroute and keeps the model. An absent section or model keeps the refusal behavior; a usable Pool account still serves first. `reasoningEffort` must be a declared effort; invalid persisted sections degrade to off and the settings API rejects invalid writes before mutation. See [main-account protection](/reference/cli/providers-accounts/#main-account-quota-protection). |
 | `nativeMainRefresh?` | `"codex" \| "opencodex"` | auto | Who refreshes the native main ChatGPT login in `CODEX_HOME`. `"codex"` asks the Codex binary (app-server `getAuthStatus`, Codex 0.130.0 or newer, found through the same runtime resolution as `ocx sync`, including the ChatGPT app bundle) and never writes `auth.json`; this also works when Codex stores its login in the OS keyring. `"opencodex"` keeps OpenCodex's guarded `auth.json` writer. Unset picks `"codex"` when a supported binary is found; without one, both unset and `"codex"` fall back to the guarded writer. `ocx doctor` shows the active mode under Native main refresh. |
+| `nativeMainAuthGraceSeconds?` | `number` | `60` | How long OpenCodex keeps the last good native main login in memory while `auth.json` in `CODEX_HOME` is missing, empty, half-written, or unreadable. The period starts at the first such observation, and a read error does not restart it. A complete file ends it and is used at once, so a new login or an account switch is never delayed. When the period ends without a complete file, the login is cleared from memory; this is how a logout takes effect. An integer from 1 to 86400; an invalid value is ignored with a warning on load and refused on write. Read when the credential store starts: restart the proxy after a change. |
 | `resetCreditAutoRedeem?` | `{ enabled?: boolean; leadTimeMinutes?: number }` | off | Opt-in: redeem the main Codex account's soonest-expiring reset credit `leadTimeMinutes` (1–60, default 10) before it expires. Every attempt re-reads the upstream credit list first and skips when the credit is gone (for example, redeemed by hand); the `redeem_request_id` is journaled in `$OPENCODEX_HOME/reset-credit-auto-redeem.json` before the call so a crash replays the same idempotent request instead of spending a second credit. Servers sharing this configuration directory coordinate reservations and settlements so one process does not replace another's request record. Logs carry a hashed account key only. |
 | `syncResumeHistory?` | `boolean` | `true` | Reversible Codex App history compatibility. Original metadata is backed up and restored by `ocx stop` / `ocx restore`. |
 | `spendPoolAliases?` | `Record<string, string>` | unset | Explicit historical salted pool-alias to canonical provider mappings. Keep this key at the top level, outside `spend`. See [Historical pool continuity](#historical-pool-continuity). |

scripts/test-layout/layout.json

@@ -651,7 +651,7 @@
     "codex-log-guard-processes.test.ts": "codex-integration",
     "codex-log-guard-protection.test.ts": "codex-integration",
     "codex-log-guard-status-zero-write.test.ts": "codex-integration",
-    "codex-main-account-refresh.test.ts": "codex-integration",
+    "codex-main-account-refresh.test.ts": "codex-integration", "native-main-credential-memory.test.ts": "codex-integration", "native-main-credential-literal-guard.test.ts": "codex-integration",
     "codex-main-rotation.test.ts": "codex-integration",
     "codex-management-convergence.test.ts": "codex-integration",
     "codex-metadata-integrity.test.ts": "codex-integration",

src/cli/doctor.ts

@@ -25,6 +25,7 @@ import { readInstalledServiceToken } from "../lib/service-secrets";
 import { PROXY_ENV_KEYS, proxyEnvPresent } from "../lib/proxy-env";
 import { LOCAL_MANAGEMENT_READ_PATHS } from "../lib/local-management-capability";
 import { readCodexTokens } from "../codex/auth-collision";
+import { observeNativeMainAuthFile } from "../codex/native-main-credentials";
 import { withNativeMainSharedClaim } from "../codex/native-main-claim";
 import { probeNativeProfileRecoveryState, resolveNativeProfileContext } from "../codex/native-profile-store";
 import { NativeProfileError } from "../codex/native-profile-types";
@@ -308,12 +309,20 @@ const PROBE_TIMEOUT_MS = 8000;
 
 export type PathRow = { label: string; path: string; exists: boolean };
 
+function nativeMainAuthPathExists(path: string): boolean {
+  const observation = observeNativeMainAuthFile(path);
+  const exists = observation.fileStatus !== "missing";
+  observation.snapshot?.raw.fill(0);
+  observation.raw?.fill(0);
+  return exists;
+}
+
 export function collectPaths(): PathRow[] {
   const codexHome = resolveCodexHomeDirImpl();
   const opencodexHome = getConfigDir();
   return [
     { label: "CODEX_HOME", path: codexHome, exists: existsSync(codexHome) },
-    { label: "CODEX_HOME/auth.json", path: join(codexHome, "auth.json"), exists: existsSync(join(codexHome, "auth.json")) },
+    { label: "CODEX_HOME/auth.json", path: join(codexHome, "auth.json"), exists: nativeMainAuthPathExists(join(codexHome, "auth.json")) },
     { label: "OPENCODEX_HOME", path: opencodexHome, exists: existsSync(opencodexHome) },
     { label: "OPENCODEX_HOME/config.json", path: getConfigPath(), exists: existsSync(getConfigPath()) },
   ];

src/codex/account-lifecycle.ts

@@ -20,6 +20,7 @@ import { clearCodexPoolRefreshFailure } from "./pool-refresh-backoff";
 import { invalidateCodexWebSocketsForAccount } from "./websocket-registry";
 import { clearMainAccountCredentialPresence, clearMainAccountInfoCache, observeMainQuotaCredential, observeMainQuotaIdentity } from "./main-account-cache";
 import { extractAccountId, extractAccountIdClaims } from "../oauth/chatgpt";
+import { observeNativeMainAuthFile } from "./native-main-credentials";
 import { forgetCodexAccountPause } from "./account-pause";
 import { forgetCodexAccountCreditUse } from "./account-credit-use";
 import { setCodexAccountUseRemainingQuota } from "./account-use-remaining";
@@ -101,8 +102,12 @@ export function reconcileMainCodexAccountRuntimeState(): boolean {
  * Only a valid replacement observation or confirmed account transition supersedes that evidence.
  */
 export function initializeMainAccountPolicyBinding(authPath: string): boolean {
-  // Startup observes the pinned owned path inside the exclusive claim: bound the read so a
-  // replaced non-regular or oversized file cannot stall startup inside that claim.
+  // Startup observes the pinned owned path inside the exclusive claim. The observer's bounded
+  // read is the only credential-file read; every policy consumer below uses its memory copy.
+  const observed = observeNativeMainAuthFile(authPath);
+  observed.raw?.fill(0);
+  observed.snapshot?.raw.fill(0);
+  if (!observed.canRefreshOrWrite) return false;
   const result = readCodexTokensResult(authPath, { bounded: true });
   if (result.status !== "ok") return false;
   const { tokens } = result;

src/codex/account-usability.ts

@@ -95,7 +95,7 @@ export function codexAccountUnusableReason(
     }
     const credential = getMainAccountCredentialStatus();
     if (credential.refreshGrantRejected) return "needs_reauth";
-    if (isAccountNeedsReauth(accountId) && !credential.hasRefreshGrant) return "needs_reauth";
+    if (isAccountNeedsReauth(accountId)) return "needs_reauth";
     // Main account: a refresh grant is enough to route; materialization refreshes before I/O.
     const mainLive = options.isMainAccountTokenLive
       ? options.isMainAccountTokenLive()

src/codex/auth-api/account-pause-group.ts

@@ -7,7 +7,8 @@ import { linkedCodexAccountIds } from "../account-identity";
 import { resolveCodexHomeDir } from "../home";
 import { tryAcquireNativeMainProfileClaim } from "../native-main-admission";
 import { withNativeMainSharedClaim } from "../native-main-claim";
-import { MAX_AUTH_BYTES, readBounded, resolveNativeProfileContext } from "../native-profile-store";
+import { resolveNativeProfileContext } from "../native-profile-store";
+import { getNativeMainAuthMemory } from "../native-main-credentials";
 import { NativeProfileError } from "../native-profile-types";
 import { isNativeMainClaimUnavailable, nativeMainProfileBusyResponse } from "./http";
 
@@ -17,8 +18,10 @@ type MainAuth = CodexTokenReadResult | { status: "api-key-only" };
 function readMainAuth(authPath: string): MainAuth {
   const main = readCodexTokensResult(authPath, { bounded: true });
   if (main.status !== "invalid") return main;
+  const observation = getNativeMainAuthMemory(authPath);
+  if (observation.status !== "ok" || !observation.snapshot) return main;
   try {
-    const j: unknown = JSON.parse(readBounded(authPath, MAX_AUTH_BYTES).toString("utf-8"));
+    const j: unknown = JSON.parse(observation.snapshot.raw.toString("utf-8"));
     if (j === null || typeof j !== "object" || Array.isArray(j)) return main;
     const envelope = j as { auth_mode?: unknown; OPENAI_API_KEY?: unknown; tokens?: unknown };
     const apiKeyOnly = envelope.tokens == null
@@ -27,6 +30,9 @@ function readMainAuth(authPath: string): MainAuth {
     return apiKeyOnly ? { status: "api-key-only" } : main;
   } catch {
     return main;
+  } finally {
+    observation.snapshot.raw.fill(0);
+    observation.raw?.fill(0);
   }
 }