From 57f00ef5e13aa7b9323e905206142ec2ca4c570f Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:52:49 +0900 Subject: [PATCH 1/2] fix(service): require stable numeric health sentinel endpoints Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../docs/fr/reference/configuration/server.md | 2 +- .../docs/ja/reference/configuration/server.md | 2 +- .../docs/ko/reference/configuration/server.md | 2 +- .../docs/reference/configuration/server.md | 2 +- .../docs/ru/reference/configuration/server.md | 2 +- .../docs/tr/reference/configuration/server.md | 2 +- .../zh-cn/reference/configuration/server.md | 2 +- .../zh-tw/reference/configuration/server.md | 2 +- src/service/health-sentinel-main.ts | 22 +++++- src/service/health-sentinel-probe.ts | 25 +++++-- structure/ops/service-health-recovery.md | 1 + .../service-health-sentinel-runner.test.ts | 67 +++++++++++++++++++ 12 files changed, 113 insertions(+), 18 deletions(-) diff --git a/docs-site/src/content/docs/fr/reference/configuration/server.md b/docs-site/src/content/docs/fr/reference/configuration/server.md index 398f3b43303..20e8336a1f8 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/server.md +++ b/docs-site/src/content/docs/fr/reference/configuration/server.md @@ -23,7 +23,7 @@ exécute des fonctionnalités d'assistance autour des demandes du fournisseur. | `storageCleanupPolicy?` | `StorageCleanupPolicy` | désactivé | Politique facultative de nettoyage des sessions archivées. Elle n'est jamais activée implicitement. | | `appOwnedMemoryBudgetMb?` | `number` | `256` | Plafond en Mio pour les journaux, caches, objets binaires et charges utiles de continuation évincables qui appartiennent à l'application. Plage : 64–4096 ; il ne s'agit pas d'un plafond RSS. | | `metricsExport.enabled?` | `boolean` | `false` | Active les métriques de requêtes agrégées, locales au processus, sur `GET /api/metrics` authentifié. Redémarrage requis ; lorsque désactivé, le chemin renvoie 404 et aucune activité d'export n'est démarrée. | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Récupération optionnelle du service en arrière-plan : un petit processus sentinelle interroge `/healthz` et, lorsque le proxy reste vivant mais ne répond plus pendant environ une minute de sondes consécutives, le termine pour que le gestionnaire de service (Planificateur de tâches, launchd, systemd, WinSW) le redémarre. Seul le processus enfant du service géré en lance une ; les requêtes détenues par le processus bloqué ne sont pas rejouées. Au plus trois récupérations par heure. Redémarrage requis. | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Récupération optionnelle du service en arrière-plan : un petit processus sentinelle interroge `/healthz` et, lorsque le proxy reste vivant mais ne répond plus pendant environ une minute de sondes consécutives, le termine pour que le gestionnaire de service (Planificateur de tâches, launchd, systemd, WinSW) le redémarre. Seul le processus enfant du service géré en lance une ; les requêtes détenues par le processus bloqué ne sont pas rejouées. Au plus trois récupérations par heure. Redémarrage requis. La récupération automatique exige une adresse de liaison numérique (ou le mappage localhost/générique par défaut) ; les noms de liaison DNS ne sont pas surveillés. Une adresse ou un port enregistré modifié arrête la surveillance. | | `codexAutoStart?` | `boolean` | `true` | Autorise le lanceur intermédiaire Codex à exécuter `ocx ensure` avant de démarrer Codex. Avec la valeur false, cette vérification ne fait rien. | | `codexShimAutoRestore?` | `boolean` | `true` | Restaure le lanceur intermédiaire installé après son remplacement par une mise à jour externe de Codex terminée. Désactivation par variable d'environnement : `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`. | | `syncResumeHistory?` | `boolean` | `true` | Compatibilité historique Codex App réversible. Les métadonnées originales sont sauvegardées et restaurées par `ocx stop` / `ocx restore`. | diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index 96a82614d78..f7f1665247e 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -23,7 +23,7 @@ description: リスナー、リモート アクセス、アドミッション | `storageCleanupPolicy?` | `StorageCleanupPolicy` |無効 |アーカイブされたセッションのクリーンアップ ポリシーをオプトインします。暗黙的に有効になることはありません。 | | `appOwnedMemoryBudgetMb?` | `number` | `256` |排除可能なアプリ所有のログ、キャッシュ、BLOB、および継続ペイロードの MiB の上限。範囲は 64 ～ 4096。 RSSキャップではありません。 | | `metricsExport.enabled?` | `boolean` | `false` | 認証済み `GET /api/metrics` でプロセスローカルの集約リクエストメトリクスを有効にします。再起動が必要です。無効時は 404 となり、エクスポーター処理は開始されません。 | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | バックグラウンドサービス向けのオプトイン復旧。小さな監視プロセスが `/healthz` を確認し、プロキシが生きたまま連続プローブに約 1 分間まったく応答しない場合に終了させ、サービスマネージャー（タスク スケジューラ、launchd、systemd、WinSW）に再起動させます。管理対象サービスの子プロセスでのみ起動し、停止したプロセスが保持していたリクエストは再送しません。1 時間あたり最大 3 回。再起動が必要です。 | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | バックグラウンドサービス向けのオプトイン復旧。小さな監視プロセスが `/healthz` を確認し、プロキシが生きたまま連続プローブに約 1 分間まったく応答しない場合に終了させ、サービスマネージャー（タスク スケジューラ、launchd、systemd、WinSW）に再起動させます。管理対象サービスの子プロセスでのみ起動し、停止したプロセスが保持していたリクエストは再送しません。1 時間あたり最大 3 回。再起動が必要です。自動復旧には数値のバインドアドレス（または既定の localhost/ワイルドカード マッピング）が必要です。DNS バインド名は監視されません。記録されたアドレスまたはポートが変更されると監視が停止します。 | | `codexAutoStart?` | `boolean` | `true` | Codex を起動する前に、Codex シムで `ocx ensure` を実行させます。 False を指定すると、操作が行われないことが保証されます。 | | `codexShimAutoRestore?` | `boolean` | `true` |完了した外部 Codex アップデートによってインストールされたシムが置き換えられた後、インストールされているシムを復元します。環境オプトアウト: `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`。 | | `syncResumeHistory?` | `boolean` | `true` | Codex App 履歴の互換性を元に戻すことができます。元のメタデータは `ocx stop` / `ocx restore` によってバックアップおよび復元されます。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index f6f761b0742..145679df1e4 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -23,7 +23,7 @@ description: 리스너, 원격 접근, admission 키, 타임아웃, 저장소, | `storageCleanupPolicy?` | `StorageCleanupPolicy` | disabled | 선택적으로 활성화하는 보관 세션 정리 정책입니다. 절대 암묵적으로 활성화되지 않습니다. | | `appOwnedMemoryBudgetMb?` | `number` | `256` | 제거 가능한 앱 소유 로그, 캐시, blob, continuation payload에 대한 MiB 단위 상한입니다. 범위는 64–4096이며 RSS 상한은 아닙니다. | | `metricsExport.enabled?` | `boolean` | `false` | 인증된 `GET /api/metrics`에서 프로세스 로컬 집계 요청 메트릭을 활성화합니다. 재시작이 필요하며, 비활성화 상태에서는 404를 반환하고 exporter 동작을 시작하지 않습니다. | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | 백그라운드 서비스용 선택적 복구입니다. 작은 감시 프로세스가 `/healthz`를 확인하고, 프록시가 살아 있지만 연속 프로브에 약 1분 동안 전혀 응답하지 않으면 종료해 서비스 관리자(작업 스케줄러, launchd, systemd, WinSW)가 다시 시작하게 합니다. 관리되는 서비스 자식 프로세스에서만 시작되며, 멈춘 프로세스가 처리 중이던 요청은 재전송하지 않습니다. 시간당 최대 3회. 재시작이 필요합니다. | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | 백그라운드 서비스용 선택적 복구입니다. 작은 감시 프로세스가 `/healthz`를 확인하고, 프록시가 살아 있지만 연속 프로브에 약 1분 동안 전혀 응답하지 않으면 종료해 서비스 관리자(작업 스케줄러, launchd, systemd, WinSW)가 다시 시작하게 합니다. 관리되는 서비스 자식 프로세스에서만 시작되며, 멈춘 프로세스가 처리 중이던 요청은 재전송하지 않습니다. 시간당 최대 3회. 재시작이 필요합니다. 자동 복구에는 숫자 바인드 주소(또는 기본 localhost/와일드카드 매핑)가 필요합니다. DNS 바인드 이름은 모니터링하지 않습니다. 기록된 주소 또는 포트가 변경되면 모니터링이 중지됩니다. | | `codexAutoStart?` | `boolean` | `true` | Codex shim이 Codex를 실행하기 전에 `ocx ensure`를 돌리도록 허용합니다. `false`이면 ensure는 아무 작업도 하지 않습니다. | | `codexShimAutoRestore?` | `boolean` | `true` | 완료된 외부 Codex 업데이트가 설치된 shim을 교체한 뒤 복원합니다. 환경 변수로 끌 수 있습니다: `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`. | | `syncResumeHistory?` | `boolean` | `true` | 되돌릴 수 있는 Codex App history 호환성입니다. 원래 메타데이터는 `ocx stop` / `ocx restore`가 백업하고 복원합니다. | diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index bdf85e12079..e97beb10013 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -29,7 +29,7 @@ runs helper features around provider requests. | `usageLedgerMaxBytes?` | `number` | unset | Opt-in ceiling in bytes for `usage.jsonl`. Absent means the request history grows without limit, which stays the default. See [usage history size](#usage-history-size). | | `appOwnedMemoryBudgetMb?` | `number` | `256` | Cap in MiB for evictable app-owned logs, caches, blobs, and continuation payloads. Range 64–4096; not an RSS cap. | | `metricsExport.enabled?` | `boolean` | `false` | Enable process-local aggregate request metrics at authenticated `GET /api/metrics`. Restart required; disabled mode returns 404 and starts no exporter activity. | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Opt-in recovery for the background service: a small sentinel process probes `/healthz` and, when the proxy stays alive but answers nothing for about a minute of back-to-back probes, terminates it so the service manager (Task Scheduler, launchd, systemd, WinSW) restarts it. Only the managed service child starts one; requests the stuck process held are not replayed. At most three recoveries per hour. Restart required. | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Opt-in recovery for the background service: a small sentinel process probes `/healthz` and, when the proxy stays alive but answers nothing for about a minute of back-to-back probes, terminates it so the service manager (Task Scheduler, launchd, systemd, WinSW) restarts it. Only the managed service child starts one; requests the stuck process held are not replayed. At most three recoveries per hour. Restart required. Automatic recovery requires a numeric bind address (or the default localhost/wildcard mapping); DNS bind names are not monitored. A changed recorded address or port stops monitoring. | | `spend?` | `{ root?: { maxTokens?: number }; identity?: { maxTokens?: number }; pool?: { maxTokens?: number }; retentionDays?: number }` | unset | Durable token ceilings, off unless you write one. Each scope bounds settled spend plus in-flight reservations plus unresolved spend: `root` is one task including its whole fan-out, `identity` is one account across every task it serves, and `pool` is one provider pool. They intersect, so a request is admitted only when all three have room — which is what holds a ceiling against a client that mints a new task id per request. A reservation is the request's whole input plus its enforceable output ceiling, counted as if every cached prefix misses. Observe-only mode still journals, so every server owns the state directory's single-writer lease; an explicit sibling must use a separate `OPENCODEX_HOME`. Spend survives an ordinary process restart when its writes reached the filesystem, but the journal does not promise survival across host power loss because each append is not fsynced. Raising or removing a ceiling changes the allowance for requests started afterward; requests already running keep their starting policy. `maxTokens` must be a positive integer (0 would refuse everything), `retentionDays` is 1–365 and defaults to 7, and an unknown key in this section is rejected rather than ignored. A refusal is a local HTTP 429 carrying `x-opencodex-local-refusal: workflow_spend_exhausted`, and its message names the scope and the ceiling; no provider is contacted. With an applicable ceiling, dispatch is also refused if its token reservation cannot be booked, including full tracking capacity. Requests without an applicable ceiling remain observe-only. | | `codexAutoStart?` | `boolean` | `true` | Let the Codex shim run `ocx ensure` before launching Codex. False makes ensure a no-op. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 83fccc6e222..d66c298b873 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -24,7 +24,7 @@ description: Listener, удалённый доступ, admission key, тайм | `storageCleanupPolicy?` | `StorageCleanupPolicy` | disabled | Opt-in policy очистки архивированных сессий. Никогда не включается неявно. | | `appOwnedMemoryBudgetMb?` | `number` | `256` | Лимит в MiB для eviction-friendly app-owned log'ов, cache'ей, blob'ов и continuation payload'ов. Это не RSS-cap. Диапазон 64–4096. | | `metricsExport.enabled?` | `boolean` | `false` | Включает локальные для процесса агрегированные метрики запросов на аутентифицированном `GET /api/metrics`. Требуется перезапуск; в выключенном состоянии маршрут возвращает 404 и экспортёр не запускает никакой активности. | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Необязательное восстановление фоновой службы: небольшой процесс-наблюдатель опрашивает `/healthz` и, если прокси жив, но около минуты не отвечает на идущие подряд пробы, завершает его, чтобы менеджер службы (Планировщик заданий, launchd, systemd, WinSW) перезапустил его. Запускается только в дочернем процессе управляемой службы; запросы зависшего процесса не повторяются. Не более трёх восстановлений в час. Требуется перезапуск. | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Необязательное восстановление фоновой службы: небольшой процесс-наблюдатель опрашивает `/healthz` и, если прокси жив, но около минуты не отвечает на идущие подряд пробы, завершает его, чтобы менеджер службы (Планировщик заданий, launchd, systemd, WinSW) перезапустил его. Запускается только в дочернем процессе управляемой службы; запросы зависшего процесса не повторяются. Не более трёх восстановлений в час. Требуется перезапуск. Для автоматического восстановления требуется числовой адрес привязки (или сопоставление localhost/подстановочного адреса по умолчанию); DNS-имена привязки не отслеживаются. Изменение записанного адреса или порта останавливает наблюдение. | | `codexAutoStart?` | `boolean` | `true` | Разрешает shim'у Codex запускать `ocx ensure` перед стартом Codex. При false `ensure` становится no-op. | | `codexShimAutoRestore?` | `boolean` | `true` | Восстанавливает установленный shim после завершённого внешнего обновления Codex, которое заменило его. Для отключения через окружение: `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`. | | `syncResumeHistory?` | `boolean` | `true` | Обратимый режим совместимости истории Codex App. Исходные metadata резервируются и восстанавливаются через `ocx stop` / `ocx restore`. | diff --git a/docs-site/src/content/docs/tr/reference/configuration/server.md b/docs-site/src/content/docs/tr/reference/configuration/server.md index 214cd5ae5cd..70f08045e9c 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/server.md +++ b/docs-site/src/content/docs/tr/reference/configuration/server.md @@ -24,7 +24,7 @@ yardımcı özellikleri nasıl çalıştıracağını kontrol eder. | `storageCleanupPolicy?` | `StorageCleanupPolicy` | devre dışı | İsteğe bağlı arşivlenmiş oturum temizleme politikası. Asla örtük olarak etkinleştirilmez. | | `appOwnedMemoryBudgetMb?` | `number` | `256` | Çıkarılabilir uygulamaya ait günlükler, önbellekler, bloblar ve devam yükleri için MiB cinsinden sınır. Aralık 64–4096; bir RSS sınırı değildir. | | `metricsExport.enabled?` | `boolean` | `false` | Kimliği doğrulanmış `GET /api/metrics` üzerinde süreç yerel toplu istek metriklerini etkinleştirir. Yeniden başlatma gerekir; devre dışıyken yol 404 döndürür ve dışa aktarıcı etkinliği başlamaz. | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Arka plan hizmeti için isteğe bağlı kurtarma: küçük bir gözcü süreç `/healthz` adresini yoklar ve proxy canlı kalıp art arda yoklamalara yaklaşık bir dakika hiç yanıt vermezse onu sonlandırır; hizmet yöneticisi (Görev Zamanlayıcı, launchd, systemd, WinSW) yeniden başlatır. Yalnızca yönetilen hizmet alt süreci başlatır; takılan sürecin tuttuğu istekler yeniden oynatılmaz. Saatte en fazla üç kurtarma. Yeniden başlatma gerekir. | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | Arka plan hizmeti için isteğe bağlı kurtarma: küçük bir gözcü süreç `/healthz` adresini yoklar ve proxy canlı kalıp art arda yoklamalara yaklaşık bir dakika hiç yanıt vermezse onu sonlandırır; hizmet yöneticisi (Görev Zamanlayıcı, launchd, systemd, WinSW) yeniden başlatır. Yalnızca yönetilen hizmet alt süreci başlatır; takılan sürecin tuttuğu istekler yeniden oynatılmaz. Saatte en fazla üç kurtarma. Yeniden başlatma gerekir. Otomatik kurtarma sayısal bir bağlantı adresi (veya varsayılan localhost/joker eşleştirme) gerektirir; DNS bağlantı adları izlenmez. Kaydedilen adresin veya bağlantı noktasının değişmesi izlemeyi durdurur. | | `codexAutoStart?` | `boolean` | `true` | Codex dolgusunun Codex'i başlatmadan önce `ocx ensure` çalıştırmasına izin verin. False, ensure'ı bir işlem yapmayan (no-op) hale getirir. | | `codexShimAutoRestore?` | `boolean` | `true` | Tamamlanan harici bir Codex güncellemesi değiştirdikten sonra kurulu bir dolguyu geri yükleyin. Ortam vazgeçmesi: `OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`. | | `syncResumeHistory?` | `boolean` | `true` | Tersine çevrilebilir Codex App geçmişi uyumluluğu. Orijinal meta veriler yedeklenir ve `ocx stop` / `ocx restore` tarafından geri yüklenir. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index 63b95585b2d..821de23773b 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -24,7 +24,7 @@ description: 监听、远程访问、准入密钥、超时、存储、侧车、 | `storageCleanupPolicy?` | `StorageCleanupPolicy` | disabled | 可选启用的归档会话清理策略。不会被隐式启用。 | | `appOwnedMemoryBudgetMb?` | `number` | `256` | 可逐出应用自有日志、缓存、blob 和续传载荷的内存上限，单位 MiB。范围 64–4096；不是 RSS 上限。 | | `metricsExport.enabled?` | `boolean` | `false` | 在经过认证的 `GET /api/metrics` 上启用进程本地的请求聚合指标。需要重启；禁用时该路径返回 404，且不会启动任何导出活动。 | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | 后台服务的可选恢复：一个小型守护进程探测 `/healthz`，当代理进程仍存活但在连续探测中约一分钟毫无响应时将其终止，由服务管理器（任务计划程序、launchd、systemd、WinSW）重新启动。仅受管服务子进程会启动它；卡住进程持有的请求不会重放。每小时最多 3 次。需要重启。 | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | 后台服务的可选恢复：一个小型守护进程探测 `/healthz`，当代理进程仍存活但在连续探测中约一分钟毫无响应时将其终止，由服务管理器（任务计划程序、launchd、systemd、WinSW）重新启动。仅受管服务子进程会启动它；卡住进程持有的请求不会重放。每小时最多 3 次。需要重启。自动恢复需要数字绑定地址（或默认的 localhost/通配符映射）；不监控 DNS 绑定名称。记录的地址或端口发生更改时会停止监控。 | | `codexAutoStart?` | `boolean` | `true` | 允许 Codex shim 在启动 Codex 之前运行 `ocx ensure`。设为 false 会让 ensure 变成无操作。 | | `codexShimAutoRestore?` | `boolean` | `true` | 在完成外部 Codex 更新并覆盖安装的 shim 之后恢复该 shim。环境退出开关：`OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`。 | | `syncResumeHistory?` | `boolean` | `true` | 可逆的 Codex App 历史兼容性。原始元数据会被备份，并由 `ocx stop` / `ocx restore` 恢复。 | diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md index c78905a7b5b..df3f820c695 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md @@ -22,7 +22,7 @@ description: 監聽器、遠端存取、許可金鑰、逾時、儲存、sidecar | `storageCleanupPolicy?` | `StorageCleanupPolicy` | 停用 | 選擇加入的已封存 session 清理政策。永不隱含啟用。 | | `appOwnedMemoryBudgetMb?` | `number` | `256` | 以 MiB 為單位、可被驅逐的 app 擁有日誌、快取、blob 與 continuation payload 上限。範圍 64–4096；非 RSS 上限。 | | `metricsExport.enabled?` | `boolean` | `false` | 在已驗證的 `GET /api/metrics` 啟用程序本機的彙總請求指標。需要重新啟動；停用時路徑回傳 404，且不會啟動任何匯出活動。 | -| `serviceHealthRecovery.enabled?` | `boolean` | `false` | 背景服務的選用復原：一個小型監看程序探測 `/healthz`，當代理程序仍存活但在連續探測中約一分鐘毫無回應時將其終止，交由服務管理員（工作排程器、launchd、systemd、WinSW）重新啟動。僅受管服務子程序會啟動它；卡住程序持有的請求不會重送。每小時最多 3 次。需要重新啟動。 | +| `serviceHealthRecovery.enabled?` | `boolean` | `false` | 背景服務的選用復原：一個小型監看程序探測 `/healthz`，當代理程序仍存活但在連續探測中約一分鐘毫無回應時將其終止，交由服務管理員（工作排程器、launchd、systemd、WinSW）重新啟動。僅受管服務子程序會啟動它；卡住程序持有的請求不會重送。每小時最多 3 次。需要重新啟動。自動復原需要數字繫結位址（或預設的 localhost/萬用字元對應）；不監控 DNS 繫結名稱。記錄的位址或連接埠變更時會停止監控。 | | `codexAutoStart?` | `boolean` | `true` | 讓 Codex shim 在啟動 Codex 前執行 `ocx ensure`。False 使 ensure 為 no-op。 | | `codexShimAutoRestore?` | `boolean` | `true` | 在完成的外部 Codex 更新取代已安裝的 shim 後還原它。環境退出：`OPENCODEX_CODEX_SHIM_AUTO_RESTORE=0`。 | | `syncResumeHistory?` | `boolean` | `true` | 可逆的 Codex App 歷史相容性。原始中繼資料由 `ocx stop` / `ocx restore` 備份並還原。 | diff --git a/src/service/health-sentinel-main.ts b/src/service/health-sentinel-main.ts index 88e1d401d39..4e5b3c0acc3 100644 --- a/src/service/health-sentinel-main.ts +++ b/src/service/health-sentinel-main.ts @@ -38,7 +38,7 @@ import { type HealthProbeOutcome, type HealthSentinelTiming, } from "./health-sentinel-policy"; -import { probeHealth, type HealthProbeTarget } from "./health-sentinel-probe"; +import { probeHealth, probeUrlHost, type HealthProbeTarget } from "./health-sentinel-probe"; export const HEALTH_SENTINEL_ENV = { targetPid: "OCX_HEALTH_SENTINEL_TARGET_PID", @@ -143,6 +143,11 @@ type Attempt = | { kind: "gone" } | { kind: "intent"; entry: RecoveryEntry }; +function matchesProbeTarget(runtime: SentinelRuntimeRecord | null, pid: number, target: HealthProbeTarget): boolean { + return runtime !== null && runtime.pid === pid && !runtime.sibling + && runtime.port === target.port && probeUrlHost(runtime.hostname) === target.hostname; +} + export async function runHealthSentinel(deps: HealthSentinelDeps): Promise { const timing: HealthSentinelTiming = { ...DEFAULT_HEALTH_SENTINEL_TIMING, ...deps.timing }; const standDown = (reason: string): HealthSentinelResult => { @@ -162,12 +167,18 @@ export async function runHealthSentinel(deps: HealthSentinelDeps): Promise; const { pid, port, hostname } = parsed; + if (hostname !== undefined && typeof hostname !== "string") return null; if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0) return null; if (typeof port !== "number" || !Number.isSafeInteger(port) || port <= 0 || port > 65_535) return null; return { pid, port, ...(typeof hostname === "string" ? { hostname } : {}), sibling: parsed.siblingOfPort !== undefined }; diff --git a/src/service/health-sentinel-probe.ts b/src/service/health-sentinel-probe.ts index b1919aa24f5..9f5bbc789f5 100644 --- a/src/service/health-sentinel-probe.ts +++ b/src/service/health-sentinel-probe.ts @@ -7,6 +7,8 @@ * response framing and an absolute deadline. Only the timing decides liveness; the body's `pid` * lets the policy tell the target from another process that took the port. */ +import { isIP } from "node:net"; + import { directLocalHttpFetch } from "../server/direct-local-http"; import type { HealthProbeOutcome } from "./health-sentinel-policy"; @@ -17,12 +19,19 @@ export type HealthProbeIo = { fetch?: (url: string, timeoutMs: number) => Promise; }; -/** The URL host to dial for a recorded listen hostname: wildcard binds answer on loopback. */ -export function probeUrlHost(hostname: string | undefined): string { - const value = (hostname ?? "").trim().replace(/^\[|\]$/g, ""); - if (!value || value === "0.0.0.0" || value === "localhost") return "127.0.0.1"; - if (value === "::") return "[::1]"; - return value.includes(":") ? `[${value}]` : value; +/** Numeric probe endpoint only; unknown DNS destinations cannot authorize recovery. */ +export function probeUrlHost(hostname: string | undefined): string | null { + let value = (hostname ?? "").trim(); + if (value.startsWith("[") || value.endsWith("]")) { + if (!value.startsWith("[") || !value.endsWith("]")) return null; + value = value.slice(1, -1); + if (isIP(value) !== 6) return null; + } + if (!value || value === "0.0.0.0" || /^localhost\.?$/i.test(value)) return "127.0.0.1"; + if (isIP(value) === 4) return value; + if (isIP(value) !== 6 || value.includes("%")) return null; + const host = new URL(`http://[${value}]/`).hostname; + return host === "[::]" ? "[::1]" : host; } /** `pid` from a `/healthz` JSON body, or null when the body does not carry one. */ @@ -39,8 +48,10 @@ export async function probeHealth(target: HealthProbeTarget, timeoutMs: number, const now = io.now ?? Date.now; const fetchFn = io.fetch ?? ((url: string, deadlineMs: number) => directLocalHttpFetch(url, { headers: { accept: "application/json" } }, { timeoutMs: deadlineMs })); const startedAt = now(); + const hostname = probeUrlHost(target.hostname); + if (hostname === null) return { kind: "error", startedAt, endedAt: now() }; try { - const response = await fetchFn(`http://${probeUrlHost(target.hostname)}:${target.port}/healthz`, timeoutMs); + const response = await fetchFn(`http://${hostname}:${target.port}/healthz`, timeoutMs); // The framed body is already in memory; reading it cannot wait on the target. const body = await response.text().catch(() => ""); return { kind: "answered", pid: healthzPid(body), startedAt, endedAt: now() }; diff --git a/structure/ops/service-health-recovery.md b/structure/ops/service-health-recovery.md index b14485a07c8..4db38de387e 100644 --- a/structure/ops/service-health-recovery.md +++ b/structure/ops/service-health-recovery.md @@ -51,6 +51,7 @@ stop; a drain that began during startup prevents the spawn (`didRunDrainStartHoo (wall-clock lag monitor, so sleep counts) voids the streak. `src/service/health-sentinel-probe.ts` asks `/healthz` through `directLocalHttpFetch`. +Probes accept only numeric IPv4/IPv6 destinations. Default and localhost binds use numeric loopback, as do wildcard binds. Other DNS bind names make the sentinel stand down without probing; the proxy remains available. The first normalized address and port remain pinned for the run, and runtime-record changes stand down both before probing and before termination. ## Binding the kill to the proxy diff --git a/tests/service/service-health-sentinel-runner.test.ts b/tests/service/service-health-sentinel-runner.test.ts index 60e025b6868..b138bb9249a 100644 --- a/tests/service/service-health-sentinel-runner.test.ts +++ b/tests/service/service-health-sentinel-runner.test.ts @@ -206,6 +206,54 @@ describe("health sentinel runner (#6671)", () => { expect(await runHealthSentinel(deps)).toEqual({ outcome: "target_exited" }); expect(terminated(events)).toBe(false); }); + + test("DNS bind names never arm recovery or reach a probe", async () => { + const w = world(["answer", "timeout"]); + w.runtime.hostname = "health.attacker.invalid"; + expect((await runHealthSentinel(w.deps)).outcome).toBe("stood_down"); + expect(w.events.filter(event => event.startsWith("probe:"))).toHaveLength(0); + expect(terminated(w.events)).toBe(false); + expect(w.history).toHaveLength(0); + }); + + test.each(["127.0.0.2", "health.attacker.invalid"])("address change to %s after arming stands down", async hostname => { + const w = world(["answer", "timeout"]); + const probe = w.deps.probe; + w.deps.probe = async (target, timeout) => { + const outcome = await probe(target, timeout); + w.runtime.hostname = hostname; + return outcome; + }; + expect((await runHealthSentinel(w.deps)).outcome).toBe("stood_down"); + expect(w.events.filter(event => event.startsWith("probe:"))).toHaveLength(1); + expect(terminated(w.events)).toBe(false); + expect(w.history).toHaveLength(0); + }); + + test.each(["address", "port"])("%s change during lease admission preserves the process", async changed => { + const w = world(["answer", "timeout"]); + const lease = w.deps.withOwnershipLease; + w.deps.withOwnershipLease = run => { + if (changed === "address") w.runtime.hostname = "127.0.0.2"; + else w.runtime.port++; + return lease(run); + }; + expect((await runHealthSentinel(w.deps)).outcome).toBe("stood_down"); + expect(terminated(w.events)).toBe(false); + expect(w.history).toHaveLength(0); + }); + + test("endpoint change while writing intent is rechecked before termination", async () => { + const w = world(["answer", "timeout"]); + const write = w.deps.writeHistory; + w.deps.writeHistory = entry => { + if (entry.state === "intent") w.runtime.hostname = "127.0.0.2"; + return write(entry); + }; + expect((await runHealthSentinel(w.deps)).outcome).toBe("stood_down"); + expect(terminated(w.events)).toBe(false); + expect(w.history.map(entry => entry.state)).toEqual(["intent", "skipped"]); + }); }); describe("health sentinel state files", () => { @@ -264,6 +312,8 @@ describe("health sentinel state files", () => { expect(readSentinelRuntime(path)?.sibling).toBe(true); writeFileSync(path, JSON.stringify({ pid: -1, port: 4100 })); expect(readSentinelRuntime(path)).toBeNull(); + writeFileSync(path, JSON.stringify({ pid: 12, port: 4100, hostname: 4100 })); + expect(readSentinelRuntime(path)).toBeNull(); }); test("timing override keeps only finite non-negative known fields", () => { @@ -307,4 +357,21 @@ describe("health sentinel probe", () => { expect(probeUrlHost("fd00::1")).toBe("[fd00::1]"); expect(probeUrlHost(undefined)).toBe("127.0.0.1"); }); + + test.each(["health.attacker.invalid", "127.1", "2130706433", "[::1", "::1]", "fe80::1%en0"])("refuses unpinned destination %s before transport", async hostname => { + let calls = 0; + const outcome = await probeHealth({ hostname, port: 4100 }, 100, { + fetch: async () => { calls++; throw Object.assign(new Error("redirected timeout"), { name: "TimeoutError" }); }, + }); + expect(outcome.kind).toBe("error"); + expect(calls).toBe(0); + }); + + test.each([ + [undefined, "127.0.0.1"], ["localhost.", "127.0.0.1"], ["LOCALHOST", "127.0.0.1"], + ["0.0.0.0", "127.0.0.1"], ["127.0.0.2", "127.0.0.2"], ["::", "[::1]"], + ["[::1]", "[::1]"], ["0:0:0:0:0:0:0:0", "[::1]"], ["fd00::1", "[fd00::1]"], + ])("normalizes stable bind %s to %s", (hostname, expected) => { + expect(probeUrlHost(hostname)).toBe(expected); + }); }); From 8d57b33770c6ea9991e47f1374f97d74642a09b8 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:59:57 +0900 Subject: [PATCH 2/2] fix(service): check freshness after the final runtime read Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/service/health-sentinel-main.ts | 7 ++-- structure/ops/service-health-recovery.md | 4 +-- .../service-health-sentinel-runner.test.ts | 36 +++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/service/health-sentinel-main.ts b/src/service/health-sentinel-main.ts index 4e5b3c0acc3..ddcdd18f121 100644 --- a/src/service/health-sentinel-main.ts +++ b/src/service/health-sentinel-main.ts @@ -213,7 +213,10 @@ export async function runHealthSentinel(deps: HealthSentinelDeps): Promise timing.maxEvidenceAgeMs || deps.isTargetGone()) { settle({ ...entry, state: "skipped" }); @@ -222,7 +225,7 @@ export async function runHealthSentinel(deps: HealthSentinelDeps): Promise { } }); + test.each([10 * 60_000, -2 * 60 * 60_000])("final runtime re-read invalidates stale evidence after clock shift %s", async shift => { + const { deps, events, history, advance } = world(["answer", "timeout"]); + const read = deps.readRuntime; + let shifted = false; + deps.readRuntime = () => { + const record = read(); + if (!shifted && history.some(entry => entry.state === "intent")) { + shifted = true; + advance(shift); + } + return record; + }; + expect(await runHealthSentinel(deps)).toEqual({ outcome: "recovered", state: "terminated" }); + expect(history.map(entry => entry.state)).toEqual(["intent", "skipped", "intent", "terminated"]); + const discarded = events.findIndex(event => event.includes("discarding stale evidence")); + const kill = events.findIndex(event => event.startsWith("terminate:")); + expect(discarded).toBeGreaterThan(-1); + expect(kill).toBeGreaterThan(discarded); + const fresh = events.slice(discarded, kill).filter(event => event === "probe:timeout").length; + expect(fresh).toBeGreaterThanOrEqual(3); + expect(fresh).toBeLessThanOrEqual(6); + }); + + test("target exit during final runtime re-read prevents termination", async () => { + const { deps, events, history, exitTarget } = world(["answer", "timeout"]); + const read = deps.readRuntime; + deps.readRuntime = () => { + const record = read(); + if (history.some(entry => entry.state === "intent")) exitTarget(); + return record; + }; + expect(await runHealthSentinel(deps)).toEqual({ outcome: "target_exited" }); + expect(terminated(events)).toBe(false); + expect(history.map(entry => entry.state)).toEqual(["intent", "skipped"]); + }); + test("a healthy or refusing target is never terminated", async () => { for (const kind of ["answer", "refused"] as const) { const { deps, events, exitTarget } = world(["answer", kind]);