fix(web): bound stale portal signing keys and serialize JWKS refreshes

docs/portal-handoff.md

@@ -156,8 +156,10 @@ skew allowance — so a minute after you stop using it).
 
 **An outage is survivable, but not indefinitely.** If the endpoint is
 unreachable the box keeps using the keys it already cached, so a brief blip
-does not lock anyone out. The cost is that a **withdrawn** key stays usable
-for up to the cache hour. If you need a key dead sooner than that, the
+does not lock anyone out, but never for more than two hours after the last
+successful fetch: past that the box rejects handovers until the endpoint
+answers (Basic authentication still works). The cost is that a **withdrawn**
+key stays usable for up to those two hours. If you need a key dead sooner than that, the
 revocation to reach for is per-account (§6), not the key set.
 
 ## 3. Delivery: POST, never a URL

flake.nix

@@ -2949,6 +2949,27 @@ open(sys.argv[3], "w").write(header + yaml.safe_dump(data, sort_keys=True))' \
               cp log "$out"
             '';
 
+          # Stale-key bound and refresh serialization for the portal JWKS
+          # cache (issue #634), against the assembled daemon.
+          portal-jwks =
+            pkgs.runCommand "agent-box-portal-jwks"
+              {
+                nativeBuildInputs = [ pkgs.python3 ];
+                daemon = ./tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings;
+                tests = ./tests/test-portal-jwks.py;
+              } ''
+              install -d repo/tests/golden/web/payloads/agent-box-settings/bin
+              cp "$daemon" \
+                repo/tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings
+              cp "$tests" repo/tests/test-portal-jwks.py
+              python3 repo/tests/test-portal-jwks.py > log 2>&1 || {
+                cat log
+                exit 1
+              }
+              cat log
+              cp log "$out"
+            '';
+
           # Browser-independent protocol coverage for the resumable receiver
           # (issue #898), against the same assembled daemon that ships. This
           # includes a deliberately severed PATCH request and continuation

modules/src/settings-daemon.py

@@ -284,6 +284,9 @@
 # expires anyway, so this is the ceiling on how long a WITHDRAWN key stays
 # usable rather than how quickly a new one is picked up.
 JWKS_TTL = 3600
+# Hard ceiling on trusting a cached key set, outage or not (issue #634):
+# the TTL plus one hour of grace for a portal that is briefly unreachable.
+JWKS_MAX_STALE = 2 * JWKS_TTL
 # Floor between refetches triggered by an unknown `kid`. Without it the
 # unauthenticated handover route is an outbound-request amplifier: a caller
 # minting tokens with random `kid`s would have the box hammer the portal.
@@ -7218,39 +7221,63 @@ def portal_jwks_keys(kid, force=False):
     lever on the portal this floor exists to deny.
     """
     now = int(time.time())
-    cached = portal_jwks_read_cache() or {}
-    try:
-        fetched = int(cached.get("fetched", 0))
-    except (TypeError, ValueError):
-        fetched = 0
-    try:
-        attempted = int(cached.get("attempted", 0))
-    except (TypeError, ValueError):
-        attempted = 0
-    keys = cached.get("keys") or []
-    if not isinstance(keys, list):
-        keys = []
-    fresh = keys and now - fetched < JWKS_TTL
-    if fresh and not force:
-        return keys
-    if now - attempted < JWKS_REFETCH_FLOOR:
-        # An attempt happened too recently to try again, whether that was
-        # the ordinary refresh or a forced retry. Answer from what we have
-        # rather than making either path a lever on the portal.
-        return keys
+    with portal_jwks_lock():
+        cached = portal_jwks_read_cache() or {}
+        fetched, attempted, keys = portal_jwks_unpack(cached)
+        # Every return below passes through this bound: a key set older
+        # than JWKS_MAX_STALE is no key set, however the refresh went.
+        usable = keys if now - fetched < JWKS_MAX_STALE else []
+        fresh = usable and now - fetched < JWKS_TTL
+        if fresh and not force:
+            return usable
+        if now - attempted < JWKS_REFETCH_FLOOR:
+            # An attempt happened too recently to try again, whether that
+            # was the ordinary refresh or a forced retry. Answer from what
+            # we have rather than making either path a lever on the portal.
+            return usable
+        # Reserve the attempt BEFORE the network call, under the lock, so
+        # concurrent requests see it and do not each start a fetch.
+        portal_jwks_write_cache(
+            {"fetched": fetched, "attempted": now, "keys": keys})
     document = portal_jwks_fetch()
     if document is None:
-        # Fetch failed. Stale keys still beat none: the portal being briefly
-        # unreachable should not lock out a handover signed by a key we
-        # already hold. A withdrawn key outliving its withdrawal by up to
-        # JWKS_TTL is the cost, and it is why that value is not days.
-        # `attempted` still moves, so a persistent outage stays capped at
-        # one fetch per JWKS_REFETCH_FLOOR rather than one per request.
+        # Fetch failed. Stale keys still beat none, but only until
+        # JWKS_MAX_STALE: the portal being briefly unreachable should not
+        # lock out a handover signed by a key we already hold, while a
+        # withdrawn key must not outlive that bound however long the outage.
+        return usable
+    with portal_jwks_lock():
         return portal_jwks_write_cache(
-            {"fetched": fetched, "attempted": now, "keys": keys})["keys"]
-    return portal_jwks_write_cache(
-        {"fetched": now, "attempted": now,
-         "keys": document.get("keys", [])})["keys"]
+            {"fetched": now, "attempted": now,
+             "keys": document.get("keys", [])})["keys"]
+
+
+def portal_jwks_unpack(cached):
+    """(fetched, attempted, keys) from a cache record, tolerating junk."""
+    def seconds(name):
+        try:
+            return int(cached.get(name, 0))
+        except (TypeError, ValueError):
+            return 0
+    keys = cached.get("keys") or []
+    return seconds("fetched"), seconds("attempted"), (
+        keys if isinstance(keys, list) else [])
+
+
+@contextlib.contextmanager
+def portal_jwks_lock():
+    """Serialize cache read-decide-reserve across threads and processes."""
+    with _PORTAL_JWKS_THREAD_LOCK:
+        fd = os.open(portal_jwks_cache_path() + ".lock",
+                     os.O_RDWR | os.O_CREAT, 0o600)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            yield
+        finally:
+            os.close(fd)
+
+
+_PORTAL_JWKS_THREAD_LOCK = threading.Lock()
 
 
 # SubjectPublicKeyInfo prefix for an Ed25519 key: SEQUENCE { SEQUENCE {

tests/golden/web/payloads/agent-box-settings/bin/agent-box-settings

@@ -782,6 +782,9 @@ PORTAL_JWKS_URL = os.environ.get("AGENT_BOX_PORTAL_JWKS_URL", "") or (
 # expires anyway, so this is the ceiling on how long a WITHDRAWN key stays
 # usable rather than how quickly a new one is picked up.
 JWKS_TTL = 3600
+# Hard ceiling on trusting a cached key set, outage or not (issue #634):
+# the TTL plus one hour of grace for a portal that is briefly unreachable.
+JWKS_MAX_STALE = 2 * JWKS_TTL
 # Floor between refetches triggered by an unknown `kid`. Without it the
 # unauthenticated handover route is an outbound-request amplifier: a caller
 # minting tokens with random `kid`s would have the box hammer the portal.
@@ -10313,39 +10316,63 @@ def portal_jwks_keys(kid, force=False):
     lever on the portal this floor exists to deny.
     """
     now = int(time.time())
-    cached = portal_jwks_read_cache() or {}
-    try:
-        fetched = int(cached.get("fetched", 0))
-    except (TypeError, ValueError):
-        fetched = 0
-    try:
-        attempted = int(cached.get("attempted", 0))
-    except (TypeError, ValueError):
-        attempted = 0
-    keys = cached.get("keys") or []
-    if not isinstance(keys, list):
-        keys = []
-    fresh = keys and now - fetched < JWKS_TTL
-    if fresh and not force:
-        return keys
-    if now - attempted < JWKS_REFETCH_FLOOR:
-        # An attempt happened too recently to try again, whether that was
-        # the ordinary refresh or a forced retry. Answer from what we have
-        # rather than making either path a lever on the portal.
-        return keys
+    with portal_jwks_lock():
+        cached = portal_jwks_read_cache() or {}
+        fetched, attempted, keys = portal_jwks_unpack(cached)
+        # Every return below passes through this bound: a key set older
+        # than JWKS_MAX_STALE is no key set, however the refresh went.
+        usable = keys if now - fetched < JWKS_MAX_STALE else []
+        fresh = usable and now - fetched < JWKS_TTL
+        if fresh and not force:
+            return usable
+        if now - attempted < JWKS_REFETCH_FLOOR:
+            # An attempt happened too recently to try again, whether that
+            # was the ordinary refresh or a forced retry. Answer from what
+            # we have rather than making either path a lever on the portal.
+            return usable
+        # Reserve the attempt BEFORE the network call, under the lock, so
+        # concurrent requests see it and do not each start a fetch.
+        portal_jwks_write_cache(
+            {"fetched": fetched, "attempted": now, "keys": keys})
     document = portal_jwks_fetch()
     if document is None:
-        # Fetch failed. Stale keys still beat none: the portal being briefly
-        # unreachable should not lock out a handover signed by a key we
-        # already hold. A withdrawn key outliving its withdrawal by up to
-        # JWKS_TTL is the cost, and it is why that value is not days.
-        # `attempted` still moves, so a persistent outage stays capped at
-        # one fetch per JWKS_REFETCH_FLOOR rather than one per request.
+        # Fetch failed. Stale keys still beat none, but only until
+        # JWKS_MAX_STALE: the portal being briefly unreachable should not
+        # lock out a handover signed by a key we already hold, while a
+        # withdrawn key must not outlive that bound however long the outage.
+        return usable
+    with portal_jwks_lock():
         return portal_jwks_write_cache(
-            {"fetched": fetched, "attempted": now, "keys": keys})["keys"]
-    return portal_jwks_write_cache(
-        {"fetched": now, "attempted": now,
-         "keys": document.get("keys", [])})["keys"]
+            {"fetched": now, "attempted": now,
+             "keys": document.get("keys", [])})["keys"]
+
+
+def portal_jwks_unpack(cached):
+    """(fetched, attempted, keys) from a cache record, tolerating junk."""
+    def seconds(name):
+        try:
+            return int(cached.get(name, 0))
+        except (TypeError, ValueError):
+            return 0
+    keys = cached.get("keys") or []
+    return seconds("fetched"), seconds("attempted"), (
+        keys if isinstance(keys, list) else [])
+
+
+@contextlib.contextmanager
+def portal_jwks_lock():
+    """Serialize cache read-decide-reserve across threads and processes."""
+    with _PORTAL_JWKS_THREAD_LOCK:
+        fd = os.open(portal_jwks_cache_path() + ".lock",
+                     os.O_RDWR | os.O_CREAT, 0o600)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            yield
+        finally:
+            os.close(fd)
+
+
+_PORTAL_JWKS_THREAD_LOCK = threading.Lock()
 
 
 # SubjectPublicKeyInfo prefix for an Ed25519 key: SEQUENCE { SEQUENCE {