fix(ocap-kernel): keep an absent endpoint from killing the run loop

packages/kernel-node-runtime/test/e2e/remote-comms.test.ts

@@ -1467,5 +1467,87 @@ describe.sequential('Remote Communications E2E', () => {
       },
       NETWORK_TIMEOUT,
     );
+
+    it(
+      'schedules a persisted reap from an unsolicited peer BOYD, over real libp2p',
+      async () => {
+        // The transport-layer half of the "one message bricks a kernel" claim:
+        // that a peer the victim has never granted anything reaches
+        // `handleRemoteMessage` at all. The kernel-store and kernel-test layers
+        // are covered elsewhere (`store/methods/gc.test.ts`, and the
+        // `is not bricked by a peer asking it to bring out its dead` integration
+        // test); here nothing is stubbed — two real kernels over a real relay,
+        // and no ocap URL is ever issued or redeemed successfully, so the victim
+        // grants the attacker nothing and holds no prior relationship with it.
+        //
+        // `bringOutYourDead` needs no authority beyond being able to reach the
+        // victim: an inbound message auto-creates the remote it came from
+        // (`RemoteManager.remoteFor`) and schedules a reap against it, in the
+        // persisted reap queue. This test proves that reap lands from a stranger;
+        // that a persisted reap survives shutdown to brick the next boot is what
+        // the kernel-test integration test then pins.
+        const victim = kernel1;
+        const attacker = kernel2;
+        await victim.initRemoteComms({
+          relays: testRelays,
+          ...testBackoffOptions,
+        });
+        await attacker.initRemoteComms({
+          relays: testRelays,
+          ...testBackoffOptions,
+        });
+        const { peerId1: victimPeerId } = await getPeerIds(victim, attacker);
+
+        // A local vat on the attacker, purely to give its run loop something to
+        // crank so the scheduled BOYD is flushed to the wire.
+        await launchVatAndGetURL(attacker, makeRemoteVatConfig('Mallory'));
+        const malloryRef = getVatRootRef(attacker, kernelStore2, 'Mallory');
+
+        // A well-formed ocap URL naming the victim, with a fabricated object id
+        // that cannot decrypt (base58btc of 32 zero bytes). Redeeming it fails —
+        // but `remoteFor` establishes the attacker's handle to the victim before
+        // the redemption is even sent, and the message itself makes the victim
+        // auto-create its own remote for the attacker. No object is exported
+        // either way, so the failure is swallowed: the point is the side effect.
+        const fabricatedOid = 'z11111111111111111111111111111111';
+        const fabricatedURL = `ocap:${fabricatedOid}@${victimPeerId},${testRelays[0]}`;
+        await attacker.redeemOcapURL(fabricatedURL).catch(() => undefined);
+
+        // Let both run loops drain and park before the BOYD is sent, so the
+        // victim's loop cannot eat its own reap: a reap does not wake a parked
+        // loop, and nothing else touches the victim after this point.
+        await waitUntilQuiescent();
+
+        // The attack, in one message: reap the attacker's remotes (one BOYD to
+        // the victim) and crank the attacker so it is delivered.
+        attacker.reapRemotes();
+        for (let i = 0; i < 3; i++) {
+          await attacker.queueMessage(malloryRef, 'ping', []);
+          await waitUntilQuiescent(100);
+        }
+
+        // Read the reap queue from disk after shutting the victim down, so we
+        // see what the victim's own store committed rather than this test's
+        // stale cache — and prove the reap survives the shutdown that carries it
+        // into the next incarnation. `afterEach`'s second `stop()` is harmless:
+        // `stopWithTimeout` swallows the already-closed error.
+        await victim.stop();
+        const victimDb = await makeSQLKernelDatabase({
+          dbFilename: dbFilename1,
+        });
+        const reapQueue = JSON.parse(
+          victimDb.kernelKVStore.get('reapQueue') ?? '[]',
+        );
+        victimDb.close();
+
+        // The victim scheduled a reap for a peer it never granted anything to,
+        // against a remote (`r…`) rather than a local vat.
+        expect(reapQueue).not.toStrictEqual([]);
+        expect(
+          reapQueue.every((endpointId: string) => endpointId.startsWith('r')),
+        ).toBe(true);
+      },
+      NETWORK_TIMEOUT,
+    );
   });
 });

packages/kernel-test/src/remote-comms.test.ts

@@ -3,7 +3,7 @@ import { peerIdFromPrivateKey } from '@libp2p/peer-id';
 import { NodejsPlatformServices } from '@metamask/kernel-node-runtime';
 import type { KernelDatabase } from '@metamask/kernel-store';
 import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs';
-import { fromHex } from '@metamask/kernel-utils';
+import { fromHex, waitUntilQuiescent } from '@metamask/kernel-utils';
 import { makeKernelStore, kunser, Kernel } from '@metamask/ocap-kernel';
 import type {
   KernelStore,
@@ -494,6 +494,95 @@ describe('Remote Communications (Integration Tests)', () => {
       await rm(tempDir, { recursive: true, force: true });
     }
   });
+
+  it('is not bricked by a peer asking it to bring out its dead', async () => {
+    // `bringOutYourDead` is an ordinary arm of the remote protocol: any peer can
+    // send one, unsolicited. The kernel answers by scheduling a reap against the
+    // remote it came from, in the persisted reap queue.
+    //
+    // `scheduleReap` does not wake a parked run loop, so an idle kernel holds
+    // that reap indefinitely — and carries it into its next incarnation, which
+    // starts its run loop inside `Kernel.make`, before an embedder can call
+    // `initRemoteComms` to restore any remote to deliver it to. One message from
+    // a peer is therefore enough to stop a kernel ever booting again, given only
+    // that it restarts at some point.
+    const tempDir = await mkdtemp(join(tmpdir(), 'kernel-test-rc-reap-'));
+    const dbFile = join(tempDir, 'victim.db');
+    try {
+      // Only the victim needs to survive a restart, so only it needs a file.
+      await kernel1.stop();
+      const victimStore = makeKernelStore(
+        await makeSQLKernelDatabase({ dbFilename: dbFile }),
+      );
+      let victim = await makeTestKernel(
+        'victim',
+        await makeSQLKernelDatabase({ dbFilename: dbFile }),
+        directNetwork,
+        true,
+        'kernel1-peer',
+        '01',
+      );
+
+      // One exchange, so each kernel holds a remote for the other.
+      await runTestVats(victim, makeSenderSubclusterConfig('Sender'));
+      const receiver = (await runTestVats(
+        kernel2,
+        makeReceiverSubclusterConfig('Receiver'),
+      )) as BootstrapResult;
+      await victim.queueMessage(
+        victimStore.getRootObject('v1') as KRef,
+        'sendMessage',
+        [receiver.ocapURL, 'hello', ['once']],
+      );
+
+      // The attack, in one message. The peer is given local work purely so its
+      // own loop cranks and sends the request; nothing touches the victim
+      // afterwards, so the victim's loop stays parked and never delivers the
+      // reap it just queued.
+      kernel2.reapRemotes();
+      await kernel2.queueMessage(
+        makeKernelStore(kernelDatabase2).getRootObject('v1') as KRef,
+        'hello',
+        ['probe'],
+      );
+      await waitUntilQuiescent();
+      await victim.stop();
+
+      // Asserted, not assumed: if the victim had cranked it would have eaten its
+      // own reap while the remote still existed, and the rest would prove nothing.
+      const armed = await makeSQLKernelDatabase({ dbFilename: dbFile });
+      expect(
+        JSON.parse(armed.kernelKVStore.get('reapQueue') ?? '[]'),
+      ).not.toStrictEqual([]);
+
+      // Twice, because it is unrecoverable rather than merely fatal: the crank
+      // that dies is rolled back, which puts the reap back on the queue for the
+      // boot after this one.
+      let database = armed;
+      const bootStates = [];
+      for (const boot of [1, 2]) {
+        victim = await makeTestKernel(
+          `victim-boot${boot}`,
+          database,
+          directNetwork,
+          false,
+          'kernel1-peer',
+          '01',
+        );
+        bootStates.push((await victim.getStatus()).runLoop);
+        await victim.stop();
+        database = await makeSQLKernelDatabase({ dbFilename: dbFile });
+      }
+      // Asserted together rather than per boot, so a failure reports both: the
+      // point is that the second is no better than the first.
+      expect(bootStates).toStrictEqual([
+        { state: 'running' },
+        { state: 'running' },
+      ]);
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+    }
+  });
 });
 
 /**

packages/ocap-kernel/CHANGELOG.md

@@ -61,6 +61,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- A `notify`, GC action, or `bringOutYourDead` addressed to an endpoint that is not running is skipped instead of killing the run loop ([#1029](https://github.com/MetaMask/ocap-kernel/pull/1029))
+  - These deliveries looked up their endpoint unguarded, so an endpoint named by persisted state but absent from the running kernel threw `VatNotFoundError` from inside the crank, which killed the run loop permanently. Because the crank was rolled back the item was restored to the queue, so the next boot dequeued it and died too
+  - Reached by a peer's routine remote GC. A kernel answers a peer's `bringOutYourDead` by scheduling a reap against the remote it came from, and the reap queue is persisted. `scheduleReap` does not wake a parked run loop, so on an otherwise idle kernel that reap sits in the queue until something else gives the loop work — and a kernel shut down in the meantime carries it into the next incarnation. That incarnation starts its run loop inside `Kernel.make`, before an embedder can call `initRemoteComms`, which is what restores remote handles; reaps are taken ahead of the run queue, so the loop's first act is to deliver one addressed to a remote that does not exist yet. The kernel is dead before `Kernel.make` returns, and stays dead on every boot after that
+  - A reap is the delivery that reaches this most easily, because nothing filters it: a GC action is dropped by `shouldProcessAction` once the endpoint has no c-list entry, and a `notify` short-circuits on the same check, but a reap carries no kref and is handed back with no liveness check at all. Nothing purges the reap queue when its endpoint goes away
+  - Also reachable while a terminated vat awaits cleanup, which happens one vat per crank, since `deleteVat` takes its config and subcluster membership but leaves its c-lists and reachable flags in place. Not via `terminateSubcluster`, which drains every pending cleanup after each vat it terminates
+  - Unlike a `send`, none of these has a caller to reject; `send` already tolerated a vanished endpoint by rejecting the caller with `ENDPOINT_UNREACHABLE`
+  - A skipped GC action still performs the kernel's own half — clearing the reachable flag, or tearing the c-list entry down — since that does not depend on the endpoint being there to be told. Skipping it would leave a dropped export flagged reachable, so the same action would be derived again on every sweep
+  - Unless the c-list entries have gone in the meantime: an action is selected only while they exist, but the run loop cleans one terminated vat between that selection and the delivery, and cleaning a vat takes its whole c-list. The cleanup has done the kernel's half in that case, and translating the krefs anyway would report an unmapped kref by throwing — out of the crank, killing the run loop exactly as the unguarded lookup did
+  - A skipped `notify` no longer translates the resolution first. Those translations import if needed, which would mint c-list entries and take references in an endpoint that will never be told and so can never release them
+  - An endpoint id that names neither a vat nor a remote still throws, since that is corrupt state rather than an endpoint that has gone away
 - A message delivered to a kernel-owned kref with no registered service now rejects the caller with `ENDPOINT_UNREACHABLE` instead of throwing, which escaped the crank and killed the run loop — turning one unreachable reference into a dead kernel ([#1007](https://github.com/MetaMask/ocap-kernel/pull/1007))
   - Reachable without any kernel bug: an anonymous kernel object hosts something that cannot outlive the process, such as an accepted socket connection, so a vat holding one across a restart or a message to one still queued from the previous incarnation lands here. That surviving reference is exactly what stops the init sweep deleting the object, so its `kernel` owner survives with it
   - Matches what `KernelRouter` already does for a delivery whose endpoint has vanished. A message sent with no result promise has nobody to report to, so it is logged instead