feat(cli): nirium verify --gateway repeatable with retry/backoff fallback

packages/cli/CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to the `nirium-cli` package are documented here.
 ### Added
 
 - `nirium anchor` ([#133](https://github.com/nirium-protocol/nirium/issues/133)): anchors a sha-256 hash (`--hash`), a file's sha-256 (`--file`, the file itself is never uploaded), or a small JSON record (`--record`, max 8KB, enforced client-side) to IPFS via an Audit Trail node's `POST /api/audit/log`. Optional `--tx`/`--tag`; no `--network` flag by design - the agent box decides its own network. Prints the CID, content hash, gateway URL, and a `nirium verify` suggestion.
+- `nirium verify --gateway` is now repeatable, tried in order as fallbacks, with `--retries`/`--timeout` ([#135](https://github.com/nirium-protocol/nirium/issues/135)). A `404` retries the *same* gateway with backoff (a fresh pin can take a moment to propagate); any other failure moves straight to the next gateway. When every gateway 404s, that's reported as `availability: "not_yet_available"`, distinct from a real hash mismatch - nothing was fetched to compare in the first place.
 
 ## 3.0.1 - 2026-10-24
 

packages/cli/README.md

@@ -264,13 +264,32 @@ and the signature is still verified over `nirium-audit-v1:<content_sha256>`.
 Signatures are accepted as base64 or hex, raw ed25519 or SEP-53
 (Freighter `signMessage`), like the anchoring node.
 
+`-g, --gateway <url>` is repeatable and tried in order as fallbacks. A
+`404` is retried on the *same* gateway up to `--retries` times with a
+short backoff first - a fresh pin can take a moment to propagate, so a
+404 there doesn't mean the CID is wrong, just not there *yet*. Any other
+failure (a real error, or a timed-out request) moves straight to the
+next gateway instead, since retrying the same one won't help:
+
+```bash
+nirium verify <cid> --gateway https://gateway.pinata.cloud --gateway https://ipfs.io --retries 3
+```
+
 | Option | Purpose | Default |
 |---|---|---|
-| `-g, --gateway <url>` | IPFS gateway to fetch from | `https://gateway.pinata.cloud` |
+| `-g, --gateway <url>` | IPFS gateway to fetch from - repeat for fallbacks | `https://gateway.pinata.cloud` |
+| `--retries <n>` | retries per gateway on a 404, with backoff | `2` |
+| `--timeout <ms>` | per-request timeout | `8000` |
 | `--json` | machine-readable result | off |
 | `--hash <hex>` | expected evidence hash for a hash-only anchor (`sha-256:` prefix optional) | none |
 | `--evidence <file>` | file whose SHA-256 is compared with the anchored hash | none |
 
+If every gateway comes back 404 after exhausting retries, that's
+reported as **NOT YET AVAILABLE** (`availability: "not_yet_available"`
+in `--json`) - distinct from a hash mismatch, because nothing was
+actually fetched to compare. A mix of real errors across gateways is
+reported as a gateway error instead. Both exit non-zero, same as before.
+
 Exits non-zero (and prints which check failed) if the hash doesn't match
 or the signature doesn't verify. A pass also requires that at least one
 of the two checks actually ran and succeeded, a matching `--hash`/

packages/cli/bin/nirium.js

@@ -162,7 +162,9 @@ program
 program
     .command('verify')
     .argument('<cid>', 'IPFS Content Identifier (CID) of the audit document')
-    .option('-g, --gateway <url>', 'IPFS gateway URL', 'https://gateway.pinata.cloud')
+    .option('-g, --gateway <url>', 'IPFS gateway URL - repeatable, tried in order as fallbacks', (val, prev) => [...(prev || []), val], [])
+    .option('--retries <n>', 'Retries per gateway on a 404 (content may not be propagated yet), with backoff', '2')
+    .option('--timeout <ms>', 'Per-request timeout in milliseconds', '8000')
     .option('--json', 'Output result as JSON')
     .option('--hash <hex>', 'Expected sha-256 of the evidence (hex, optional sha-256: prefix), for hash-only anchors')
     .option('--evidence <file>', 'Original evidence file; its sha-256 is compared with the anchored hash')
@@ -177,7 +179,20 @@ program
             console.error(`❌ ${err.message}`);
             process.exit(1);
         }
-        const result = await fetchAndVerifyCid(cid, options.gateway, globalThis.fetch, { expectedHash });
+
+        const retries = Number(options.retries);
+        const timeoutMs = Number(options.timeout);
+        if (!Number.isInteger(retries) || retries < 0) {
+            console.error(`❌ Error: Invalid --retries '${options.retries}': must be a non-negative integer.`);
+            process.exit(1);
+        }
+        if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
+            console.error(`❌ Error: Invalid --timeout '${options.timeout}': must be a positive number of milliseconds.`);
+            process.exit(1);
+        }
+
+        const gateways = options.gateway.length > 0 ? options.gateway : ['https://gateway.pinata.cloud'];
+        const result = await fetchAndVerifyCid(cid, gateways, globalThis.fetch, { expectedHash, retries, timeoutMs });
         if (options.json) {
             console.log(JSON.stringify(result, null, 2));
         } else {
@@ -1616,25 +1631,89 @@ function verifyAuditDocument(doc, cid, { expectedHash } = {}) {
     return { ok, verified: ok, cid, mode, hashMatch: hashStatus === 'match', hashStatus, expectedHash: anchoredHash, computedHash, signatureStatus, signatureScheme, signerKey, statement, declaredStatement, agentId };
 }
 
-async function fetchAndVerifyCid(cid, gatewayUrl = 'https://gateway.pinata.cloud', fetchFn = globalThis.fetch, opts = {}) {
-    const url = `${gatewayUrl.replace(/\/$/, '')}/ipfs/${cid}`;
-    try {
-        const res = await fetchFn(url);
-        if (!res.ok) {
-            return { ok: false, verified: false, cid, hashMatch: false, expectedHash: '', computedHash: '', signatureStatus: 'absent', error: `IPFS gateway returned HTTP ${res.status}` };
+// `gateways` may be a single URL string (back-compat for direct callers)
+// or an array tried in order. On a 404, the SAME gateway is retried up to
+// `retries` times with a flat backoff before moving to the next gateway -
+// a 404 can mean "not propagated to this gateway yet", which a moment's
+// wait often resolves, unlike a real 4xx/5xx or network failure, which
+// moves on immediately since retrying the same gateway won't help.
+// `sleepFn` exists only so tests don't have to sit through real backoff.
+async function fetchAndVerifyCid(cid, gateways = 'https://gateway.pinata.cloud', fetchFn = globalThis.fetch, opts = {}) {
+    const list = Array.isArray(gateways) ? gateways : [gateways];
+    const retries = Number.isInteger(opts.retries) && opts.retries >= 0 ? opts.retries : 2;
+    const timeoutMs = Number.isFinite(opts.timeoutMs) && opts.timeoutMs > 0 ? opts.timeoutMs : 8000;
+    const sleep = opts.sleepFn || ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const BACKOFF_MS = 300;
+
+    const attempts = [];
+    for (const gatewayUrl of list) {
+        const url = `${gatewayUrl.replace(/\/$/, '')}/ipfs/${cid}`;
+        for (let attempt = 0; attempt <= retries; attempt += 1) {
+            if (attempt > 0) await sleep(BACKOFF_MS);
+            try {
+                const res = await fetchFn(url, { signal: AbortSignal.timeout(timeoutMs) });
+                if (res.status === 404) {
+                    attempts.push({ gateway: gatewayUrl, attempt, status: 404 });
+                    continue; // worth retrying THIS gateway: may just not be propagated yet
+                }
+                if (!res.ok) {
+                    attempts.push({ gateway: gatewayUrl, attempt, status: res.status });
+                    break; // a real error from this gateway - try the next one, not more retries here
+                }
+                const doc = await res.json();
+                return { ...verifyAuditDocument(doc, cid, opts), gateway: gatewayUrl, attempts: [...attempts, { gateway: gatewayUrl, attempt, status: 200 }] };
+            } catch (err) {
+                attempts.push({ gateway: gatewayUrl, attempt, error: err?.message || String(err) });
+                break; // network-level failure (unreachable, timeout) - try the next gateway
+            }
         }
-        const doc = await res.json();
-        return verifyAuditDocument(doc, cid, opts);
-    } catch (err) {
-        return { ok: false, verified: false, cid, hashMatch: false, expectedHash: '', computedHash: '', signatureStatus: 'absent', error: `Failed to fetch CID from gateway: ${err?.message || String(err)}` };
     }
+
+    // Every gateway exhausted without a usable (200) response. All-404
+    // means the content plausibly just isn't propagated anywhere yet -
+    // a different situation from a gateway being unreachable or erroring,
+    // which this reports distinctly so a caller doesn't read "not found
+    // anywhere" as "the hash doesn't match" (it was never fetched at all).
+    const allNotFound = attempts.length > 0 && attempts.every((a) => a.status === 404);
+    return {
+        ok: false, verified: false, cid, hashMatch: false, expectedHash: '', computedHash: '', signatureStatus: 'absent',
+        availability: allNotFound ? 'not_yet_available' : 'gateway_error',
+        error: allNotFound
+            ? `CID not found on ${list.length} gateway(s) after ${retries} retr${retries === 1 ? 'y' : 'ies'} each - it may not be propagated yet, not necessarily wrong`
+            : `Could not fetch the CID from any of ${list.length} gateway(s) - see attempts for each failure`,
+        attempts,
+    };
 }
 
 function formatVerifyOutput(result) {
     const lines = [];
     lines.push(`🔍 Nirium Audit Verifier`);
     if (result.cid) lines.push(`CID:            ${result.cid}`);
     lines.push(`--------------------------------------------------`);
+
+    // The document was never fetched at all (every gateway failed), so
+    // there is no hash or signature to report on - printing those sections
+    // would show a false "MISMATCH" for a CID that was simply never
+    // retrieved. `availability` tells apart "not found anywhere yet"
+    // (may just not be propagated) from an actual gateway/network error.
+    if (result.hashStatus === undefined && result.error) {
+        lines.push(result.availability === 'not_yet_available'
+            ? `ℹ NOT YET AVAILABLE: ${result.error}`
+            : `❌ GATEWAY ERROR:     ${result.error}`);
+        if (Array.isArray(result.attempts) && result.attempts.length > 0) {
+            lines.push(`--------------------------------------------------`);
+            lines.push('Attempts:');
+            for (const a of result.attempts) {
+                lines.push(`   ${a.gateway} (try ${a.attempt + 1}): ${a.error ? a.error : `HTTP ${a.status}`}`);
+            }
+        }
+        lines.push(`--------------------------------------------------`);
+        lines.push(result.availability === 'not_yet_available'
+            ? `❌ NOT YET AVAILABLE (not a hash mismatch - try again shortly, or add --gateway for another source)`
+            : `❌ VERIFICATION FAILED (could not reach the content, not a hash mismatch)`);
+        return lines.join('\n');
+    }
+
     if (result.hashStatus === 'unchecked') {
         lines.push(`ℹ HASH:        NOT CHECKED (hash-only anchor: ${result.expectedHash.slice(0, 16)}...)`);
         lines.push(`   Pass --hash <hex> or --evidence <file> to compare it with your original evidence.`);

packages/cli/package.json

@@ -9,7 +9,7 @@
   },
   "scripts": {
     "prepublishOnly": "node --check bin/nirium.js",
-    "test": "node --test test/doctor-seller.test.js test/config-secret.test.js test/serve-api-key.test.js test/verify-hash-only.test.js test/pay-agent-construction.test.js test/pay-status-and-cap.test.js test/status.test.js test/anchor.test.js"
+    "test": "node --test test/doctor-seller.test.js test/config-secret.test.js test/serve-api-key.test.js test/verify-hash-only.test.js test/verify-retry-gateway.test.js test/pay-agent-construction.test.js test/pay-status-and-cap.test.js test/status.test.js test/anchor.test.js"
   },
   "keywords": [
     "stellar",

packages/cli/test/verify-retry-gateway.test.js

@@ -0,0 +1,163 @@
+/**
+ * `nirium verify --gateway` (repeatable, fallback) and `--retries`/
+ * `--timeout` (#135): a 404 is retried on the SAME gateway (content may
+ * just not be propagated there yet); any other failure moves straight
+ * to the next gateway. Everything runs against local HTTP servers, no
+ * real network and no real delay (--retries kept small to keep the
+ * fixed 300ms backoff from adding up).
+ */
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import http from 'node:http';
+import { once } from 'node:events';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import test from 'node:test';
+
+const cliRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const cli = path.join(cliRoot, 'bin/nirium.js');
+
+const CID = 'QmRetryTestFixture';
+const HASH = 'b'.repeat(64);
+const DOC = { content_sha256: HASH };
+
+function makeGateway() {
+    const state = { requests: [], mode: 'always-404' };
+    const server = http.createServer((req, res) => {
+        state.requests.push(req.url);
+        if (state.mode === 'always-404') {
+            res.writeHead(404).end();
+            return;
+        }
+        if (state.mode === 'fail-once-then-ok') {
+            if (state.requests.length === 1) { res.writeHead(404).end(); return; }
+            res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(DOC));
+            return;
+        }
+        if (state.mode === '500') {
+            res.writeHead(500).end();
+            return;
+        }
+        res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(DOC));
+    });
+    return { server, state };
+}
+
+const gatewayA = makeGateway();
+const gatewayB = makeGateway();
+
+let baseA;
+let baseB;
+
+test.before(async () => {
+    gatewayA.server.listen(0, '127.0.0.1');
+    await once(gatewayA.server, 'listening');
+    baseA = `http://127.0.0.1:${gatewayA.server.address().port}`;
+
+    gatewayB.server.listen(0, '127.0.0.1');
+    await once(gatewayB.server, 'listening');
+    baseB = `http://127.0.0.1:${gatewayB.server.address().port}`;
+});
+
+test.after(() => Promise.all([
+    new Promise((resolve, reject) => gatewayA.server.close((err) => (err ? reject(err) : resolve()))),
+    new Promise((resolve, reject) => gatewayB.server.close((err) => (err ? reject(err) : resolve()))),
+]));
+
+test.beforeEach(() => {
+    gatewayA.state.requests = [];
+    gatewayA.state.mode = 'always-404';
+    gatewayB.state.requests = [];
+    gatewayB.state.mode = 'always-404';
+});
+
+function run(args) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(process.execPath, [cli, 'verify', CID, '--json', ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
+        let stdout = '';
+        let stderr = '';
+        const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('timed out')); }, 15_000);
+        child.stdout.on('data', (d) => { stdout += d; });
+        child.stderr.on('data', (d) => { stderr += d; });
+        child.on('close', (code) => { clearTimeout(timer); resolve({ code, stdout, stderr }); });
+        child.on('error', reject);
+    });
+}
+
+test('404-then-200 on retry: same gateway recovers, no fallback needed', async () => {
+    gatewayA.state.mode = 'fail-once-then-ok';
+    const r = await run(['--gateway', baseA, '--retries', '2', '--hash', HASH]);
+    assert.equal(r.code, 0, r.stdout + r.stderr);
+    const json = JSON.parse(r.stdout);
+    assert.equal(json.ok, true);
+    assert.equal(json.gateway, baseA);
+    assert.equal(gatewayA.state.requests.length, 2); // first 404, second succeeds
+});
+
+test('one gateway down (always 404), second gateway alive: falls back and succeeds', async () => {
+    gatewayB.state.mode = 'ok';
+    const r = await run(['--gateway', baseA, '--gateway', baseB, '--retries', '1', '--hash', HASH]);
+    assert.equal(r.code, 0, r.stdout + r.stderr);
+    const json = JSON.parse(r.stdout);
+    assert.equal(json.ok, true);
+    assert.equal(json.gateway, baseB);
+    assert.equal(gatewayA.state.requests.length, 2); // 1 + 1 retry, both 404
+    assert.equal(gatewayB.state.requests.length, 1); // succeeds on the first try
+});
+
+test('all gateways down (404 everywhere): reported as not-yet-available, not a hash mismatch', async () => {
+    const r = await run(['--gateway', baseA, '--gateway', baseB, '--retries', '1']);
+    assert.equal(r.code, 1);
+    const json = JSON.parse(r.stdout);
+    assert.equal(json.ok, false);
+    assert.equal(json.availability, 'not_yet_available');
+    assert.equal(json.hashStatus, undefined);
+    assert.match(json.error, /not found/i);
+    assert.equal(gatewayA.state.requests.length, 2);
+    assert.equal(gatewayB.state.requests.length, 2);
+});
+
+test('a non-404 error (500) on the first gateway moves straight to the next, no retries wasted on it', async () => {
+    gatewayA.state.mode = '500';
+    gatewayB.state.mode = 'ok';
+    const r = await run(['--gateway', baseA, '--gateway', baseB, '--retries', '3', '--hash', HASH]);
+    assert.equal(r.code, 0, r.stdout + r.stderr);
+    const json = JSON.parse(r.stdout);
+    assert.equal(json.ok, true);
+    assert.equal(json.gateway, baseB);
+    assert.equal(gatewayA.state.requests.length, 1); // no retries spent on a real error
+});
+
+test('mixed failures (one 500, one 404) report gateway_error, not not_yet_available', async () => {
+    gatewayA.state.mode = '500';
+    const r = await run(['--gateway', baseA, '--gateway', baseB, '--retries', '1']);
+    assert.equal(r.code, 1);
+    const json = JSON.parse(r.stdout);
+    assert.equal(json.availability, 'gateway_error');
+});
+
+test('human output distinguishes NOT YET AVAILABLE from a hash mismatch', async () => {
+    const r = await new Promise((resolve, reject) => {
+        const child = spawn(process.execPath, [cli, 'verify', CID, '--gateway', baseA, '--retries', '0'], { stdio: ['ignore', 'pipe', 'pipe'] });
+        let stdout = '';
+        const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('timed out')); }, 10_000);
+        child.stdout.on('data', (d) => { stdout += d; });
+        child.on('close', (code) => { clearTimeout(timer); resolve({ code, stdout }); });
+    });
+    assert.equal(r.code, 1);
+    assert.match(r.stdout, /NOT YET AVAILABLE/);
+    assert.doesNotMatch(r.stdout, /MISMATCH/);
+});
+
+
+test('--retries rejects a negative value', async () => {
+    const r = await run(['--gateway', baseA, '--retries', '-1']);
+    assert.equal(r.code, 1);
+    assert.match(r.stderr, /Invalid --retries/);
+});
+
+test('--timeout rejects a non-positive value', async () => {
+    const r = await run(['--gateway', baseA, '--timeout', '0']);
+    assert.equal(r.code, 1);
+    assert.match(r.stderr, /Invalid --timeout/);
+});