From c84bd112a2bb565d41b1195bc8848fe9faded2aa Mon Sep 17 00:00:00 2001 From: emmzy4tech Date: Wed, 21 Oct 2026 17:34:22 +0100 Subject: [PATCH 1/5] feat(backend): add sandboxed wasm execution engine with gas and memory limits (BE-010) Runs untrusted contract bytecode in an isolated worker thread with an empty env, wasm-metering gas instrumentation (1,000,000 gas), a 50MB linear-memory ceiling and a 2s wall-clock backstop. Returns structured per-test results with gas usage. Co-Authored-By: Claude Opus 5.5 --- backend/src/services/wasmExecutionSandbox.ts | 457 +++++++++++++++++++ backend/src/utils/wasmValidator.ts | 231 ++++++++++ backend/tests/wasmExecutionSandbox.test.ts | 298 ++++++++++++ backend/tests/wasmValidator.test.ts | 85 ++++ 4 files changed, 1071 insertions(+) create mode 100644 backend/src/services/wasmExecutionSandbox.ts create mode 100644 backend/src/utils/wasmValidator.ts create mode 100644 backend/tests/wasmExecutionSandbox.test.ts create mode 100644 backend/tests/wasmValidator.test.ts diff --git a/backend/src/services/wasmExecutionSandbox.ts b/backend/src/services/wasmExecutionSandbox.ts new file mode 100644 index 00000000..5d932fa5 --- /dev/null +++ b/backend/src/services/wasmExecutionSandbox.ts @@ -0,0 +1,457 @@ +/** + * Sandboxed WASM Execution Engine — BE-010 + * + * Evaluates untrusted student contract bytecode against a list of test cases + * with hard, non-negotiable resource ceilings: + * + * - 50 MB linear-memory ceiling. Local memories must declare a maximum + * within budget (checked statically by `wasmValidator`); imported + * memories are supplied by the sandbox already capped. `memory.grow` + * past the cap simply returns -1, so a memory bomb degrades into a + * gas-burning loop that the meter then stops. + * - 1,000,000 gas. Bytecode is instrumented with `wasm-metering` before it + * is ever compiled, so every basic block charges the shared budget and + * infinite loops halt deterministically. + * - 2 second wall-clock execution timeout as a backstop. The worker is + * force-terminated, which also covers pathological cases the meter + * cannot see (e.g. a single very expensive host-side operation). + * - Deep recursion is bounded by the worker's own small stack and surfaces + * as a `stack-overflow` result rather than crashing the host. + * + * Isolation: guest code runs inside a dedicated `worker_threads` Worker that + * is given an empty environment (`env: {}`), no inherited exec flags, and a + * capped V8 heap. The only import the instantiated module receives is the + * gas hook — the validator rejects any module that imports a function, + * table or global — so there is no path from guest code to the filesystem, + * network sockets, child processes or environment variables. + * + * Gas consumption is mirrored into a SharedArrayBuffer on every metering + * call, which lets the host report an accurate figure even when it has to + * kill a worker that is stuck past the wall-clock limit. + */ + +import { randomUUID } from 'crypto'; +import os from 'os'; +import { Worker } from 'worker_threads'; +import logger from '../utils/logger.js'; +import { validateWasmModule, type WasmValidationCode } from '../utils/wasmValidator.js'; + +export const SANDBOX_MEMORY_LIMIT_BYTES = 50 * 1024 * 1024; +export const SANDBOX_GAS_LIMIT = 1_000_000; +export const SANDBOX_TIMEOUT_MS = 2_000; +/** Budget for spawning the worker and metering/compiling the module, separate from execution time. */ +export const SANDBOX_STARTUP_TIMEOUT_MS = 10_000; +export const MAX_TEST_CASES = 64; + +export type SandboxTerminationReason = + | 'completed' + | 'gas-exhausted' + | 'timeout' + | 'memory-limit-exceeded' + | 'stack-overflow' + | 'trap' + | 'validation-failed' + | 'internal-error'; + +export type WasmArgument = number | string; +export type WasmValue = number | string | null; + +export interface WasmTestCase { + name: string; + functionName: string; + /** Strings of the form `123` / `-5n` are passed as BigInt (for i64 parameters). */ + args?: WasmArgument[]; + /** When omitted, the test passes as long as the call returns without trapping. */ + expected?: WasmValue; +} + +export type WasmTestStatus = 'passed' | 'failed' | 'error' | 'skipped'; + +export interface WasmTestOutcome { + name: string; + functionName: string; + status: WasmTestStatus; + expected?: WasmValue; + actual: WasmValue; + gasUsed: number; + durationMs: number; + terminationReason?: SandboxTerminationReason; + error?: string; +} + +export interface SandboxRunRequest { + wasmBase64: string; + tests: WasmTestCase[]; + /** Overrides may only tighten the defaults, never loosen them. */ + gasLimit?: number; + memoryLimitBytes?: number; + timeoutMs?: number; +} + +export interface SandboxRunResult { + executionId: string; + status: 'passed' | 'failed' | 'error'; + terminationReason: SandboxTerminationReason; + gasUsed: number; + gasLimit: number; + memoryPeakBytes: number; + memoryLimitBytes: number; + durationMs: number; + tests: WasmTestOutcome[]; + summary: { total: number; passed: number; failed: number; errored: number; skipped: number }; + validationCode?: WasmValidationCode; + error?: string; +} + +type WorkerToHostMessage = + | { type: 'ready' } + | { type: 'test'; index: number; outcome: WasmTestOutcome; memoryBytes: number } + | { type: 'done'; terminationReason: SandboxTerminationReason } + | { type: 'fatal'; terminationReason: SandboxTerminationReason; error: string }; + +/** + * The worker's program. Kept as a source string and spawned with + * `eval: true` so it runs identically under tsx, compiled `dist/` output and + * babel-jest without depending on how the host module was loaded. + */ +const WORKER_SOURCE = ` +const { parentPort, workerData } = require('worker_threads'); +const metering = require('wasm-metering'); + +const { wasm, tests, gasLimit, memoryLimitPages, importsMemory, gasCounter } = workerData; +const gas = new Int32Array(gasCounter); +const OUT_OF_GAS = '__SANDBOX_OUT_OF_GAS__'; +const send = (msg) => parentPort.postMessage(msg); + +const classify = (err, gasExceeded) => { + const message = err && err.message ? String(err.message) : String(err); + if (gasExceeded || message.includes(OUT_OF_GAS)) { + return { reason: 'gas-exhausted', message: 'Execution exceeded the gas budget of ' + gasLimit + ' units.' }; + } + if (err instanceof RangeError && /call stack/i.test(message)) { + return { reason: 'stack-overflow', message: 'Maximum call stack depth exceeded (unbounded recursion).' }; + } + if (err instanceof RangeError && /memory|allocat/i.test(message)) { + return { reason: 'memory-limit-exceeded', message }; + } + return { reason: 'trap', message }; +}; + +const toArg = (value) => { + if (typeof value === 'string' && /^-?\\d+n?$/.test(value)) return BigInt(value.replace(/n$/, '')); + return value; +}; + +const normalize = (value) => { + if (value === undefined || value === null) return null; + if (typeof value === 'bigint') return value.toString(); + if (typeof value === 'number') return value; + return String(value); +}; + +const sameValue = (actual, expected) => { + if (typeof actual === 'number' && typeof expected === 'number') { + return Object.is(actual, expected) || actual === expected; + } + return String(actual) === String(expected); +}; + +let module; +try { + const meteredBytes = metering.meterWASM(Buffer.from(wasm), { meterType: 'i32' }); + module = new WebAssembly.Module(meteredBytes); +} catch (err) { + send({ type: 'fatal', terminationReason: 'validation-failed', error: 'Unable to meter module: ' + (err && err.message ? err.message : String(err)) }); + module = null; +} + +if (module) { + let gasUsed = 0; + let gasExceeded = false; + const usegas = (amount) => { + gasUsed += amount; + Atomics.store(gas, 0, Math.min(gasUsed, 0x7fffffff)); + if (gasUsed > gasLimit) { + gasExceeded = true; + throw new Error(OUT_OF_GAS); + } + }; + + send({ type: 'ready' }); + + let terminationReason = 'completed'; + for (let index = 0; index < tests.length; index++) { + const test = tests[index]; + const gasBefore = gasUsed; + const started = process.hrtime.bigint(); + const outcome = { name: test.name, functionName: test.functionName, actual: null, gasUsed: 0, durationMs: 0 }; + if (test.expected !== undefined) outcome.expected = test.expected; + let memoryBytes = 0; + + try { + // Fresh instance per test: no state leaks between cases, gas is shared. + const imports = { metering: { usegas } }; + let memory = null; + if (importsMemory) { + memory = new WebAssembly.Memory({ initial: 1, maximum: memoryLimitPages }); + imports.env = { memory }; + } + const instance = new WebAssembly.Instance(module, imports); + const exported = instance.exports.memory; + if (!memory && exported instanceof WebAssembly.Memory) memory = exported; + + const fn = instance.exports[test.functionName]; + if (typeof fn !== 'function') { + outcome.status = 'error'; + outcome.terminationReason = 'trap'; + outcome.error = "Exported function '" + test.functionName + "' not found."; + } else { + try { + outcome.actual = normalize(fn(...(test.args || []).map(toArg))); + } finally { + if (memory) memoryBytes = memory.buffer.byteLength; + } + outcome.terminationReason = 'completed'; + outcome.status = test.expected === undefined || sameValue(outcome.actual, test.expected) ? 'passed' : 'failed'; + } + } catch (err) { + const { reason, message } = classify(err, gasExceeded); + outcome.status = 'error'; + outcome.terminationReason = reason; + outcome.error = message; + } + + outcome.gasUsed = Math.min(gasUsed, gasLimit) - Math.min(gasBefore, gasLimit); + outcome.durationMs = Number(process.hrtime.bigint() - started) / 1e6; + send({ type: 'test', index, outcome, memoryBytes }); + + if (gasExceeded) { + terminationReason = 'gas-exhausted'; + break; + } + } + + send({ type: 'done', terminationReason }); +} +`; + +// --------------------------------------------------------------------------- +// Concurrency guard — each sandbox is a full V8 isolate, so cap how many a +// single process runs at once regardless of how many callers there are. +// --------------------------------------------------------------------------- + +const MAX_CONCURRENT_SANDBOXES = Math.max( + 1, + Number(process.env.WASM_SANDBOX_MAX_CONCURRENCY) || Math.min(4, os.availableParallelism?.() ?? 2) +); +let activeSandboxes = 0; +const sandboxWaiters: Array<() => void> = []; + +async function acquireSlot(): Promise { + if (activeSandboxes < MAX_CONCURRENT_SANDBOXES) { + activeSandboxes++; + return; + } + await new Promise((resolve) => sandboxWaiters.push(resolve)); +} + +function releaseSlot(): void { + const next = sandboxWaiters.shift(); + if (next) next(); + else activeSandboxes--; +} + +const clampLimit = (requested: number | undefined, ceiling: number): number => + requested !== undefined && Number.isFinite(requested) && requested > 0 ? Math.min(requested, ceiling) : ceiling; + +function summarize(tests: WasmTestOutcome[]): SandboxRunResult['summary'] { + const count = (status: WasmTestStatus) => tests.filter((t) => t.status === status).length; + return { + total: tests.length, + passed: count('passed'), + failed: count('failed'), + errored: count('error'), + skipped: count('skipped'), + }; +} + +/** @internal Test seam: lets tests substitute the worker program to exercise the host-side backstops. */ +export interface SandboxInternals { + workerSource?: string; +} + +/** + * Runs every test case against the submitted module. Never rejects because of + * guest behaviour — hostile or broken submissions come back as structured + * results with `status: 'error'` and a `terminationReason`. + */ +export async function runWasmEvaluation( + request: SandboxRunRequest, + internals: SandboxInternals = {} +): Promise { + const executionId = randomUUID(); + const gasLimit = clampLimit(request.gasLimit, SANDBOX_GAS_LIMIT); + const memoryLimitBytes = clampLimit(request.memoryLimitBytes, SANDBOX_MEMORY_LIMIT_BYTES); + const timeoutMs = clampLimit(request.timeoutMs, SANDBOX_TIMEOUT_MS); + const start = process.hrtime.bigint(); + const elapsedMs = () => Number(process.hrtime.bigint() - start) / 1e6; + + const tests = request.tests.slice(0, MAX_TEST_CASES); + const pending: WasmTestOutcome[] = tests.map((test) => ({ + name: test.name, + functionName: test.functionName, + status: 'skipped', + ...(test.expected !== undefined ? { expected: test.expected } : {}), + actual: null, + gasUsed: 0, + durationMs: 0, + })); + + const build = ( + terminationReason: SandboxTerminationReason, + gasUsed: number, + memoryPeakBytes: number, + error?: string, + validationCode?: WasmValidationCode + ): SandboxRunResult => { + const summary = summarize(pending); + const ranToCompletion = terminationReason === 'completed'; + const status: SandboxRunResult['status'] = !ranToCompletion + ? 'error' + : summary.total > 0 && summary.passed === summary.total + ? 'passed' + : 'failed'; + return { + executionId, + status, + terminationReason, + gasUsed: Math.min(gasUsed, gasLimit), + gasLimit, + memoryPeakBytes, + memoryLimitBytes, + durationMs: elapsedMs(), + tests: pending, + summary, + ...(validationCode ? { validationCode } : {}), + ...(error ? { error } : {}), + }; + }; + + if (tests.length === 0) { + return build('validation-failed', 0, 0, 'At least one test case is required.'); + } + + const validation = validateWasmModule(request.wasmBase64, { + memoryLimitBytes, + requiredExports: [...new Set(tests.map((t) => t.functionName))], + }); + if (!validation.valid) { + logger.warn('[wasmExecutionSandbox] Rejected submission before execution', { + executionId, + code: validation.issue.code, + }); + return build('validation-failed', 0, 0, validation.issue.message, validation.issue.code); + } + + await acquireSlot(); + try { + return await new Promise((resolve) => { + const gasCounter = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); + const gasView = new Int32Array(gasCounter); + let memoryPeakBytes = 0; + let currentIndex = 0; + let settled = false; + let executionTimer: NodeJS.Timeout | undefined; + + const worker = new Worker(internals.workerSource ?? WORKER_SOURCE, { + eval: true, + env: {}, + execArgv: [], + argv: [], + stdout: true, + stderr: true, + workerData: { + wasm: validation.bytes, + tests, + gasLimit, + memoryLimitPages: validation.memoryLimitPages, + importsMemory: validation.importsMemory, + gasCounter, + }, + resourceLimits: { + // Metering a module expands it into an AST, so the JS heap needs + // some headroom; guest linear memory is capped separately above. + maxOldGenerationSizeMb: 128, + maxYoungGenerationSizeMb: 16, + stackSizeMb: 4, + }, + }); + + const settle = (reason: SandboxTerminationReason, error?: string) => { + if (settled) return; + settled = true; + clearTimeout(startupTimer); + clearTimeout(executionTimer); + worker.removeAllListeners(); + void worker.terminate(); + resolve(build(reason, Atomics.load(gasView, 0), memoryPeakBytes, error)); + }; + + const killCurrent = (reason: SandboxTerminationReason, message: string) => { + const current = pending[currentIndex]; + if (current && current.status === 'skipped') { + current.status = 'error'; + current.terminationReason = reason; + current.error = message; + } + settle(reason, message); + }; + + const startupTimer = setTimeout(() => { + logger.warn('[wasmExecutionSandbox] Worker failed to start in time', { executionId }); + settle('internal-error', `Sandbox did not start within ${SANDBOX_STARTUP_TIMEOUT_MS}ms.`); + }, SANDBOX_STARTUP_TIMEOUT_MS); + + worker.on('message', (msg: WorkerToHostMessage) => { + switch (msg.type) { + case 'ready': + clearTimeout(startupTimer); + executionTimer = setTimeout(() => { + logger.warn('[wasmExecutionSandbox] Wall-clock limit hit — terminating worker', { + executionId, + timeoutMs, + }); + killCurrent('timeout', `Execution exceeded the wall-clock limit of ${timeoutMs}ms.`); + }, timeoutMs); + break; + case 'test': + pending[msg.index] = msg.outcome; + currentIndex = msg.index + 1; + memoryPeakBytes = Math.max(memoryPeakBytes, msg.memoryBytes); + break; + case 'done': + settle(msg.terminationReason); + break; + case 'fatal': + settle(msg.terminationReason, msg.error); + break; + } + }); + + worker.on('error', (err: Error & { code?: string }) => { + if (err.code === 'ERR_WORKER_OUT_OF_MEMORY') { + killCurrent('memory-limit-exceeded', 'Sandbox exceeded its heap allowance.'); + return; + } + logger.error('[wasmExecutionSandbox] Worker crashed', { executionId, error: err.message }); + killCurrent('internal-error', err.message); + }); + + worker.on('exit', (code) => { + if (!settled) killCurrent('internal-error', `Sandbox exited with code ${code} before reporting.`); + }); + }); + } finally { + releaseSlot(); + } +} diff --git a/backend/src/utils/wasmValidator.ts b/backend/src/utils/wasmValidator.ts new file mode 100644 index 00000000..87c5b96c --- /dev/null +++ b/backend/src/utils/wasmValidator.ts @@ -0,0 +1,231 @@ +/** + * WASM Submission Validator — BE-010 + * + * Static, pre-instantiation checks for untrusted student contract bytecode. + * Nothing in here ever compiles-and-runs guest code: it only parses the + * binary (via `wasmModuleInspector`) and asks the engine whether the bytes + * are a structurally valid module. + * + * A module that passes this validator is guaranteed to: + * - be at most `MAX_WASM_MODULE_BYTES` long, + * - import nothing except (optionally) a linear memory that the sandbox + * supplies itself — so guest code has no reachable host functions and + * therefore no filesystem, network, process or environment access, + * - declare a memory maximum that fits inside the sandbox RAM budget, + * - export every function the evaluation harness intends to call. + */ + +import { + inspectWasmModule, + WasmParseError, + WASM_PAGE_SIZE_BYTES, + type WasmInspection, +} from '../simulator/wasmModuleInspector.js'; + +// tsconfig targets lib ES2022 without DOM, which carries no WebAssembly typings; +// declare just the static surface this module uses (the runtime always has it). +declare const WebAssembly: { + validate(bytes: Uint8Array): boolean; + Module: { + new (bytes: Uint8Array): object; + exports(module: object): Array<{ name: string; kind: string }>; + }; +}; + +/** Upper bound on submitted bytecode size; metering a huge module is itself a DoS vector. */ +export const MAX_WASM_MODULE_BYTES = 1024 * 1024; + +/** Import module/field names reserved for the sandbox's gas hook. Guests must not declare them. */ +export const METERING_IMPORT_MODULE = 'metering'; + +export type WasmValidationCode = + | 'empty-module' + | 'module-too-large' + | 'invalid-encoding' + | 'invalid-module' + | 'host-import-forbidden' + | 'memory-unbounded' + | 'memory-limit-exceeded' + | 'unsupported-feature' + | 'missing-export'; + +export interface WasmValidationIssue { + code: WasmValidationCode; + message: string; +} + +export interface WasmValidationSuccess { + valid: true; + bytes: Uint8Array; + inspection: WasmInspection; + /** True when the module imports its memory and the sandbox must provide a capped one. */ + importsMemory: boolean; + /** Page ceiling the sandbox will enforce for this module. */ + memoryLimitPages: number; + exportedFunctions: string[]; +} + +export interface WasmValidationFailure { + valid: false; + issue: WasmValidationIssue; +} + +export type WasmValidationResult = WasmValidationSuccess | WasmValidationFailure; + +export interface WasmValidationOptions { + memoryLimitBytes: number; + /** Exported function names the caller intends to invoke. */ + requiredExports?: string[]; + maxModuleBytes?: number; +} + +const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/; + +const MEMORY_SECTION_ID = 5; +/** Exception-handling proposal: would let guest code catch the out-of-gas trap and keep spinning. */ +const TAG_SECTION_ID = 13; + +function readVarUint(bytes: Uint8Array, offset: number): { value: number; next: number } { + let value = 0; + let shift = 0; + let pos = offset; + for (;;) { + if (pos >= bytes.length) throw new Error('unexpected end of stream'); + const byte = bytes[pos++]!; + value += (byte & 0x7f) * 2 ** shift; + if ((byte & 0x80) === 0) return { value, next: pos }; + shift += 7; + if (shift > 35) throw new Error('varuint too long'); + } +} + +/** + * Counts locally-defined memories and detects post-MVP sections the inspector + * deliberately ignores. Assumes the header was already checked by the inspector. + */ +function scanSections(bytes: Uint8Array): { localMemories: number; hasTagSection: boolean } { + let localMemories = 0; + let hasTagSection = false; + let pos = 8; + while (pos < bytes.length) { + const id = bytes[pos++]!; + const size = readVarUint(bytes, pos); + const end = size.next + size.value; + if (id === MEMORY_SECTION_ID) localMemories += readVarUint(bytes, size.next).value; + if (id === TAG_SECTION_ID) hasTagSection = true; + pos = end; + } + return { localMemories, hasTagSection }; +} + +const fail = (code: WasmValidationCode, message: string): WasmValidationFailure => ({ + valid: false, + issue: { code, message }, +}); + +/** Strictly decodes base64 — `Buffer.from` silently drops invalid characters, which we don't want. */ +export function decodeWasmBase64(wasmBase64: string): Uint8Array | null { + const normalized = wasmBase64.replace(/\s+/g, ''); + if (normalized.length === 0 || normalized.length % 4 !== 0 || !BASE64_PATTERN.test(normalized)) { + return null; + } + return new Uint8Array(Buffer.from(normalized, 'base64')); +} + +export function validateWasmModule( + input: Uint8Array | string, + options: WasmValidationOptions +): WasmValidationResult { + const maxBytes = options.maxModuleBytes ?? MAX_WASM_MODULE_BYTES; + + if (typeof input === 'string' && input.length > Math.ceil((maxBytes * 4) / 3) + 4) { + return fail('module-too-large', `Module exceeds the ${maxBytes}-byte size limit.`); + } + + const bytes = typeof input === 'string' ? decodeWasmBase64(input) : input; + if (bytes === null) { + return fail('invalid-encoding', 'wasmBase64 is not valid base64.'); + } + if (bytes.byteLength === 0) { + return fail('empty-module', 'Module is empty.'); + } + if (bytes.byteLength > maxBytes) { + return fail('module-too-large', `Module is ${bytes.byteLength} bytes; the limit is ${maxBytes}.`); + } + + let inspection: WasmInspection; + try { + inspection = inspectWasmModule(bytes); + } catch (err) { + const message = err instanceof WasmParseError ? err.message : String(err); + return fail('invalid-module', message); + } + + // Validation only — this does not compile or execute anything. + if (!WebAssembly.validate(bytes)) { + return fail('invalid-module', 'Module failed WebAssembly structural validation.'); + } + + const sections = scanSections(bytes); + const importedMemories = inspection.imports.filter((imp) => imp.kind === 'mem').length; + if (sections.localMemories + importedMemories > 1) { + return fail('unsupported-feature', 'Modules may declare at most one linear memory.'); + } + if (sections.hasTagSection) { + return fail('unsupported-feature', 'The exception-handling proposal (tag section) is not permitted.'); + } + + let importsMemory = false; + for (const imp of inspection.imports) { + if (imp.kind === 'mem') { + importsMemory = true; + continue; + } + const reserved = imp.module === METERING_IMPORT_MODULE ? ' (the metering namespace is reserved)' : ''; + return fail( + 'host-import-forbidden', + `Module imports '${imp.module}.${imp.name}' (${imp.kind})${reserved}. ` + + 'The sandbox exposes no host functions, tables or globals to guest code.' + ); + } + + const memoryLimitPages = Math.max(1, Math.floor(options.memoryLimitBytes / WASM_PAGE_SIZE_BYTES)); + + if (inspection.memory && !inspection.memory.imported) { + // A locally-defined memory's ceiling is baked into the binary, so it must + // already be within budget — we cannot clamp it after the fact. + if (inspection.memory.maxPages === null) { + return fail( + 'memory-unbounded', + 'Module defines a memory without a declared maximum; unbounded growth is not permitted.' + ); + } + if (inspection.memory.maxPages > memoryLimitPages) { + return fail( + 'memory-limit-exceeded', + `Declared memory maximum of ${inspection.memory.maxPages} pages ` + + `(${inspection.memory.maxPages * WASM_PAGE_SIZE_BYTES} bytes) exceeds the sandbox limit of ` + + `${memoryLimitPages} pages (${options.memoryLimitBytes} bytes).` + ); + } + } + + if (inspection.memory && inspection.memory.minPages > memoryLimitPages) { + return fail( + 'memory-limit-exceeded', + `Initial memory of ${inspection.memory.minPages} pages exceeds the sandbox limit of ${memoryLimitPages} pages.` + ); + } + + const exportedFunctions = WebAssembly.Module.exports(new WebAssembly.Module(bytes)) + .filter((exp) => exp.kind === 'function') + .map((exp) => exp.name); + + for (const name of options.requiredExports ?? []) { + if (!exportedFunctions.includes(name)) { + return fail('missing-export', `Module does not export a function named '${name}'.`); + } + } + + return { valid: true, bytes, inspection, importsMemory, memoryLimitPages, exportedFunctions }; +} diff --git a/backend/tests/wasmExecutionSandbox.test.ts b/backend/tests/wasmExecutionSandbox.test.ts new file mode 100644 index 00000000..8269144e --- /dev/null +++ b/backend/tests/wasmExecutionSandbox.test.ts @@ -0,0 +1,298 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { + runWasmEvaluation, + SANDBOX_GAS_LIMIT, + SANDBOX_MEMORY_LIMIT_BYTES, + SANDBOX_TIMEOUT_MS, +} from '../src/services/wasmExecutionSandbox.js'; +import { compileWatToBase64, WAT_ADD, WAT_DISALLOWED_IMPORT, WAT_INFINITE_LOOP, WAT_TRAP } from './fixtures/wasmFixtures.js'; + +/** + * BE-010: sandboxed execution with gas metering, a 50MB memory ceiling and a + * 2s wall-clock limit. These tests run real worker threads. + */ + +const spawnedWorkerOptions: Array> = []; + +jest.mock('worker_threads', () => { + const actual = jest.requireActual('worker_threads') as typeof import('worker_threads'); + class RecordingWorker extends actual.Worker { + constructor(source: string, options: Record) { + spawnedWorkerOptions.push(options); + super(source, options); + } + } + return { ...actual, Worker: RecordingWorker }; +}); + +jest.setTimeout(30_000); + +const WAT_RECURSION = `(module + (func $down (export "down") (param i32) (result i32) + (call $down (i32.add (local.get 0) (i32.const 1)))))`; + +/** 800 pages == exactly 50MB. */ +const WAT_GROW = `(module + (memory (export "memory") 1 800) + (func (export "grow") (param i32) (result i32) + (memory.grow (local.get 0))))`; + +const WAT_MEMORY_BOMB = `(module + (memory (export "memory") 1 800) + (func (export "bomb") (result i32) + (loop $again + (drop (memory.grow (i32.const 64))) + (br $again)) + (i32.const 0)))`; + +/** 816 pages == 51MB — one page budget over the ceiling. */ +const WAT_MEMORY_OVER_50MB = `(module + (memory (export "memory") 1 816) + (func (export "run") (result i32) (i32.const 0)))`; + +const WAT_IMPORTED_MEMORY = `(module + (import "env" "memory" (memory 1)) + (func (export "pages") (result i32) + (drop (memory.grow (i32.const 100000))) + (memory.size)))`; + +const WAT_WASI = `(module + (import "wasi_snapshot_preview1" "fd_write" (func $fd_write (param i32 i32 i32 i32) (result i32))) + (func (export "run") (result i32) (i32.const 0)))`; + +const WAT_I64 = `(module + (func (export "mul") (param i64 i64) (result i64) + (i64.mul (local.get 0) (local.get 1))))`; + +describe('wasmExecutionSandbox.runWasmEvaluation', () => { + it('enforces the spec limits: 50MB memory, 1,000,000 gas, 2s timeout', () => { + expect(SANDBOX_MEMORY_LIMIT_BYTES).toBe(50 * 1024 * 1024); + expect(SANDBOX_GAS_LIMIT).toBe(1_000_000); + expect(SANDBOX_TIMEOUT_MS).toBe(2_000); + }); + + it('returns structured per-test outputs and accurate gas usage', async () => { + const wasmBase64 = await compileWatToBase64(WAT_ADD); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [ + { name: 'adds small numbers', functionName: 'add', args: [2, 3], expected: 5 }, + { name: 'wrong expectation', functionName: 'add', args: [2, 2], expected: 5 }, + { name: 'no expectation', functionName: 'add', args: [1, 1] }, + ], + }); + + expect(result.terminationReason).toBe('completed'); + expect(result.status).toBe('failed'); + expect(result.summary).toEqual({ total: 3, passed: 2, failed: 1, errored: 0, skipped: 0 }); + expect(result.tests.map((t) => t.status)).toEqual(['passed', 'failed', 'passed']); + expect(result.tests[1]).toMatchObject({ actual: 4, expected: 5 }); + + // Gas is per test, and the total is exactly the sum of the parts. + for (const t of result.tests) expect(t.gasUsed).toBeGreaterThan(0); + expect(result.gasUsed).toBe(result.tests.reduce((sum, t) => sum + t.gasUsed, 0)); + expect(result.gasLimit).toBe(SANDBOX_GAS_LIMIT); + }); + + it('reports overall status "passed" only when every test passes', async () => { + const wasmBase64 = await compileWatToBase64(WAT_ADD); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [{ name: 'ok', functionName: 'add', args: [40, 2], expected: 42 }], + }); + expect(result.status).toBe('passed'); + }); + + it('halts an infinite loop at the gas limit and skips the remaining tests', async () => { + const wasmBase64 = await compileWatToBase64(WAT_INFINITE_LOOP); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [ + { name: 'spins forever', functionName: 'run' }, + { name: 'never reached', functionName: 'run' }, + ], + }); + + expect(result.status).toBe('error'); + expect(result.terminationReason).toBe('gas-exhausted'); + expect(result.gasUsed).toBe(SANDBOX_GAS_LIMIT); + expect(result.tests[0]).toMatchObject({ status: 'error', terminationReason: 'gas-exhausted' }); + expect(result.tests[1]?.status).toBe('skipped'); + expect(result.durationMs).toBeLessThan(SANDBOX_TIMEOUT_MS + 1_000); + }); + + it('halts unbounded recursion without crashing the host', async () => { + const wasmBase64 = await compileWatToBase64(WAT_RECURSION); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [{ name: 'recurses', functionName: 'down', args: [0] }], + }); + + expect(result.status).toBe('error'); + expect(['gas-exhausted', 'stack-overflow']).toContain(result.tests[0]?.terminationReason); + }); + + it('caps linear memory at 50MB: growth past the ceiling fails inside the guest', async () => { + const wasmBase64 = await compileWatToBase64(WAT_GROW); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [ + // From 1 page to the 800-page ceiling: succeeds, returns old size. + { name: 'grow to ceiling', functionName: 'grow', args: [799], expected: 1 }, + // Fresh instance: asking for 1000 pages is refused with -1. + { name: 'grow past ceiling', functionName: 'grow', args: [1000], expected: -1 }, + ], + }); + + expect(result.status).toBe('passed'); + expect(result.memoryPeakBytes).toBe(SANDBOX_MEMORY_LIMIT_BYTES); + expect(result.memoryPeakBytes).toBeLessThanOrEqual(result.memoryLimitBytes); + }); + + it('defuses a memory bomb: repeated growth stays under the cap and is halted by gas', async () => { + const wasmBase64 = await compileWatToBase64(WAT_MEMORY_BOMB); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [{ name: 'bomb', functionName: 'bomb' }], + }); + + expect(result.terminationReason).toBe('gas-exhausted'); + expect(result.memoryPeakBytes).toBeLessThanOrEqual(SANDBOX_MEMORY_LIMIT_BYTES); + }); + + it('supplies a capped memory to modules that import one', async () => { + const wasmBase64 = await compileWatToBase64(WAT_IMPORTED_MEMORY); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [{ name: 'pages', functionName: 'pages', expected: 1 }], + }); + + // memory.grow(100000) was refused, so the memory is still 1 page. + expect(result.status).toBe('passed'); + }); + + it('rejects modules whose declared memory exceeds 50MB before running anything', async () => { + const wasmBase64 = await compileWatToBase64(WAT_MEMORY_OVER_50MB); + const result = await runWasmEvaluation({ wasmBase64, tests: [{ name: 't', functionName: 'run' }] }); + + expect(result.terminationReason).toBe('validation-failed'); + expect(result.validationCode).toBe('memory-limit-exceeded'); + expect(result.gasUsed).toBe(0); + expect(result.tests[0]?.status).toBe('skipped'); + }); + + it('terminates a stuck worker at the wall-clock limit and reports the gas it had consumed', async () => { + // Under metering no legal guest outlives 1,000,000 gas, so the timeout is a + // backstop for faults the meter can't see. Simulate one: a worker that + // signals ready, records some gas, then hangs. + const stuckWorker = ` + const { parentPort, workerData } = require('worker_threads'); + Atomics.store(new Int32Array(workerData.gasCounter), 0, 4321); + parentPort.postMessage({ type: 'ready' }); + for (;;) {} + `; + const wasmBase64 = await compileWatToBase64(WAT_ADD); + const result = await runWasmEvaluation( + { wasmBase64, timeoutMs: 200, tests: [{ name: 'hangs', functionName: 'add', args: [1, 2] }] }, + { workerSource: stuckWorker } + ); + + expect(result.terminationReason).toBe('timeout'); + expect(result.tests[0]).toMatchObject({ status: 'error', terminationReason: 'timeout' }); + expect(result.gasUsed).toBe(4321); + expect(result.durationMs).toBeGreaterThanOrEqual(200); + expect(result.durationMs).toBeLessThan(2_000); + }); + + it('uses the 2s execution limit by default', async () => { + const stuckWorker = ` + const { parentPort } = require('worker_threads'); + parentPort.postMessage({ type: 'ready' }); + for (;;) {} + `; + const wasmBase64 = await compileWatToBase64(WAT_ADD); + const result = await runWasmEvaluation( + { wasmBase64, tests: [{ name: 'hangs', functionName: 'add', args: [1, 2] }] }, + { workerSource: stuckWorker } + ); + + expect(result.terminationReason).toBe('timeout'); + expect(result.error).toContain(`${SANDBOX_TIMEOUT_MS}ms`); + expect(result.durationMs).toBeGreaterThanOrEqual(SANDBOX_TIMEOUT_MS); + }); + + it('never lets callers loosen the limits', async () => { + const wasmBase64 = await compileWatToBase64(WAT_ADD); + const result = await runWasmEvaluation({ + wasmBase64, + gasLimit: 10 * SANDBOX_GAS_LIMIT, + memoryLimitBytes: 10 * SANDBOX_MEMORY_LIMIT_BYTES, + timeoutMs: 60_000, + tests: [{ name: 'ok', functionName: 'add', args: [1, 2], expected: 3 }], + }); + + expect(result.gasLimit).toBe(SANDBOX_GAS_LIMIT); + expect(result.memoryLimitBytes).toBe(SANDBOX_MEMORY_LIMIT_BYTES); + }); + + it('reports a trap as a per-test error and keeps running later tests', async () => { + const trap = await compileWatToBase64(WAT_TRAP); + const result = await runWasmEvaluation({ + wasmBase64: trap, + tests: [ + { name: 'traps', functionName: 'run' }, + { name: 'traps again', functionName: 'run' }, + ], + }); + + expect(result.terminationReason).toBe('completed'); + expect(result.status).toBe('failed'); + expect(result.tests.every((t) => t.status === 'error' && t.terminationReason === 'trap')).toBe(true); + }); + + it('passes string arguments as BigInt for i64 parameters', async () => { + const wasmBase64 = await compileWatToBase64(WAT_I64); + const result = await runWasmEvaluation({ + wasmBase64, + tests: [{ name: 'big', functionName: 'mul', args: ['9007199254740993', '2'], expected: '18014398509481986' }], + }); + expect(result.tests[0]).toMatchObject({ status: 'passed', actual: '18014398509481986' }); + }); + + describe('host isolation', () => { + it.each([ + ['an env host function', WAT_DISALLOWED_IMPORT], + ['a WASI filesystem call', WAT_WASI], + ])('rejects modules importing %s', async (_label, wat) => { + const wasmBase64 = await compileWatToBase64(wat); + const result = await runWasmEvaluation({ wasmBase64, tests: [{ name: 't', functionName: 'run' }] }); + + expect(result.terminationReason).toBe('validation-failed'); + expect(result.validationCode).toBe('host-import-forbidden'); + }); + + it('spawns the worker with an empty environment and no inherited exec flags', async () => { + spawnedWorkerOptions.length = 0; + const wasmBase64 = await compileWatToBase64(WAT_ADD); + await runWasmEvaluation({ wasmBase64, tests: [{ name: 'ok', functionName: 'add', args: [1, 1] }] }); + + expect(spawnedWorkerOptions).toHaveLength(1); + const options = spawnedWorkerOptions[0]!; + expect(options.env).toEqual({}); + expect(options.execArgv).toEqual([]); + expect(options.resourceLimits).toBeDefined(); + }); + }); + + it('rejects a submission whose tests reference a missing export', async () => { + const wasmBase64 = await compileWatToBase64(WAT_ADD); + const result = await runWasmEvaluation({ wasmBase64, tests: [{ name: 't', functionName: 'nope' }] }); + expect(result.validationCode).toBe('missing-export'); + }); + + it('rejects an empty test list', async () => { + const wasmBase64 = await compileWatToBase64(WAT_ADD); + const result = await runWasmEvaluation({ wasmBase64, tests: [] }); + expect(result.terminationReason).toBe('validation-failed'); + }); +}); diff --git a/backend/tests/wasmValidator.test.ts b/backend/tests/wasmValidator.test.ts new file mode 100644 index 00000000..d4159390 --- /dev/null +++ b/backend/tests/wasmValidator.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from '@jest/globals'; +import wabtInit from 'wabt'; +import { decodeWasmBase64, validateWasmModule } from '../src/utils/wasmValidator.js'; +import { compileWat, WAT_ADD, WAT_MEMORY_UNBOUNDED } from './fixtures/wasmFixtures.js'; + +const LIMIT = 50 * 1024 * 1024; + +async function compileWithFeatures(wat: string, features: Record): Promise { + const wabt = await wabtInit(); + const mod = wabt.parseWat('feature.wat', wat, features); + const { buffer } = mod.toBinary({}); + mod.destroy(); + return new Uint8Array(buffer); +} + +describe('wasmValidator (BE-010)', () => { + it('accepts a pure module and lists its exported functions', async () => { + const result = validateWasmModule(await compileWat(WAT_ADD), { memoryLimitBytes: LIMIT, requiredExports: ['add'] }); + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.exportedFunctions).toEqual(['add']); + expect(result.memoryLimitPages).toBe(800); + } + }); + + it('rejects malformed base64 instead of silently dropping characters', () => { + expect(decodeWasmBase64('AGFzbQ$$')).toBeNull(); + const result = validateWasmModule('not base64!!', { memoryLimitBytes: LIMIT }); + expect(result).toMatchObject({ valid: false, issue: { code: 'invalid-encoding' } }); + }); + + it('rejects bytes that are not a wasm module', () => { + const result = validateWasmModule(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9]), { memoryLimitBytes: LIMIT }); + expect(result).toMatchObject({ valid: false, issue: { code: 'invalid-module' } }); + }); + + it('rejects oversized modules before parsing them', () => { + const result = validateWasmModule(new Uint8Array(32), { memoryLimitBytes: LIMIT, maxModuleBytes: 16 }); + expect(result).toMatchObject({ valid: false, issue: { code: 'module-too-large' } }); + }); + + it('rejects unbounded local memory', async () => { + const result = validateWasmModule(await compileWat(WAT_MEMORY_UNBOUNDED), { memoryLimitBytes: LIMIT }); + expect(result).toMatchObject({ valid: false, issue: { code: 'memory-unbounded' } }); + }); + + it('rejects modules that try to declare the reserved metering import themselves', async () => { + const wat = `(module (import "metering" "usegas" (func (param i32))) (func (export "run")))`; + const result = validateWasmModule(await compileWat(wat), { memoryLimitBytes: LIMIT }); + expect(result).toMatchObject({ valid: false, issue: { code: 'host-import-forbidden' } }); + }); + + it('rejects imported globals and tables, not just functions', async () => { + const global = `(module (import "env" "g" (global i32)) (func (export "run")))`; + const table = `(module (import "env" "t" (table 1 funcref)) (func (export "run")))`; + for (const wat of [global, table]) { + const result = validateWasmModule(await compileWat(wat), { memoryLimitBytes: LIMIT }); + expect(result).toMatchObject({ valid: false, issue: { code: 'host-import-forbidden' } }); + } + }); + + it('rejects a second (multi-memory) linear memory that would bypass the ceiling', async () => { + const wat = `(module (memory 1 1) (memory 1) (func (export "run")))`; + const result = validateWasmModule(await compileWithFeatures(wat, { multi_memory: true }), { + memoryLimitBytes: LIMIT, + }); + expect(result.valid).toBe(false); + }); + + it('rejects the exception-handling proposal so guests cannot catch the out-of-gas trap', async () => { + const wat = `(module (tag $e) (func (export "run")))`; + const result = validateWasmModule(await compileWithFeatures(wat, { exceptions: true }), { + memoryLimitBytes: LIMIT, + }); + expect(result).toMatchObject({ valid: false }); + }); + + it('reports a missing required export', async () => { + const result = validateWasmModule(await compileWat(WAT_ADD), { + memoryLimitBytes: LIMIT, + requiredExports: ['add', 'sub'], + }); + expect(result).toMatchObject({ valid: false, issue: { code: 'missing-export' } }); + }); +}); From 25c51dd809c704461cbd581886f74eda61cbac5a Mon Sep 17 00:00:00 2001 From: emmzy4tech Date: Wed, 21 Oct 2026 17:34:22 +0100 Subject: [PATCH 2/5] feat(backend): add tiered bullmq queues with retries, rate limits and dlq (BE-006) Critical/standard/low queues with 5 retries using exponential jitter backoff, per-user enqueue quotas, a cluster-wide Soroban RPC limiter for minting, a durable dead-letter queue with alerts, and admin inspect/replay/purge endpoints. Co-Authored-By: Claude Opus 5.5 --- backend/src/queues/queueManager.ts | 328 ++++++++++++++++++++ backend/src/routes/admin/queues.routes.ts | 154 ++++++++++ backend/src/workers/dlqProcessor.ts | 264 ++++++++++++++++ backend/src/workers/mintingWorker.ts | 156 ++++++++++ backend/tests/admin.queues.routes.test.ts | 124 ++++++++ backend/tests/queueManager.test.ts | 358 ++++++++++++++++++++++ 6 files changed, 1384 insertions(+) create mode 100644 backend/src/queues/queueManager.ts create mode 100644 backend/src/routes/admin/queues.routes.ts create mode 100644 backend/src/workers/dlqProcessor.ts create mode 100644 backend/src/workers/mintingWorker.ts create mode 100644 backend/tests/admin.queues.routes.test.ts create mode 100644 backend/tests/queueManager.test.ts diff --git a/backend/src/queues/queueManager.ts b/backend/src/queues/queueManager.ts new file mode 100644 index 00000000..53d230dc --- /dev/null +++ b/backend/src/queues/queueManager.ts @@ -0,0 +1,328 @@ +/** + * Distributed Queue Manager — BE-006 + * + * Central registry for the BullMQ queues that carry background work, split by + * criticality so a flood of low-value jobs can never starve the ones that + * matter: + * + * critical → `tx-minting` (Soroban certificate minting) + * standard → `ipfs-pinning` (IPFS pin requests) + * → `contract-evaluation` (sandboxed WASM grading, BE-010) + * low → `email-notifications` (transactional email) + * + * Every job gets 1 initial attempt + `MAX_RETRIES` (5) retries with + * exponential backoff and jitter. Once those are exhausted — or the processor + * throws BullMQ's `UnrecoverableError` — the job is copied into the + * dead-letter queue by `DlqProcessor`, which also raises alerts. + * + * Concurrency is enforced at two levels: + * - per worker process (`concurrency`) and, for queues that talk to rate + * limited upstreams such as Soroban RPC, a Redis-backed global `limiter` + * shared by every worker in the cluster; + * - per user at enqueue time, via a fixed-window counter in Redis, so one + * account cannot monopolise a queue. + */ + +import { Queue, Worker, type Job, type JobsOptions, type Processor, type WorkerOptions } from 'bullmq'; +import { Redis } from 'ioredis'; +import logger from '../utils/logger.js'; +import { DlqProcessor, DEAD_LETTER_QUEUE_NAME, type DeadLetterJobData } from '../workers/dlqProcessor.js'; + +export const QUEUE_NAMES = { + TX_MINTING: 'tx-minting', + IPFS_PINNING: 'ipfs-pinning', + EMAIL_NOTIFICATIONS: 'email-notifications', + CONTRACT_EVALUATION: 'contract-evaluation', +} as const; + +export type ManagedQueueName = (typeof QUEUE_NAMES)[keyof typeof QUEUE_NAMES]; +export type QueueTier = 'critical' | 'standard' | 'low'; + +/** Retries after the first attempt; BullMQ's `attempts` therefore is MAX_RETRIES + 1. */ +export const MAX_RETRIES = 5; +export const BACKOFF_BASE_DELAY_MS = 2_000; +/** Fraction of each backoff delay that is randomised, to avoid thundering-herd retries. */ +export const BACKOFF_JITTER = 0.5; + +export interface QueueDefinition { + name: ManagedQueueName; + tier: QueueTier; + /** Jobs processed in parallel by a single worker process. */ + concurrency: number; + /** Cluster-wide throughput cap shared by all workers on this queue. */ + limiter?: { max: number; duration: number }; + /** Per-user enqueue quota. */ + userRateLimit: { max: number; windowMs: number }; +} + +const envInt = (name: string, fallback: number): number => { + const parsed = Number.parseInt(process.env[name] ?? '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +export function buildQueueDefinitions(): Record { + return { + [QUEUE_NAMES.TX_MINTING]: { + name: QUEUE_NAMES.TX_MINTING, + tier: 'critical', + concurrency: envInt('MINTING_WORKER_CONCURRENCY', 2), + // Soroban RPC rejects bursts of submissions; keep the whole cluster under it. + limiter: { max: envInt('SOROBAN_RPC_MAX_SUBMISSIONS_PER_SEC', 5), duration: 1_000 }, + userRateLimit: { max: envInt('MINTING_USER_RATE_LIMIT_PER_MIN', 5), windowMs: 60_000 }, + }, + [QUEUE_NAMES.IPFS_PINNING]: { + name: QUEUE_NAMES.IPFS_PINNING, + tier: 'standard', + concurrency: envInt('IPFS_WORKER_CONCURRENCY', 5), + userRateLimit: { max: envInt('IPFS_USER_RATE_LIMIT_PER_MIN', 30), windowMs: 60_000 }, + }, + [QUEUE_NAMES.CONTRACT_EVALUATION]: { + name: QUEUE_NAMES.CONTRACT_EVALUATION, + tier: 'standard', + concurrency: envInt('EVALUATION_WORKER_CONCURRENCY', 2), + userRateLimit: { max: envInt('EVALUATION_USER_RATE_LIMIT_PER_MIN', 10), windowMs: 60_000 }, + }, + [QUEUE_NAMES.EMAIL_NOTIFICATIONS]: { + name: QUEUE_NAMES.EMAIL_NOTIFICATIONS, + tier: 'low', + concurrency: envInt('EMAIL_WORKER_CONCURRENCY', 10), + limiter: { max: envInt('EMAIL_MAX_SENDS_PER_SEC', 20), duration: 1_000 }, + userRateLimit: { max: envInt('EMAIL_USER_RATE_LIMIT_PER_MIN', 20), windowMs: 60_000 }, + }, + }; +} + +const TIER_PRIORITY: Record = { critical: 1, standard: 5, low: 10 }; + +/** Default options applied to every job; callers may tighten but not disable retries. */ +export function defaultJobOptions(tier: QueueTier, backoffDelayMs = BACKOFF_BASE_DELAY_MS): JobsOptions { + return { + attempts: MAX_RETRIES + 1, + backoff: { type: 'exponential', delay: backoffDelayMs, jitter: BACKOFF_JITTER }, + priority: TIER_PRIORITY[tier], + removeOnComplete: { age: 24 * 3600, count: 1_000 }, + // Keep terminal failures around for a week so they can be cross-checked + // against the DLQ copy. + removeOnFail: { age: 7 * 24 * 3600 }, + }; +} + +export class QueueRateLimitError extends Error { + readonly statusCode = 429; + constructor( + readonly queueName: ManagedQueueName, + readonly retryAfterMs: number + ) { + super(`Rate limit exceeded for queue '${queueName}'. Retry in ${Math.ceil(retryAfterMs / 1000)}s.`); + this.name = 'QueueRateLimitError'; + } +} + +export interface EnqueueOptions { + /** Account the job is attributed to; drives the per-user quota. */ + userId?: string; + /** Idempotency key; BullMQ ignores a second add with the same id. Must not contain ':'. */ + jobId?: string; + delay?: number; +} + +/** Minimal surface of the Redis client used for per-user quotas (lets tests pass ioredis-mock). */ +export interface RateLimitStore { + incr(key: string): Promise; + pexpire(key: string, ms: number): Promise; + quit?(): Promise; +} + +export interface QueueManagerOptions { + connection?: { host: string; port: number; password?: string; username?: string; db?: number }; + rateLimitStore?: RateLimitStore; + definitions?: Record; + dlq?: DlqProcessor; + /** Redis key prefix for every queue (BullMQ default: 'bull'). */ + prefix?: string; + /** Base delay of the exponential retry backoff. */ + backoffDelayMs?: number; + /** Injected for tests. */ + queueFactory?: (name: string) => Queue; +} + +function connectionFromEnv(): NonNullable { + const url = new URL(process.env.REDIS_URL || 'redis://localhost:6379'); + return { + host: url.hostname, + port: Number(url.port) || 6379, + ...(url.password ? { password: decodeURIComponent(url.password) } : {}), + ...(url.username ? { username: decodeURIComponent(url.username) } : {}), + ...(url.pathname.length > 1 ? { db: Number(url.pathname.slice(1)) || 0 } : {}), + }; +} + +export class QueueManager { + readonly definitions: Record; + private readonly connection: NonNullable; + private readonly prefix: string | undefined; + private readonly backoffDelayMs: number; + private readonly queueFactory: (name: string) => Queue; + private readonly queues = new Map(); + private readonly workers: Worker[] = []; + private rateLimitStore: RateLimitStore | undefined; + private dlqProcessor: DlqProcessor | undefined; + + constructor(options: QueueManagerOptions = {}) { + this.definitions = options.definitions ?? buildQueueDefinitions(); + this.connection = options.connection ?? connectionFromEnv(); + this.rateLimitStore = options.rateLimitStore; + this.dlqProcessor = options.dlq; + this.prefix = options.prefix; + this.backoffDelayMs = options.backoffDelayMs ?? BACKOFF_BASE_DELAY_MS; + this.queueFactory = + options.queueFactory ?? + ((name) => + new Queue(name, { + connection: { ...this.connection, maxRetriesPerRequest: null }, + ...(this.prefix ? { prefix: this.prefix } : {}), + })); + } + + /** Lazily creates queues so importing this module never opens a Redis connection. */ + getQueue(name: ManagedQueueName | typeof DEAD_LETTER_QUEUE_NAME): Queue { + let queue = this.queues.get(name); + if (!queue) { + queue = this.queueFactory(name); + this.queues.set(name, queue); + } + return queue as Queue; + } + + isManagedQueue(name: string): name is ManagedQueueName { + return Object.prototype.hasOwnProperty.call(this.definitions, name); + } + + get dlq(): DlqProcessor { + if (!this.dlqProcessor) { + this.dlqProcessor = new DlqProcessor({ + queue: this.getQueue(DEAD_LETTER_QUEUE_NAME), + resolveSourceQueue: (name) => (this.isManagedQueue(name) ? this.getQueue(name) : undefined), + }); + } + return this.dlqProcessor; + } + + private getRateLimitStore(): RateLimitStore { + if (!this.rateLimitStore) { + this.rateLimitStore = new Redis({ ...this.connection, maxRetriesPerRequest: 2, lazyConnect: true }); + } + return this.rateLimitStore; + } + + /** + * Fixed-window per-user quota. Throws `QueueRateLimitError` once the user + * exceeds `userRateLimit.max` jobs on this queue within the window. + */ + async assertWithinUserRateLimit(name: ManagedQueueName, userId: string, now = Date.now()): Promise { + const { max, windowMs } = this.definitions[name].userRateLimit; + const window = Math.floor(now / windowMs); + const key = `queue-rl:${name}:${userId}:${window}`; + const store = this.getRateLimitStore(); + const count = await store.incr(key); + if (count === 1) await store.pexpire(key, windowMs); + if (count > max) { + throw new QueueRateLimitError(name, (window + 1) * windowMs - now); + } + } + + async enqueue( + name: ManagedQueueName, + jobName: string, + data: T, + options: EnqueueOptions = {} + ): Promise> { + const definition = this.definitions[name]; + if (options.userId) { + await this.assertWithinUserRateLimit(name, options.userId); + } + + const opts: JobsOptions = { + ...defaultJobOptions(definition.tier, this.backoffDelayMs), + ...(options.jobId ? { jobId: options.jobId } : {}), + ...(options.delay ? { delay: options.delay } : {}), + }; + const payload = options.userId ? { ...data, userId: options.userId } : data; + return (this.getQueue(name) as Queue).add(jobName, payload, opts) as Promise>; + } + + /** + * Starts a worker for `name` with the queue's concurrency/limiter settings + * and wires terminal failures into the dead-letter queue. + */ + createWorker( + name: ManagedQueueName, + processor: Processor, + overrides: Partial> = {} + ): Worker { + const definition = this.definitions[name]; + const worker = new Worker(name, processor, { + connection: { ...this.connection, maxRetriesPerRequest: null }, + ...(this.prefix ? { prefix: this.prefix } : {}), + concurrency: overrides.concurrency ?? definition.concurrency, + ...(overrides.limiter ?? definition.limiter ? { limiter: overrides.limiter ?? definition.limiter } : {}), + }); + + worker.on('failed', (job, err) => { + if (!job) return; + this.dlq.handleFailure(name, job as Job, err).catch((dlqErr: unknown) => { + logger.error('[QueueManager] Failed to dead-letter job', { + queue: name, + jobId: job.id, + error: dlqErr instanceof Error ? dlqErr.message : String(dlqErr), + }); + }); + }); + worker.on('error', (err) => { + logger.error('[QueueManager] Worker error', { queue: name, error: err.message }); + }); + + this.workers.push(worker as Worker); + logger.info('[QueueManager] Worker started', { + queue: name, + tier: definition.tier, + concurrency: overrides.concurrency ?? definition.concurrency, + }); + return worker; + } + + async getStats(): Promise< + Array<{ queue: string; tier: QueueTier | 'dead-letter'; counts: Record }> + > { + const states = ['waiting', 'active', 'delayed', 'completed', 'failed', 'paused'] as const; + const managed = await Promise.all( + (Object.values(this.definitions) as QueueDefinition[]).map(async (def) => ({ + queue: def.name, + tier: def.tier, + counts: await this.getQueue(def.name).getJobCounts(...states), + })) + ); + const dlqCounts = await this.getQueue(DEAD_LETTER_QUEUE_NAME).getJobCounts('waiting', 'paused'); + return [...managed, { queue: DEAD_LETTER_QUEUE_NAME, tier: 'dead-letter', counts: dlqCounts }]; + } + + async close(): Promise { + await Promise.allSettled(this.workers.map((w) => w.close())); + await Promise.allSettled([...this.queues.values()].map((q) => q.close())); + await this.rateLimitStore?.quit?.(); + this.workers.length = 0; + this.queues.clear(); + } +} + +let instance: QueueManager | null = null; + +export function getQueueManager(): QueueManager { + if (!instance) instance = new QueueManager(); + return instance; +} + +/** Test hook: swap the process-wide manager. */ +export function setQueueManager(manager: QueueManager | null): void { + instance = manager; +} diff --git a/backend/src/routes/admin/queues.routes.ts b/backend/src/routes/admin/queues.routes.ts new file mode 100644 index 00000000..b978a317 --- /dev/null +++ b/backend/src/routes/admin/queues.routes.ts @@ -0,0 +1,154 @@ +import { Router, Request, Response } from 'express'; +import { z } from 'zod'; +import { authenticateToken } from '../../middleware/auth.js'; +import { requireAdmin } from '../../middleware/admin.js'; +import { validate } from '../../middleware/validation.js'; +import { getQueueManager } from '../../queues/queueManager.js'; +import logger from '../../utils/logger.js'; + +/** + * Admin control plane for the BullMQ queue system (BE-006): queue depth + * statistics plus inspect / replay / purge of dead-lettered jobs. + * Mounted at /api/v1/admin/queues. + */ + +const router: ReturnType = Router(); + +router.use(authenticateToken as any); +router.use(requireAdmin as any); + +const queueName = z.string().trim().min(1).max(128).optional(); + +export const queueDlqReplaySchema = z.object({ queueName }); +export const queueDlqPurgeSchema = z.object({ + queueName, + confirm: z.boolean().refine((value) => value === true, { + message: 'Purge operation must be confirmed with confirm: true', + }), +}); + +const parseInteger = (value: unknown): number | undefined => { + if (typeof value !== 'string') return undefined; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined; +}; + +const fail = (res: Response, error: unknown, message: string) => { + logger.error(`[admin/queues] ${message}`, { error: error instanceof Error ? error.message : String(error) }); + res.status(500).json({ status: 'error', error: message }); +}; + +/** + * @route GET /api/v1/admin/queues/stats + * @desc Job counts for every managed queue and the dead-letter queue + */ +router.get('/stats', async (_req: Request, res: Response) => { + try { + const queues = await getQueueManager().getStats(); + res.json({ status: 'success', data: { queues } }); + } catch (error) { + fail(res, error, 'Failed to retrieve queue statistics'); + } +}); + +/** + * @route GET /api/v1/admin/queues/dlq + * @desc List dead-lettered jobs, optionally filtered by source queue + */ +router.get('/dlq', async (req: Request, res: Response) => { + try { + const sourceQueue = typeof req.query.queue === 'string' && req.query.queue ? req.query.queue : undefined; + const offset = parseInteger(req.query.offset); + const limit = parseInteger(req.query.limit); + const result = await getQueueManager().dlq.list({ + ...(sourceQueue ? { sourceQueue } : {}), + ...(offset !== undefined ? { offset } : {}), + ...(limit !== undefined ? { limit } : {}), + }); + res.json({ status: 'success', data: { ...result, count: result.entries.length } }); + } catch (error) { + fail(res, error, 'Failed to inspect dead-letter queue'); + } +}); + +/** + * @route POST /api/v1/admin/queues/dlq/replay + * @desc Replay every dead-lettered job (optionally only those from one queue) + */ +router.post('/dlq/replay', validate(queueDlqReplaySchema), async (req: Request, res: Response) => { + try { + const { queueName: source } = req.body as z.infer; + const result = await getQueueManager().dlq.replayAll(source); + res.json({ status: 'success', data: { ...result, queueName: source ?? 'all' } }); + } catch (error) { + fail(res, error, 'Failed to replay dead-lettered jobs'); + } +}); + +/** + * @route POST /api/v1/admin/queues/dlq/purge + * @desc Permanently delete dead-lettered jobs; requires { confirm: true } + */ +router.post('/dlq/purge', validate(queueDlqPurgeSchema), async (req: Request, res: Response) => { + try { + const { queueName: source } = req.body as z.infer; + const result = await getQueueManager().dlq.purgeAll(source); + logger.warn('[admin/queues] DLQ purged', { by: (req as any).user?.id, queueName: source ?? 'all', ...result }); + res.json({ status: 'success', data: { ...result, queueName: source ?? 'all' } }); + } catch (error) { + fail(res, error, 'Failed to purge dead-letter queue'); + } +}); + +/** + * @route GET /api/v1/admin/queues/dlq/:id + * @desc Inspect a single dead-lettered job including its failure reason and stack + */ +router.get('/dlq/:id', async (req: Request, res: Response) => { + try { + const entry = await getQueueManager().dlq.get(String(req.params.id)); + if (!entry) { + res.status(404).json({ status: 'error', error: 'Dead-lettered job not found' }); + return; + } + res.json({ status: 'success', data: { entry } }); + } catch (error) { + fail(res, error, 'Failed to retrieve dead-lettered job'); + } +}); + +/** + * @route POST /api/v1/admin/queues/dlq/:id/replay + * @desc Re-enqueue one dead-lettered job on its source queue with a fresh retry budget + */ +router.post('/dlq/:id/replay', async (req: Request, res: Response) => { + try { + const result = await getQueueManager().dlq.replay(String(req.params.id)); + if (!result) { + res.status(404).json({ status: 'error', error: 'Dead-lettered job not found' }); + return; + } + res.json({ status: 'success', data: result }); + } catch (error) { + fail(res, error, 'Failed to replay dead-lettered job'); + } +}); + +/** + * @route DELETE /api/v1/admin/queues/dlq/:id + * @desc Permanently delete one dead-lettered job + */ +router.delete('/dlq/:id', async (req: Request, res: Response) => { + try { + const removed = await getQueueManager().dlq.purge(String(req.params.id)); + if (!removed) { + res.status(404).json({ status: 'error', error: 'Dead-lettered job not found' }); + return; + } + res.json({ status: 'success', data: { id: req.params.id, purged: true } }); + } catch (error) { + fail(res, error, 'Failed to purge dead-lettered job'); + } +}); + +export default router; diff --git a/backend/src/workers/dlqProcessor.ts b/backend/src/workers/dlqProcessor.ts new file mode 100644 index 00000000..77e2e543 --- /dev/null +++ b/backend/src/workers/dlqProcessor.ts @@ -0,0 +1,264 @@ +/** + * Dead-Letter Queue Processor — BE-006 + * + * Receives jobs whose retries are exhausted (or that failed with BullMQ's + * `UnrecoverableError`) from any queue managed by `QueueManager`, parks a + * full copy of them in the durable `dead-letter` BullMQ queue, and raises + * alerts. The DLQ queue intentionally has no consumer: entries stay in the + * waiting list until an operator replays or purges them through the admin + * API. + * + * Alerts fire for every dead-lettered job (structured `logger.error`, plus a + * POST to `DLQ_ALERT_WEBHOOK_URL` when configured) and, separately, once the + * DLQ depth crosses `DLQ_ALERT_THRESHOLD` — throttled so a failure storm + * produces one page, not thousands. + */ + +import type { Job, JobsOptions, Queue } from 'bullmq'; +import logger from '../utils/logger.js'; + +export const DEAD_LETTER_QUEUE_NAME = 'dead-letter'; + +export interface DeadLetterJobData { + sourceQueue: string; + sourceJobId: string; + jobName: string; + data: Record; + opts: Pick; + failedReason: string; + stacktrace: string[]; + attemptsMade: number; + failedAt: string; + userId?: string; +} + +export interface DeadLetterEntry extends DeadLetterJobData { + id: string; +} + +export interface DlqAlert { + kind: 'job-dead-lettered' | 'threshold-exceeded'; + queue: string; + message: string; + details: Record; +} + +export type DlqAlertSink = (alert: DlqAlert) => Promise | void; + +export interface DlqProcessorOptions { + queue: Queue; + resolveSourceQueue: (name: string) => Queue | undefined; + alert?: DlqAlertSink; + alertThreshold?: number; + thresholdAlertCooldownMs?: number; + now?: () => number; +} + +const DLQ_LIST_STATES = ['waiting', 'paused'] as const; + +/** BullMQ forbids ':' in custom job ids. */ +const deadLetterId = (sourceQueue: string, jobId: string) => `${sourceQueue}--${jobId}`.replace(/:/g, '_'); + +/** + * Default alert sink: always logs; additionally POSTs to a webhook (e.g. a + * Slack/PagerDuty bridge) when `DLQ_ALERT_WEBHOOK_URL` is set. + */ +export const defaultDlqAlertSink: DlqAlertSink = async (alert) => { + logger.error(`[DLQ] ${alert.message}`, { kind: alert.kind, queue: alert.queue, ...alert.details }); + + const url = process.env.DLQ_ALERT_WEBHOOK_URL; + if (!url) return; + try { + const response = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ text: alert.message, ...alert }), + signal: AbortSignal.timeout(5_000), + }); + if (!response.ok) { + logger.warn('[DLQ] Alert webhook responded with non-2xx', { status: response.status }); + } + } catch (err) { + logger.warn('[DLQ] Alert webhook delivery failed', { error: err instanceof Error ? err.message : String(err) }); + } +}; + +export class DlqProcessor { + private readonly queue: Queue; + private readonly resolveSourceQueue: (name: string) => Queue | undefined; + private readonly alert: DlqAlertSink; + private readonly alertThreshold: number; + private readonly thresholdAlertCooldownMs: number; + private readonly now: () => number; + private lastThresholdAlertAt = Number.NEGATIVE_INFINITY; + + constructor(options: DlqProcessorOptions) { + this.queue = options.queue; + this.resolveSourceQueue = options.resolveSourceQueue; + this.alert = options.alert ?? defaultDlqAlertSink; + this.alertThreshold = options.alertThreshold ?? (Number(process.env.DLQ_ALERT_THRESHOLD) || 10); + this.thresholdAlertCooldownMs = options.thresholdAlertCooldownMs ?? 5 * 60_000; + this.now = options.now ?? Date.now; + } + + /** + * A failure is terminal when BullMQ will not retry it: the job has been + * moved to the failed set (`finishedOn` is only stamped then), it used up + * its attempts, or the processor declared it unrecoverable. + */ + static isTerminalFailure(job: Pick, err?: Error): boolean { + if (err?.name === 'UnrecoverableError') return true; + if (job.finishedOn) return true; + return job.attemptsMade >= (job.opts.attempts ?? 1); + } + + /** Called from every managed worker's `failed` event. No-op for failures that will be retried. */ + async handleFailure(sourceQueue: string, job: Job, err?: Error): Promise { + if (!DlqProcessor.isTerminalFailure(job, err)) return false; + + const data = (job.data ?? {}) as Record; + const entry: DeadLetterJobData = { + sourceQueue, + sourceJobId: String(job.id), + jobName: job.name, + data, + opts: { + ...(job.opts.attempts !== undefined ? { attempts: job.opts.attempts } : {}), + ...(job.opts.priority !== undefined ? { priority: job.opts.priority } : {}), + ...(job.opts.backoff !== undefined ? { backoff: job.opts.backoff } : {}), + }, + failedReason: err?.message ?? job.failedReason ?? 'unknown error', + stacktrace: (job.stacktrace ?? []).slice(-5), + attemptsMade: job.attemptsMade, + failedAt: new Date(this.now()).toISOString(), + ...(typeof data.userId === 'string' ? { userId: data.userId } : {}), + }; + + await this.queue.add(job.name, entry, { + jobId: deadLetterId(sourceQueue, String(job.id)), + removeOnComplete: true, + removeOnFail: false, + }); + + await this.raise({ + kind: 'job-dead-lettered', + queue: sourceQueue, + message: `Job ${job.name}#${job.id} on '${sourceQueue}' moved to the DLQ after ${job.attemptsMade} attempt(s): ${entry.failedReason}`, + details: { jobId: job.id, jobName: job.name, attemptsMade: job.attemptsMade, userId: entry.userId }, + }); + await this.checkThreshold(sourceQueue); + return true; + } + + private async checkThreshold(sourceQueue: string): Promise { + const depth = await this.depth(); + const now = this.now(); + if (depth < this.alertThreshold || now - this.lastThresholdAlertAt < this.thresholdAlertCooldownMs) return; + this.lastThresholdAlertAt = now; + await this.raise({ + kind: 'threshold-exceeded', + queue: sourceQueue, + message: `Dead-letter queue depth is ${depth} (threshold ${this.alertThreshold}).`, + details: { depth, threshold: this.alertThreshold }, + }); + } + + private async raise(alert: DlqAlert): Promise { + try { + await this.alert(alert); + } catch (err) { + logger.warn('[DLQ] Alert sink threw', { error: err instanceof Error ? err.message : String(err) }); + } + } + + async depth(): Promise { + const counts = await this.queue.getJobCounts(...DLQ_LIST_STATES); + return Object.values(counts).reduce((sum, n) => sum + n, 0); + } + + private toEntry(job: Job): DeadLetterEntry { + return { id: String(job.id), ...job.data }; + } + + async list(filter: { sourceQueue?: string; offset?: number; limit?: number } = {}): Promise<{ + total: number; + entries: DeadLetterEntry[]; + }> { + const offset = Math.max(0, filter.offset ?? 0); + const limit = Math.min(Math.max(1, filter.limit ?? 50), 500); + + if (!filter.sourceQueue) { + const [jobs, total] = await Promise.all([ + this.queue.getJobs([...DLQ_LIST_STATES], offset, offset + limit - 1, true), + this.depth(), + ]); + return { total, entries: jobs.filter(Boolean).map((job) => this.toEntry(job)) }; + } + + const all = (await this.queue.getJobs([...DLQ_LIST_STATES], 0, -1, true)) + .filter(Boolean) + .filter((job) => job.data.sourceQueue === filter.sourceQueue); + return { total: all.length, entries: all.slice(offset, offset + limit).map((job) => this.toEntry(job)) }; + } + + async get(id: string): Promise { + const job = await this.queue.getJob(id); + return job ? this.toEntry(job) : null; + } + + /** + * Re-enqueues the original job on its source queue with a fresh retry + * budget, then removes it from the DLQ. + */ + async replay(id: string): Promise<{ id: string; replayedJobId: string; sourceQueue: string } | null> { + const job = await this.queue.getJob(id); + if (!job) return null; + + const source = this.resolveSourceQueue(job.data.sourceQueue); + if (!source) { + throw new Error(`Source queue '${job.data.sourceQueue}' is not managed; cannot replay.`); + } + + const replayed = await source.add(job.data.jobName, job.data.data, job.data.opts); + await job.remove(); + logger.info('[DLQ] Replayed dead-lettered job', { + dlqId: id, + sourceQueue: job.data.sourceQueue, + replayedJobId: replayed.id, + }); + return { id, replayedJobId: String(replayed.id), sourceQueue: job.data.sourceQueue }; + } + + async replayAll(sourceQueue?: string): Promise<{ replayed: number; failed: Array<{ id: string; error: string }> }> { + const jobs = (await this.queue.getJobs([...DLQ_LIST_STATES], 0, -1, true)) + .filter(Boolean) + .filter((job) => !sourceQueue || job.data.sourceQueue === sourceQueue); + + let replayed = 0; + const failed: Array<{ id: string; error: string }> = []; + for (const job of jobs) { + try { + if (await this.replay(String(job.id))) replayed++; + } catch (err) { + failed.push({ id: String(job.id), error: err instanceof Error ? err.message : String(err) }); + } + } + return { replayed, failed }; + } + + async purge(id: string): Promise { + const job = await this.queue.getJob(id); + if (!job) return false; + await job.remove(); + return true; + } + + async purgeAll(sourceQueue?: string): Promise<{ purged: number }> { + const jobs = (await this.queue.getJobs([...DLQ_LIST_STATES], 0, -1, true)) + .filter(Boolean) + .filter((job) => !sourceQueue || job.data.sourceQueue === sourceQueue); + await Promise.all(jobs.map((job) => job.remove())); + logger.warn('[DLQ] Purged dead-lettered jobs', { count: jobs.length, sourceQueue: sourceQueue ?? 'all' }); + return { purged: jobs.length }; + } +} diff --git a/backend/src/workers/mintingWorker.ts b/backend/src/workers/mintingWorker.ts new file mode 100644 index 00000000..8f1a6e1e --- /dev/null +++ b/backend/src/workers/mintingWorker.ts @@ -0,0 +1,156 @@ +/** + * Certificate Minting Worker — BE-006 + * + * Consumes the critical `tx-minting` queue and submits certificate mints to + * Soroban. Throughput is bounded twice over so the cluster never exceeds the + * RPC provider's submission limits: + * + * - the queue's Redis-backed `limiter` (SOROBAN_RPC_MAX_SUBMISSIONS_PER_SEC), + * shared by every minting worker process; + * - when the RPC still answers 429, the worker pauses the *whole queue* for + * the advertised back-off via `worker.rateLimit()` and re-queues the job + * without consuming one of its retry attempts. + * + * Other transient failures are retried with exponential jitter by BullMQ; + * configuration/validation problems are raised as `UnrecoverableError` so + * they go straight to the dead-letter queue instead of burning retries. + */ + +import { UnrecoverableError, Worker, type Job } from 'bullmq'; +import { BlockchainNotConfiguredError, CertificateBlockchainService } from '../blockchain/CertificateBlockchainService.js'; +import { getQueueManager, QUEUE_NAMES, type QueueManager } from '../queues/queueManager.js'; +import logger from '../utils/logger.js'; + +export interface MintJobData { + studentId: string; + courseId: string; + /** Certificate metadata passed through to the contract. */ + metadata: Record; + certificateId?: string; + userId?: string; +} + +export interface MintJobResult { + tokenId: string; + transactionHash: string; + contractAddress: string; + mintedAt: string; +} + +export interface Minter { + mintCertificate(metadata: Record): Promise<{ + success: boolean; + tokenId: string; + transactionHash: string; + contractAddress: string; + }>; +} + +/** Default pause applied to the queue when Soroban RPC rate-limits us without a Retry-After. */ +export const RPC_RATE_LIMIT_BACKOFF_MS = 5_000; + +export class RpcRateLimitedError extends Error { + constructor(readonly retryAfterMs = RPC_RATE_LIMIT_BACKOFF_MS) { + super(`Soroban RPC rate limit hit; backing off ${retryAfterMs}ms`); + this.name = 'RpcRateLimitedError'; + } +} + +function isRateLimitResponse(err: unknown): err is { retryAfterMs?: number } { + if (err instanceof RpcRateLimitedError) return true; + const e = err as { status?: number; statusCode?: number; response?: { status?: number }; message?: string }; + const status = e?.status ?? e?.statusCode ?? e?.response?.status; + return status === 429 || /\b429\b|rate.?limit/i.test(e?.message ?? ''); +} + +function validate(data: MintJobData): void { + if (!data || typeof data.studentId !== 'string' || !data.studentId) { + throw new UnrecoverableError('Mint job is missing studentId'); + } + if (typeof data.courseId !== 'string' || !data.courseId) { + throw new UnrecoverableError('Mint job is missing courseId'); + } + if (!data.metadata || typeof data.metadata !== 'object') { + throw new UnrecoverableError('Mint job is missing certificate metadata'); + } +} + +/** + * Builds the processor. `getWorker` is resolved lazily because the processor + * must exist before the Worker that runs it does. + */ +export function createMintProcessor(minter: Minter, getWorker: () => Pick | undefined) { + return async (job: Job): Promise => { + validate(job.data); + + try { + const result = await minter.mintCertificate({ + ...job.data.metadata, + studentId: job.data.studentId, + courseId: job.data.courseId, + }); + + // The blockchain service reports circuit-breaker fallbacks as + // `success: false` instead of throwing; treat that as retryable. + if (!result.success) { + throw new Error(`Mint was not confirmed (tx: ${result.transactionHash})`); + } + + logger.info('[mintingWorker] Certificate minted', { + jobId: job.id, + studentId: job.data.studentId, + courseId: job.data.courseId, + tokenId: result.tokenId, + attempt: job.attemptsMade + 1, + }); + + return { + tokenId: result.tokenId, + transactionHash: result.transactionHash, + contractAddress: result.contractAddress, + mintedAt: new Date().toISOString(), + }; + } catch (err) { + if (err instanceof BlockchainNotConfiguredError) { + throw new UnrecoverableError(err.message); + } + if (isRateLimitResponse(err)) { + const worker = getWorker(); + if (worker) { + const delay = (err as { retryAfterMs?: number }).retryAfterMs ?? RPC_RATE_LIMIT_BACKOFF_MS; + logger.warn('[mintingWorker] Soroban RPC rate limited; pausing queue', { jobId: job.id, delay }); + await worker.rateLimit(delay); + throw Worker.RateLimitError(); + } + } + throw err; + } + }; +} + +let mintingWorker: Worker | null = null; + +export function startMintingWorker( + manager: QueueManager = getQueueManager(), + minter: Minter = new CertificateBlockchainService() +): Worker { + if (mintingWorker) return mintingWorker; + const processor = createMintProcessor(minter, () => mintingWorker ?? undefined); + mintingWorker = manager.createWorker(QUEUE_NAMES.TX_MINTING, processor); + return mintingWorker; +} + +export async function stopMintingWorker(): Promise { + if (!mintingWorker) return; + await mintingWorker.close(); + mintingWorker = null; +} + +/** Convenience producer used by certificate issuance flows. */ +export function enqueueMint(data: MintJobData, manager: QueueManager = getQueueManager()) { + const jobId = data.certificateId ? `mint-${data.certificateId}` : undefined; + return manager.enqueue(QUEUE_NAMES.TX_MINTING, 'mint-certificate', data, { + ...(data.userId ? { userId: data.userId } : {}), + ...(jobId ? { jobId } : {}), + }); +} diff --git a/backend/tests/admin.queues.routes.test.ts b/backend/tests/admin.queues.routes.test.ts new file mode 100644 index 00000000..56721580 --- /dev/null +++ b/backend/tests/admin.queues.routes.test.ts @@ -0,0 +1,124 @@ +/** + * Route tests for the BullMQ admin control plane (BE-006): auth, validation + * and inspect / replay / purge of dead-lettered jobs. + */ +import { jest } from '@jest/globals'; +import { Request, Response, NextFunction } from 'express'; + +type AuthRole = 'unauthenticated' | 'student' | 'admin'; +let mockAuthRole: AuthRole = 'admin'; + +jest.mock('../src/middleware/auth.js', () => ({ + authenticateToken: (req: Request, res: Response, next: NextFunction) => { + if (mockAuthRole === 'unauthenticated') { + return res.status(401).json({ status: 'error', message: 'Access token required' }); + } + (req as any).user = { id: 'admin-user-id', email: 'admin@web3studentlab.org', role: mockAuthRole }; + next(); + }, +})); + +const mockDlq = { + list: jest.fn(), + get: jest.fn(), + replay: jest.fn(), + replayAll: jest.fn(), + purge: jest.fn(), + purgeAll: jest.fn(), +}; +const mockGetStats = jest.fn(); + +jest.mock('../src/queues/queueManager.js', () => ({ + getQueueManager: () => ({ dlq: mockDlq, getStats: mockGetStats }), +})); + +jest.mock('../src/utils/logger.js', () => ({ + __esModule: true, + default: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }, +})); + +import express from 'express'; +import request from 'supertest'; + +let app: express.Application; + +beforeAll(async () => { + const { default: router } = await import('../src/routes/admin/queues.routes.js'); + app = express(); + app.use(express.json()); + app.use('/api/v1/admin/queues', router); +}); + +beforeEach(() => { + mockAuthRole = 'admin'; + jest.clearAllMocks(); +}); + +describe('admin queue routes', () => { + it('requires authentication', async () => { + mockAuthRole = 'unauthenticated'; + const res = await request(app).get('/api/v1/admin/queues/dlq'); + expect(res.status).toBe(401); + }); + + it('requires the admin role', async () => { + mockAuthRole = 'student'; + const res = await request(app).get('/api/v1/admin/queues/dlq'); + expect(res.status).toBe(403); + expect(mockDlq.list).not.toHaveBeenCalled(); + }); + + it('returns queue statistics', async () => { + mockGetStats.mockResolvedValue([{ queue: 'tx-minting', tier: 'critical', counts: { waiting: 1 } }] as never); + const res = await request(app).get('/api/v1/admin/queues/stats'); + expect(res.status).toBe(200); + expect(res.body.data.queues[0].queue).toBe('tx-minting'); + }); + + it('lists dead-lettered jobs with filter and pagination', async () => { + mockDlq.list.mockResolvedValue({ total: 1, entries: [{ id: 'ipfs-pinning--1' }] } as never); + const res = await request(app).get('/api/v1/admin/queues/dlq?queue=ipfs-pinning&limit=10&offset=5'); + expect(res.status).toBe(200); + expect(mockDlq.list).toHaveBeenCalledWith({ sourceQueue: 'ipfs-pinning', limit: 10, offset: 5 }); + expect(res.body.data).toMatchObject({ total: 1, count: 1 }); + }); + + it('inspects a single job and 404s for unknown ids', async () => { + mockDlq.get.mockResolvedValueOnce({ id: 'x', failedReason: 'boom' } as never).mockResolvedValueOnce(null as never); + expect((await request(app).get('/api/v1/admin/queues/dlq/x')).body.data.entry.failedReason).toBe('boom'); + expect((await request(app).get('/api/v1/admin/queues/dlq/y')).status).toBe(404); + }); + + it('replays one job', async () => { + mockDlq.replay.mockResolvedValue({ id: 'x', replayedJobId: '42', sourceQueue: 'tx-minting' } as never); + const res = await request(app).post('/api/v1/admin/queues/dlq/x/replay'); + expect(res.status).toBe(200); + expect(res.body.data.replayedJobId).toBe('42'); + }); + + it('replays in bulk, optionally per queue', async () => { + mockDlq.replayAll.mockResolvedValue({ replayed: 3, failed: [] } as never); + const res = await request(app).post('/api/v1/admin/queues/dlq/replay').send({ queueName: 'tx-minting' }); + expect(res.status).toBe(200); + expect(mockDlq.replayAll).toHaveBeenCalledWith('tx-minting'); + }); + + it('refuses to purge without explicit confirmation', async () => { + const res = await request(app).post('/api/v1/admin/queues/dlq/purge').send({}); + expect(res.status).toBe(400); + expect(mockDlq.purgeAll).not.toHaveBeenCalled(); + }); + + it('purges all dead-lettered jobs when confirmed', async () => { + mockDlq.purgeAll.mockResolvedValue({ purged: 4 } as never); + const res = await request(app).post('/api/v1/admin/queues/dlq/purge').send({ confirm: true }); + expect(res.status).toBe(200); + expect(res.body.data).toMatchObject({ purged: 4, queueName: 'all' }); + }); + + it('purges a single job', async () => { + mockDlq.purge.mockResolvedValueOnce(true as never).mockResolvedValueOnce(false as never); + expect((await request(app).delete('/api/v1/admin/queues/dlq/x')).status).toBe(200); + expect((await request(app).delete('/api/v1/admin/queues/dlq/x')).status).toBe(404); + }); +}); diff --git a/backend/tests/queueManager.test.ts b/backend/tests/queueManager.test.ts new file mode 100644 index 00000000..05d583ee --- /dev/null +++ b/backend/tests/queueManager.test.ts @@ -0,0 +1,358 @@ +import { afterAll, beforeAll, describe, expect, it, jest } from '@jest/globals'; +import { randomUUID } from 'crypto'; +import { UnrecoverableError, type Queue } from 'bullmq'; +import { Redis } from 'ioredis'; +import RedisMock from 'ioredis-mock'; +import { + BACKOFF_JITTER, + defaultJobOptions, + MAX_RETRIES, + QUEUE_NAMES, + QueueManager, + QueueRateLimitError, +} from '../src/queues/queueManager.js'; +import { DlqProcessor, type DlqAlert } from '../src/workers/dlqProcessor.js'; + +jest.mock('../src/utils/logger.js', () => ({ + __esModule: true, + default: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }, +})); + +/** Minimal in-memory stand-in for a BullMQ Queue. */ +class FakeQueue { + private seq = 0; + readonly jobs = new Map(); + constructor(readonly name: string) {} + + async add(name: string, data: any, opts: any = {}) { + const id = opts.jobId ?? String(++this.seq); + const job = { + id, + name, + data, + opts, + remove: jest.fn(async () => { + this.jobs.delete(id); + }), + }; + this.jobs.set(id, job); + return job; + } + async getJob(id: string) { + return this.jobs.get(id); + } + async getJobs(_states: string[], start = 0, end = -1) { + const all = [...this.jobs.values()]; + return end < 0 ? all.slice(start) : all.slice(start, end + 1); + } + async getJobCounts() { + return { waiting: this.jobs.size, paused: 0 }; + } +} + +const asQueue = (q: FakeQueue) => q as unknown as Queue; + +describe('QueueManager (BE-006)', () => { + const build = () => { + const queues = new Map(); + const manager = new QueueManager({ + rateLimitStore: new RedisMock() as any, + queueFactory: (name) => { + const q = new FakeQueue(name); + queues.set(name, q); + return asQueue(q); + }, + }); + return { manager, queues }; + }; + + it('defines critical, standard and low-priority queues', () => { + const { manager } = build(); + expect(manager.definitions[QUEUE_NAMES.TX_MINTING].tier).toBe('critical'); + expect(manager.definitions[QUEUE_NAMES.IPFS_PINNING].tier).toBe('standard'); + expect(manager.definitions[QUEUE_NAMES.EMAIL_NOTIFICATIONS].tier).toBe('low'); + }); + + it('caps Soroban RPC submissions with a cluster-wide limiter and bounded concurrency', () => { + const { manager } = build(); + const minting = manager.definitions[QUEUE_NAMES.TX_MINTING]; + expect(minting.limiter).toEqual({ max: 5, duration: 1_000 }); + expect(minting.concurrency).toBeLessThanOrEqual(minting.limiter!.max); + }); + + it('gives every job 1 attempt + 5 retries with exponential backoff and jitter', () => { + const opts = defaultJobOptions('standard'); + expect(MAX_RETRIES).toBe(5); + expect(opts.attempts).toBe(6); + expect(opts.backoff).toMatchObject({ type: 'exponential', jitter: BACKOFF_JITTER }); + expect(BACKOFF_JITTER).toBeGreaterThan(0); + expect(defaultJobOptions('critical').priority).toBeLessThan(defaultJobOptions('low').priority!); + }); + + it('enqueues with the retry policy and attributes the job to the user', async () => { + const { manager, queues } = build(); + const job = await manager.enqueue(QUEUE_NAMES.IPFS_PINNING, 'pin', { cid: 'bafy' }, { userId: 'u1' }); + + const stored = queues.get(QUEUE_NAMES.IPFS_PINNING)!.jobs.get(String(job.id)); + expect(stored.data).toEqual({ cid: 'bafy', userId: 'u1' }); + expect(stored.opts).toMatchObject({ attempts: 6, backoff: { type: 'exponential' } }); + }); + + it('rate-limits each user independently per queue', async () => { + const { manager } = build(); + const { max } = manager.definitions[QUEUE_NAMES.TX_MINTING].userRateLimit; + const now = 1_700_000_000_000; + + for (let i = 0; i < max; i++) { + await manager.assertWithinUserRateLimit(QUEUE_NAMES.TX_MINTING, 'alice', now); + } + await expect(manager.assertWithinUserRateLimit(QUEUE_NAMES.TX_MINTING, 'alice', now)).rejects.toBeInstanceOf( + QueueRateLimitError + ); + // Other users and other queues are unaffected. + await expect(manager.assertWithinUserRateLimit(QUEUE_NAMES.TX_MINTING, 'bob', now)).resolves.toBeUndefined(); + await expect(manager.assertWithinUserRateLimit(QUEUE_NAMES.IPFS_PINNING, 'alice', now)).resolves.toBeUndefined(); + // A new window resets the quota. + await expect( + manager.assertWithinUserRateLimit(QUEUE_NAMES.TX_MINTING, 'alice', now + 60_000) + ).resolves.toBeUndefined(); + }); + + it('reports retry-after on rate-limit errors', async () => { + const { manager } = build(); + const now = 1_700_000_010_000; + const { max } = manager.definitions[QUEUE_NAMES.TX_MINTING].userRateLimit; + for (let i = 0; i < max; i++) await manager.assertWithinUserRateLimit(QUEUE_NAMES.TX_MINTING, 'carol', now); + const err = await manager.assertWithinUserRateLimit(QUEUE_NAMES.TX_MINTING, 'carol', now).catch((e) => e); + expect(err).toBeInstanceOf(QueueRateLimitError); + expect(err.retryAfterMs).toBeGreaterThan(0); + expect(err.retryAfterMs).toBeLessThanOrEqual(60_000); + }); +}); + +describe('DlqProcessor (BE-006)', () => { + const setup = (alertThreshold = 100) => { + const dlq = new FakeQueue('dead-letter'); + const source = new FakeQueue(QUEUE_NAMES.IPFS_PINNING); + const alerts: DlqAlert[] = []; + let now = 1_700_000_000_000; + const processor = new DlqProcessor({ + queue: asQueue(dlq) as any, + resolveSourceQueue: (name) => (name === source.name ? asQueue(source) : undefined), + alert: (a) => { + alerts.push(a); + }, + alertThreshold, + thresholdAlertCooldownMs: 60_000, + now: () => now, + }); + return { dlq, source, alerts, processor, advance: (ms: number) => (now += ms) }; + }; + + const failedJob = (overrides: Record = {}) => + ({ + id: 'job-1', + name: 'pin', + data: { cid: 'bafy', userId: 'u1' }, + opts: { attempts: 6 }, + attemptsMade: 6, + finishedOn: Date.now(), + failedReason: 'boom', + stacktrace: ['Error: boom'], + ...overrides, + }) as any; + + it('ignores failures that BullMQ will still retry', async () => { + const { processor, dlq, alerts } = setup(); + const moved = await processor.handleFailure( + QUEUE_NAMES.IPFS_PINNING, + failedJob({ attemptsMade: 2, finishedOn: undefined }), + new Error('transient') + ); + expect(moved).toBe(false); + expect(dlq.jobs.size).toBe(0); + expect(alerts).toHaveLength(0); + }); + + it('dead-letters exhausted jobs with full context and raises an alert', async () => { + const { processor, dlq, alerts } = setup(); + const moved = await processor.handleFailure(QUEUE_NAMES.IPFS_PINNING, failedJob(), new Error('boom')); + + expect(moved).toBe(true); + const entry = await processor.get('ipfs-pinning--job-1'); + expect(entry).toMatchObject({ + sourceQueue: QUEUE_NAMES.IPFS_PINNING, + sourceJobId: 'job-1', + jobName: 'pin', + data: { cid: 'bafy', userId: 'u1' }, + attemptsMade: 6, + failedReason: 'boom', + userId: 'u1', + }); + expect(dlq.jobs.size).toBe(1); + expect(alerts).toEqual([expect.objectContaining({ kind: 'job-dead-lettered', queue: QUEUE_NAMES.IPFS_PINNING })]); + }); + + it('treats UnrecoverableError as terminal even with attempts remaining', () => { + const job = failedJob({ attemptsMade: 1, finishedOn: undefined }); + expect(DlqProcessor.isTerminalFailure(job, new UnrecoverableError('bad input'))).toBe(true); + expect(DlqProcessor.isTerminalFailure(job, new Error('flaky'))).toBe(false); + }); + + it('raises a throttled threshold alert when DLQ depth crosses the limit', async () => { + const { processor, alerts, advance } = setup(2); + await processor.handleFailure('ipfs-pinning', failedJob({ id: 'a' })); + await processor.handleFailure('ipfs-pinning', failedJob({ id: 'b' })); + await processor.handleFailure('ipfs-pinning', failedJob({ id: 'c' })); + expect(alerts.filter((a) => a.kind === 'threshold-exceeded')).toHaveLength(1); + + advance(60_001); + await processor.handleFailure('ipfs-pinning', failedJob({ id: 'd' })); + expect(alerts.filter((a) => a.kind === 'threshold-exceeded')).toHaveLength(2); + }); + + it('replays a dead-lettered job onto its source queue and removes it from the DLQ', async () => { + const { processor, source, dlq } = setup(); + await processor.handleFailure(QUEUE_NAMES.IPFS_PINNING, failedJob()); + + const result = await processor.replay('ipfs-pinning--job-1'); + expect(result).toMatchObject({ sourceQueue: QUEUE_NAMES.IPFS_PINNING }); + expect(dlq.jobs.size).toBe(0); + const [replayed] = [...source.jobs.values()]; + expect(replayed).toMatchObject({ name: 'pin', data: { cid: 'bafy', userId: 'u1' }, opts: { attempts: 6 } }); + }); + + it('returns null when replaying or inspecting an unknown id', async () => { + const { processor } = setup(); + await expect(processor.replay('missing')).resolves.toBeNull(); + await expect(processor.get('missing')).resolves.toBeNull(); + await expect(processor.purge('missing')).resolves.toBe(false); + }); + + it('refuses to replay into a queue it does not manage', async () => { + const { processor } = setup(); + await processor.handleFailure('legacy-queue', failedJob()); + await expect(processor.replay('legacy-queue--job-1')).rejects.toThrow(/not managed/); + }); + + it('lists, filters and purges dead-lettered jobs', async () => { + const { processor } = setup(); + await processor.handleFailure('ipfs-pinning', failedJob({ id: '1' })); + await processor.handleFailure('ipfs-pinning', failedJob({ id: '2' })); + await processor.handleFailure('email-notifications', failedJob({ id: '3' })); + + expect((await processor.list()).total).toBe(3); + const filtered = await processor.list({ sourceQueue: 'email-notifications' }); + expect(filtered.entries.map((e) => e.sourceJobId)).toEqual(['3']); + + await expect(processor.purgeAll('ipfs-pinning')).resolves.toEqual({ purged: 2 }); + expect((await processor.list()).total).toBe(1); + }); +}); + +/** + * End-to-end against a real Redis (provided in CI): a job that always fails + * is attempted exactly 6 times (1 + 5 retries) and then lands in the DLQ. + */ +describe('QueueManager + BullMQ integration (real Redis)', () => { + let redisAvailable = false; + let manager: QueueManager; + const alerts: DlqAlert[] = []; + const prefix = `test-qm-${randomUUID()}`; + + beforeAll(async () => { + const probe = new Redis(process.env.REDIS_URL || 'redis://localhost:6379', { + lazyConnect: true, + maxRetriesPerRequest: 0, + connectTimeout: 500, + retryStrategy: () => null, + }); + probe.on('error', () => undefined); + try { + await probe.connect(); + redisAvailable = (await probe.ping()) === 'PONG'; + } catch { + redisAvailable = false; + } finally { + probe.disconnect(); + } + if (!redisAvailable) return; + + manager = new QueueManager({ prefix, backoffDelayMs: 5 }); + const dlqQueue = manager.getQueue('dead-letter') as any; + // Swap in a DLQ processor whose alerts we can observe. + (manager as any).dlqProcessor = new DlqProcessor({ + queue: dlqQueue, + resolveSourceQueue: (name) => (manager.isManagedQueue(name) ? manager.getQueue(name) : undefined), + alert: (a) => { + alerts.push(a); + }, + }); + }); + + afterAll(async () => { + if (!redisAvailable) return; + for (const name of [...Object.values(QUEUE_NAMES), 'dead-letter'] as const) { + await manager.getQueue(name as any).obliterate({ force: true }).catch(() => undefined); + } + await manager.close(); + }); + + const waitFor = async (predicate: () => Promise, timeoutMs = 15_000) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await predicate()) return; + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error('Timed out waiting for condition'); + }; + + it('retries 5 times, then dead-letters the job; replay and purge work through the DLQ', async () => { + if (!redisAvailable) { + console.warn('Redis not reachable; skipping BullMQ integration test'); + return; + } + + let attempts = 0; + const worker = manager.createWorker(QUEUE_NAMES.IPFS_PINNING, async () => { + attempts++; + throw new Error('IPFS gateway unavailable'); + }); + + await manager.enqueue(QUEUE_NAMES.IPFS_PINNING, 'pin', { cid: 'bafy-int' }, { userId: 'int-user' }); + await waitFor(async () => (await manager.dlq.depth()) === 1); + + expect(attempts).toBe(MAX_RETRIES + 1); + const { entries } = await manager.dlq.list(); + expect(entries[0]).toMatchObject({ + sourceQueue: QUEUE_NAMES.IPFS_PINNING, + attemptsMade: 6, + failedReason: 'IPFS gateway unavailable', + data: { cid: 'bafy-int', userId: 'int-user' }, + }); + expect(alerts.some((a) => a.kind === 'job-dead-lettered')).toBe(true); + + await worker.close(); + const replayed = await manager.dlq.replay(entries[0]!.id); + expect(replayed).not.toBeNull(); + expect(await manager.dlq.depth()).toBe(0); + const requeued = await manager.getQueue(QUEUE_NAMES.IPFS_PINNING).getJob(replayed!.replayedJobId); + expect(requeued?.data).toMatchObject({ cid: 'bafy-int' }); + }); + + it('dead-letters UnrecoverableError failures after a single attempt', async () => { + if (!redisAvailable) return; + + let attempts = 0; + const worker = manager.createWorker(QUEUE_NAMES.EMAIL_NOTIFICATIONS, async () => { + attempts++; + throw new UnrecoverableError('invalid recipient'); + }); + await manager.enqueue(QUEUE_NAMES.EMAIL_NOTIFICATIONS, 'send', { to: 'nobody' }); + await waitFor(async () => (await manager.dlq.list({ sourceQueue: QUEUE_NAMES.EMAIL_NOTIFICATIONS })).total === 1); + expect(attempts).toBe(1); + await worker.close(); + + await expect(manager.dlq.purgeAll(QUEUE_NAMES.EMAIL_NOTIFICATIONS)).resolves.toEqual({ purged: 1 }); + }); +}); From 590360892fd414b08298797393f47810ca29ae84 Mon Sep 17 00:00:00 2001 From: emmzy4tech Date: Wed, 21 Oct 2026 17:34:22 +0100 Subject: [PATCH 3/5] feat(graphql): add authenticated graphql-ws subscriptions for course events (BE-007) Adds exerciseStatusUpdated, courseProgressChanged and newGlobalAnnouncement over graphql-ws on the shared Apollo schema, with JWT auth on connect, per-subscription authorization, token-expiry disconnects and listener cleanup on close. Co-Authored-By: Claude Opus 5.5 --- backend/package-lock.json | 1 + backend/package.json | 3 + backend/pnpm-lock.yaml | 3 + backend/src/graphql/context.ts | 39 ++- backend/src/graphql/pubsub.ts | 249 ++++++++++++++ backend/src/graphql/resolvers.ts | 13 + .../resolvers/subscriptionResolvers.ts | 111 +++++++ backend/src/graphql/schema.ts | 68 ++++ backend/src/graphql/server.ts | 20 +- backend/src/graphqlWsServer.ts | 307 ++++++++++-------- backend/tests/graphqlSubscriptions.test.ts | 273 ++++++++++++++++ 11 files changed, 938 insertions(+), 149 deletions(-) create mode 100644 backend/src/graphql/pubsub.ts create mode 100644 backend/src/graphql/resolvers/subscriptionResolvers.ts create mode 100644 backend/tests/graphqlSubscriptions.test.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index 1224653b..2a6b4912 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -12,6 +12,7 @@ "@aws-sdk/client-kms": "^3.1076.0", "@aws-sdk/client-s3": "^3.1117.0", "@aws-sdk/lib-storage": "^3.1076.0", + "@graphql-tools/schema": "^10.0.33", "@prisma/adapter-pg": "^7.10.0", "@prisma/client": "^7.10.0", "@prisma/config": "^7.8.0", diff --git a/backend/package.json b/backend/package.json index 5991b8e3..b3005722 100644 --- a/backend/package.json +++ b/backend/package.json @@ -20,6 +20,8 @@ "bench": "tsx benchmarks/runBenchmarks.ts", "collaboration": "tsx src/collaborationServer.ts", "collab:graphql": "tsx src/graphqlWsServer.ts", + "workers:queues": "tsx src/workers/queueWorkers.ts", + "start:workers": "node dist/workers/queueWorkers.js", "load:ws": "node scripts/ws-load-test.js", "db:seed": "tsx prisma/seed.ts", "migrate:locked": "tsx scripts/migrate-with-lock.ts" @@ -32,6 +34,7 @@ "@aws-sdk/client-kms": "^3.1076.0", "@aws-sdk/client-s3": "^3.1117.0", "@aws-sdk/lib-storage": "^3.1076.0", + "@graphql-tools/schema": "^10.0.33", "@prisma/adapter-pg": "^7.10.0", "@prisma/client": "^7.10.0", "@prisma/config": "^7.8.0", diff --git a/backend/pnpm-lock.yaml b/backend/pnpm-lock.yaml index d84df68f..606e51bd 100644 --- a/backend/pnpm-lock.yaml +++ b/backend/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@aws-sdk/lib-storage': specifier: ^3.1076.0 version: 3.1141.0(@aws-sdk/client-s3@3.1141.0) + '@graphql-tools/schema': + specifier: ^10.0.33 + version: 10.1.1(graphql@16.14.2) '@prisma/adapter-pg': specifier: ^7.10.0 version: 7.10.0 diff --git a/backend/src/graphql/context.ts b/backend/src/graphql/context.ts index 367d160f..1f64697e 100644 --- a/backend/src/graphql/context.ts +++ b/backend/src/graphql/context.ts @@ -1,16 +1,50 @@ import type { PrismaClient } from '@prisma/client'; +import type { IncomingHttpHeaders } from 'http'; import redisClient from '../cache/RedisClient.js'; +import { isAccessTokenBlacklisted, verifyAccessToken } from '../auth/token.service.js'; import { createLoaders, type GraphQLLoaders } from './loaders.js'; +export type GraphQLUser = { + id: string; + role?: string; + /** Seconds since epoch at which the bearer token stops being valid. */ + exp?: number; +}; + export type GraphQLContext = { prisma: PrismaClient; redis: unknown; /** Per-request DataLoaders that batch nested relation lookups. */ loaders: GraphQLLoaders; - user?: { id: string; email: string; name: string }; + user?: GraphQLUser | null; +}; + +/** + * Verifies an access token (signature, expiry, revocation) and returns the + * identity it carries, or null. Shared by the HTTP context and the + * graphql-ws `onConnect` handshake so both transports authenticate alike. + */ +export const resolveUserFromToken = async (rawToken: unknown): Promise => { + if (typeof rawToken !== 'string') return null; + const token = rawToken.replace(/^Bearer\s+/i, '').trim(); + if (!token) return null; + try { + const decoded = verifyAccessToken(token) as { userId?: string; id?: string; role?: string; exp?: number }; + const id = decoded.userId ?? decoded.id; + if (!id || (await isAccessTokenBlacklisted(token))) return null; + return { + id, + ...(decoded.role ? { role: decoded.role } : {}), + ...(decoded.exp ? { exp: decoded.exp } : {}), + }; + } catch { + return null; + } }; -export const createGraphQLContext = async (): Promise => { +export const createGraphQLContext = async ({ + req, +}: { req?: { headers?: IncomingHttpHeaders } } = {}): Promise => { const prismaModule = await import('../db/index.js'); // Prefer the fully-extended default client so workspace isolation, read // routing and field encryption all apply to loader-driven queries. @@ -19,5 +53,6 @@ export const createGraphQLContext = async (): Promise => { prisma, redis: redisClient.getClient(), loaders: createLoaders(prisma), + user: await resolveUserFromToken(req?.headers?.authorization), }; }; diff --git a/backend/src/graphql/pubsub.ts b/backend/src/graphql/pubsub.ts new file mode 100644 index 00000000..6a773ea1 --- /dev/null +++ b/backend/src/graphql/pubsub.ts @@ -0,0 +1,249 @@ +/** + * Typed GraphQL Pub/Sub — BE-007 + * + * Fan-out bus backing the GraphQL subscriptions. Topics are keyed by the + * subscription argument (`exercise:`, `course-progress:`) + * so a publish only touches the listeners that actually asked for it, and + * delivery to in-process subscribers is synchronous — the dominant cost of + * a push is the WebSocket write itself, well inside the 50ms budget. + * + * Producers usually run in other processes (API server, BullMQ workers), so + * when a Redis bridge is enabled every publish is also relayed over a Redis + * channel and re-dispatched locally by each subscriber process. Messages + * carry the origin instance id so a process never delivers its own publish + * twice. Redis being down degrades to local-only delivery; it never throws + * into the caller. + * + * Each subscription iterator keeps a bounded buffer (oldest events dropped + * first) so a slow client cannot grow server memory without bound, and + * `return()` — invoked by graphql-ws on `complete` or socket close — removes + * the listener immediately. + */ + +import { randomUUID } from 'crypto'; +import { Redis } from 'ioredis'; +import logger from '../utils/logger.js'; + +export type ExerciseStatus = 'QUEUED' | 'RUNNING' | 'PASSED' | 'FAILED' | 'ERROR'; +export type AnnouncementSeverity = 'INFO' | 'WARNING' | 'CRITICAL'; + +export interface ExerciseTestResultEvent { + name: string; + status: string; + gasUsed: number; + durationMs: number; + error?: string | null; +} + +export interface ExerciseStatusEvent { + studentId: string; + exerciseId: string; + jobId?: string | null; + status: ExerciseStatus; + gasUsed?: number | null; + gasLimit?: number | null; + terminationReason?: string | null; + passed?: number | null; + total?: number | null; + tests?: ExerciseTestResultEvent[] | null; + message?: string | null; + emittedAt: string; +} + +export interface CourseProgressEvent { + courseId: string; + studentId: string; + percentage: number; + status: string; + completedLessons: string[]; + emittedAt: string; +} + +export interface GlobalAnnouncementEvent { + id: string; + title: string; + message: string; + severity: AnnouncementSeverity; + publishedBy?: string | null; + emittedAt: string; +} + +export const topics = { + exerciseStatus: (studentId: string) => `exercise:${studentId}`, + courseProgress: (courseId: string) => `course-progress:${courseId}`, + courseUpdated: (courseId: string) => `course:${courseId}`, + announcements: () => 'announcements', +}; + +type Listener = (payload: unknown) => void; + +export interface SubscriptionIterator extends AsyncIterableIterator { + return(): Promise>; +} + +/** Events buffered per subscriber before the oldest are dropped. */ +export const SUBSCRIBER_BUFFER_LIMIT = 100; +const REDIS_CHANNEL = 'graphql:pubsub'; + +interface BridgeMessage { + origin: string; + topic: string; + payload: unknown; +} + +export class PubSub { + private readonly listeners = new Map>(); + private readonly instanceId = randomUUID(); + private publisher: Redis | null = null; + private subscriber: Redis | null = null; + private bridgeUrl: string | null = null; + + /** Relay publishes through Redis so subscribers in other processes receive them. */ + enableRedisBridge(url: string): void { + this.bridgeUrl = url; + } + + private getPublisher(): Redis | null { + if (!this.bridgeUrl) return null; + if (!this.publisher) { + this.publisher = new Redis(this.bridgeUrl, { lazyConnect: true, maxRetriesPerRequest: 1 }); + this.publisher.on('error', (err) => logger.warn('[graphql-pubsub] Redis publisher error', { error: err.message })); + } + return this.publisher; + } + + private ensureSubscriber(): void { + if (!this.bridgeUrl || this.subscriber) return; + const subscriber = new Redis(this.bridgeUrl, { lazyConnect: true, maxRetriesPerRequest: null }); + this.subscriber = subscriber; + subscriber.on('error', (err) => logger.warn('[graphql-pubsub] Redis subscriber error', { error: err.message })); + subscriber.on('message', (_channel: string, raw: string) => { + try { + const message = JSON.parse(raw) as BridgeMessage; + if (message.origin !== this.instanceId) this.dispatch(message.topic, message.payload); + } catch (err) { + logger.warn('[graphql-pubsub] Dropped malformed bridge message', { + error: err instanceof Error ? err.message : String(err), + }); + } + }); + subscriber.subscribe(REDIS_CHANNEL).catch((err: Error) => { + logger.warn('[graphql-pubsub] Redis subscribe failed; local delivery only', { error: err.message }); + }); + } + + private dispatch(topic: string, payload: unknown): void { + const subs = this.listeners.get(topic); + if (!subs) return; + for (const listener of [...subs]) { + try { + listener(payload); + } catch (err) { + logger.warn('[graphql-pubsub] Listener threw', { topic, error: err instanceof Error ? err.message : String(err) }); + } + } + } + + publish(topic: string, payload: T): void { + this.dispatch(topic, payload); + const publisher = this.getPublisher(); + if (publisher) { + const message: BridgeMessage = { origin: this.instanceId, topic, payload }; + publisher.publish(REDIS_CHANNEL, JSON.stringify(message)).catch((err: Error) => { + logger.warn('[graphql-pubsub] Redis publish failed; delivered locally only', { topic, error: err.message }); + }); + } + } + + listenerCount(topic: string): number { + return this.listeners.get(topic)?.size ?? 0; + } + + subscribe(topic: string, filter?: (payload: T) => boolean): SubscriptionIterator { + this.ensureSubscriber(); + + const buffer: T[] = []; + let pending: ((result: IteratorResult) => void) | null = null; + let done = false; + + const listener: Listener = (raw) => { + const payload = raw as T; + if (done || (filter && !filter(payload))) return; + if (pending) { + const resolve = pending; + pending = null; + resolve({ value: payload, done: false }); + return; + } + buffer.push(payload); + if (buffer.length > SUBSCRIBER_BUFFER_LIMIT) buffer.shift(); + }; + + const subs = this.listeners.get(topic) ?? new Set(); + subs.add(listener); + this.listeners.set(topic, subs); + + const cleanup = () => { + if (done) return; + done = true; + buffer.length = 0; + subs.delete(listener); + if (subs.size === 0) this.listeners.delete(topic); + if (pending) { + pending({ value: undefined, done: true }); + pending = null; + } + }; + + const iterator: SubscriptionIterator = { + next() { + if (buffer.length > 0) return Promise.resolve({ value: buffer.shift() as T, done: false }); + if (done) return Promise.resolve({ value: undefined, done: true }); + return new Promise>((resolve) => { + pending = resolve; + }); + }, + return() { + cleanup(); + return Promise.resolve({ value: undefined, done: true }); + }, + throw(error?: unknown) { + cleanup(); + return Promise.reject(error); + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + return iterator; + } + + async close(): Promise { + await Promise.allSettled([this.publisher?.quit(), this.subscriber?.quit()]); + this.publisher = null; + this.subscriber = null; + } +} + +export const pubsub = new PubSub(); + +const bridgeUrl = process.env.GRAPHQL_PUBSUB_REDIS_URL ?? (process.env.NODE_ENV === 'test' ? undefined : process.env.REDIS_URL); +if (bridgeUrl && process.env.GRAPHQL_PUBSUB_REDIS !== 'false') { + pubsub.enableRedisBridge(bridgeUrl); +} + +const now = () => new Date().toISOString(); + +export function publishExerciseStatus(event: Omit): void { + pubsub.publish(topics.exerciseStatus(event.studentId), { ...event, emittedAt: now() }); +} + +export function publishCourseProgress(event: Omit): void { + pubsub.publish(topics.courseProgress(event.courseId), { ...event, emittedAt: now() }); +} + +export function publishGlobalAnnouncement(event: Omit): GlobalAnnouncementEvent { + const announcement: GlobalAnnouncementEvent = { ...event, id: randomUUID(), emittedAt: now() }; + pubsub.publish(topics.announcements(), announcement); + return announcement; +} diff --git a/backend/src/graphql/resolvers.ts b/backend/src/graphql/resolvers.ts index 5f61d9bf..e74e978e 100644 --- a/backend/src/graphql/resolvers.ts +++ b/backend/src/graphql/resolvers.ts @@ -3,6 +3,8 @@ import redisClient from '../cache/RedisClient.js'; import prisma from '../db/index.js'; import logger from '../utils/logger.js'; import type { GraphQLContext } from './context.js'; +import { publishCourseProgress } from './pubsub.js'; +import { subscriptionMutationResolvers, subscriptionResolvers } from './resolvers/subscriptionResolvers.js'; const CACHE_TTL = 60; @@ -244,9 +246,18 @@ export const resolvers = { studentId: input.studentId, courseId: input.courseId, }); + publishCourseProgress({ + courseId: progress.courseId, + studentId: progress.studentId, + percentage: progress.percentage, + status: progress.status, + completedLessons, + }); return progress; }, + publishGlobalAnnouncement: subscriptionMutationResolvers.publishGlobalAnnouncement, + issueCertificate: async (_parent: unknown, { studentId, courseId }: { studentId: string; courseId: string }, context: GraphQLContext) => { const tokenId = `token-${crypto.randomUUID()}`; const certificate = await prisma.certificate.create({ @@ -278,6 +289,8 @@ export const resolvers = { }, }, + Subscription: subscriptionResolvers, + Student: { enrollments: (parent: { id: string }, _args: unknown, context: GraphQLContext) => context.loaders.enrollmentsByStudentId.load(parent.id), diff --git a/backend/src/graphql/resolvers/subscriptionResolvers.ts b/backend/src/graphql/resolvers/subscriptionResolvers.ts new file mode 100644 index 00000000..d2b18167 --- /dev/null +++ b/backend/src/graphql/resolvers/subscriptionResolvers.ts @@ -0,0 +1,111 @@ +/** + * GraphQL subscription resolvers — BE-007 + * + * Authorization happens at subscribe time using the identity graphql-ws + * established from the connection's JWT (see graphqlWsServer.ts): + * + * - exerciseStatusUpdated(studentId): the student themself, or an admin. + * - courseProgressChanged(courseId): admins see every student; anyone else + * only receives events about their own progress. + * - newGlobalAnnouncement: any authenticated connection. + */ + +import { GraphQLError } from 'graphql'; +import { + publishGlobalAnnouncement, + pubsub, + topics, + type AnnouncementSeverity, + type CourseProgressEvent, + type ExerciseStatusEvent, + type GlobalAnnouncementEvent, +} from '../pubsub.js'; + +export interface SubscriptionUser { + id: string; + role?: string; +} + +export interface ContextWithUser { + user?: SubscriptionUser | null; +} + +const isAdmin = (user: SubscriptionUser) => user.role?.toLowerCase() === 'admin'; + +export function requireUser(context: ContextWithUser): SubscriptionUser { + if (!context?.user?.id) { + throw new GraphQLError('Authentication required', { extensions: { code: 'UNAUTHENTICATED' } }); + } + return context.user; +} + +const forbidden = (message: string) => new GraphQLError(message, { extensions: { code: 'FORBIDDEN' } }); + +const ANNOUNCEMENT_SEVERITIES: AnnouncementSeverity[] = ['INFO', 'WARNING', 'CRITICAL']; +const MAX_ANNOUNCEMENT_TITLE = 200; +const MAX_ANNOUNCEMENT_MESSAGE = 5_000; + +export const subscriptionResolvers = { + courseUpdated: { + subscribe: (_parent: unknown, { courseId }: { courseId: string }, context: ContextWithUser) => { + requireUser(context); + return pubsub.subscribe(topics.courseUpdated(courseId)); + }, + resolve: (payload: unknown) => payload, + }, + + exerciseStatusUpdated: { + subscribe: (_parent: unknown, { studentId }: { studentId: string }, context: ContextWithUser) => { + const user = requireUser(context); + if (user.id !== studentId && !isAdmin(user)) { + throw forbidden('You can only subscribe to your own exercise updates'); + } + return pubsub.subscribe(topics.exerciseStatus(studentId)); + }, + resolve: (payload: ExerciseStatusEvent) => payload, + }, + + courseProgressChanged: { + subscribe: (_parent: unknown, { courseId }: { courseId: string }, context: ContextWithUser) => { + const user = requireUser(context); + const filter = isAdmin(user) ? undefined : (event: CourseProgressEvent) => event.studentId === user.id; + return pubsub.subscribe(topics.courseProgress(courseId), filter); + }, + resolve: (payload: CourseProgressEvent) => payload, + }, + + newGlobalAnnouncement: { + subscribe: (_parent: unknown, _args: unknown, context: ContextWithUser) => { + requireUser(context); + return pubsub.subscribe(topics.announcements()); + }, + resolve: (payload: GlobalAnnouncementEvent) => payload, + }, +}; + +export const subscriptionMutationResolvers = { + publishGlobalAnnouncement: ( + _parent: unknown, + { input }: { input: { title: string; message: string; severity?: AnnouncementSeverity | null } }, + context: ContextWithUser + ): GlobalAnnouncementEvent => { + const user = requireUser(context); + if (!isAdmin(user)) throw forbidden('Admin access required'); + + const title = input.title.trim(); + const message = input.message.trim(); + if (!title || title.length > MAX_ANNOUNCEMENT_TITLE) { + throw new GraphQLError(`title must be 1-${MAX_ANNOUNCEMENT_TITLE} characters`, { + extensions: { code: 'BAD_USER_INPUT' }, + }); + } + if (!message || message.length > MAX_ANNOUNCEMENT_MESSAGE) { + throw new GraphQLError(`message must be 1-${MAX_ANNOUNCEMENT_MESSAGE} characters`, { + extensions: { code: 'BAD_USER_INPUT' }, + }); + } + const severity = input.severity && ANNOUNCEMENT_SEVERITIES.includes(input.severity) ? input.severity : 'INFO'; + + return publishGlobalAnnouncement({ title, message, severity, publishedBy: user.id }); + }, +}; diff --git a/backend/src/graphql/schema.ts b/backend/src/graphql/schema.ts index 448c3ac4..5282e51d 100644 --- a/backend/src/graphql/schema.ts +++ b/backend/src/graphql/schema.ts @@ -102,6 +102,8 @@ export const typeDefs = ` enrollStudent(input: EnrollmentInput!): Enrollment! updateLearningProgress(input: ProgressUpdateInput!): LearningProgress! issueCertificate(studentId: ID!, courseId: ID!): Certificate! + "Broadcasts an announcement to every newGlobalAnnouncement subscriber. Admin only." + publishGlobalAnnouncement(input: GlobalAnnouncementInput!): GlobalAnnouncement! } type HealthResponse { @@ -111,7 +113,73 @@ export const typeDefs = ` redis: String! } + enum ExerciseStatus { + QUEUED + RUNNING + PASSED + FAILED + ERROR + } + + type ExerciseTestResult { + name: String! + status: String! + gasUsed: Int! + durationMs: Float! + error: String + } + + type ExerciseStatusEvent { + studentId: ID! + exerciseId: ID! + jobId: ID + status: ExerciseStatus! + gasUsed: Int + gasLimit: Int + terminationReason: String + passed: Int + total: Int + tests: [ExerciseTestResult!] + message: String + emittedAt: String! + } + + type CourseProgressEvent { + courseId: ID! + studentId: ID! + percentage: Int! + status: String! + completedLessons: [String!]! + emittedAt: String! + } + + enum AnnouncementSeverity { + INFO + WARNING + CRITICAL + } + + type GlobalAnnouncement { + id: ID! + title: String! + message: String! + severity: AnnouncementSeverity! + publishedBy: ID + emittedAt: String! + } + + input GlobalAnnouncementInput { + title: String! + message: String! + severity: AnnouncementSeverity + } + type Subscription { courseUpdated(courseId: ID!): Course! + "Sandbox evaluation lifecycle for one student. Students may only subscribe to their own id." + exerciseStatusUpdated(studentId: ID!): ExerciseStatusEvent! + "Progress changes within a course. Non-admin subscribers only receive their own events." + courseProgressChanged(courseId: ID!): CourseProgressEvent! + newGlobalAnnouncement: GlobalAnnouncement! } `; diff --git a/backend/src/graphql/server.ts b/backend/src/graphql/server.ts index d386e84e..a3b81d4a 100644 --- a/backend/src/graphql/server.ts +++ b/backend/src/graphql/server.ts @@ -1,4 +1,5 @@ -import { ApolloServer } from '@apollo/server'; +import { ApolloServer, type ApolloServerPlugin } from '@apollo/server'; +import type { GraphQLSchema } from 'graphql'; import { expressMiddleware } from '@as-integrations/express4'; import { json, type RequestHandler } from 'express'; import cors from 'cors'; @@ -11,10 +12,16 @@ import logger from '../utils/logger.js'; import { depthLimitRule, complexityLimitRule } from './validationRules.js'; import config from '../config/env.config.js'; -export const createGraphQLServer = async () => { +export interface GraphQLServerOptions { + /** Pre-built executable schema, so HTTP and WebSocket transports share one instance. */ + schema?: GraphQLSchema; + plugins?: ApolloServerPlugin[]; +} + +export const createGraphQLServer = async (options: GraphQLServerOptions = {}) => { + const schemaConfig = options.schema ? { schema: options.schema } : { typeDefs, resolvers }; const server = new ApolloServer({ - typeDefs, - resolvers, + ...schemaConfig, introspection: !['production', 'staging'].includes(process.env.NODE_ENV || 'development'), validationRules: [ depthLimitRule(() => config.graphql?.maxDepth ?? 6), @@ -30,6 +37,7 @@ export const createGraphQLServer = async () => { }; }, }, + ...(options.plugins ?? []), ], }); @@ -38,8 +46,8 @@ export const createGraphQLServer = async () => { return server; }; -export const graphQLMiddleware = async (): Promise => { - const server = await createGraphQLServer(); +export const graphQLMiddleware = async (options: GraphQLServerOptions = {}): Promise => { + const server = await createGraphQLServer(options); return [ json(), diff --git a/backend/src/graphqlWsServer.ts b/backend/src/graphqlWsServer.ts index 0933b860..43ccd94d 100644 --- a/backend/src/graphqlWsServer.ts +++ b/backend/src/graphqlWsServer.ts @@ -1,172 +1,197 @@ +/** + * GraphQL over WebSocket server — BE-007 + * + * Serves the full schema over two transports from one HTTP server: + * - HTTP POST /graphql → Apollo Server (queries & mutations) + * - WS /graphql → graphql-ws (subscriptions; queries/mutations too) + * + * Both transports share one executable schema, so the subscription resolvers + * in `graphql/resolvers/subscriptionResolvers.ts` are live on either. + * + * Connection lifecycle: + * - Authentication: `connection_init` must carry a JWT access token in + * `connectionParams.authorization` (or `.token`). Missing, invalid, + * expired or revoked tokens are rejected with close code 4403 before any + * operation is accepted; sockets that never send `connection_init` are + * closed after `connectionInitWaitTimeout`. + * - Token expiry: an authenticated socket is closed with 4401 when its + * token expires, so a revoked session cannot keep listening forever. + * - Liveness: the server pings every `keepAliveMs` and terminates sockets + * that stop answering, which in turn fires `onClose`. + * - Cleanup: on `complete` or socket close graphql-ws calls `return()` on + * every active subscription iterator, which unregisters its pub/sub + * listener immediately (see graphql/pubsub.ts). + * + * Run standalone with `npm run collab:graphql`. + */ + +import { makeExecutableSchema } from '@graphql-tools/schema'; +import { ApolloServerPluginDrainHttpServer } from '@apollo/server/plugin/drainHttpServer'; import express from 'express'; -import { buildSchema, execute, subscribe, validate, type ExecutionArgs } from 'graphql'; +import { execute, subscribe, validate, type ExecutionArgs, type GraphQLSchema } from 'graphql'; import { useServer } from 'graphql-ws/use/ws'; import http from 'http'; -import jwt from 'jsonwebtoken'; +import type { AddressInfo } from 'net'; import { WebSocketServer } from 'ws'; +import config from './config/env.config.js'; +import { resolveUserFromToken, type GraphQLUser } from './graphql/context.js'; +import { resolvers } from './graphql/resolvers.js'; import { typeDefs } from './graphql/schema.js'; import { graphQLMiddleware } from './graphql/server.js'; -import { depthLimitRule } from './graphql/validationRules.js'; -import config from './config/env.config.js'; +import { complexityLimitRule, depthLimitRule } from './graphql/validationRules.js'; +import logger from './utils/logger.js'; -const PORT = Number(process.env.PORT || process.env.WS_PORT || 4001); -const JWT_SECRET = process.env.JWT_SECRET || 'dev-secret'; const MAX_DEPTH = config.graphql?.maxDepth ?? 6; - -class PubSub { - private events = new Map void>>(); - - publish(topic: string, payload: any) { - const subs = this.events.get(topic); - if (!subs) return; - for (const cb of subs) cb(payload); - } - - subscribe(topic: string) { - const set = this.events.get(topic) ?? new Set(); - this.events.set(topic, set); - const queue: any[] = []; - let pullResolve: ((v: IteratorResult) => void) | null = null; - - const push = (value: any) => { - if (pullResolve) { - pullResolve({ value, done: false }); - pullResolve = null; - } else { - queue.push(value); - } - }; - - const cb = (payload: any) => push(payload); - set.add(cb); - - const asyncIterator = { - async next() { - if (queue.length) return { value: queue.shift(), done: false }; - return await new Promise>(res => (pullResolve = res)); - }, - return() { - set.delete(cb); - return Promise.resolve({ value: undefined, done: true }); - }, - throw(error: any) { - return Promise.reject(error); - }, - [Symbol.asyncIterator]() { - return this; - }, - }; - - return asyncIterator; - } +const MAX_COMPLEXITY = config.graphql?.maxComplexity ?? 500; + +/** Close codes (4400-4499 are reserved by graphql-ws for application use). */ +export const WS_CLOSE_UNAUTHORIZED = 4403; +export const WS_CLOSE_TOKEN_EXPIRED = 4401; + +export interface GraphqlWsServerOptions { + port?: number; + /** Ping interval used to detect and drop dead sockets. */ + keepAliveMs?: number; + /** How long a socket may stay open without sending `connection_init`. */ + connectionInitWaitTimeoutMs?: number; + schema?: GraphQLSchema; } -const pubsub = new PubSub(); +export interface GraphqlWsServerHandle { + httpServer: http.Server; + wsServer: WebSocketServer; + port: number; + stats: () => { connections: number; sockets: number }; + close: () => Promise; +} -const schema = buildSchema(typeDefs); +type ConnectionExtra = { user: GraphQLUser; expiryTimer: NodeJS.Timeout }; -const rootValue: Record = { - courseUpdated: ({ courseId }: { courseId: string }) => pubsub.subscribe(`course:${courseId}`), -}; +export const buildExecutableSchema = () => makeExecutableSchema({ typeDefs, resolvers }); /** * graphql-ws calls `execute`/`subscribe` directly, bypassing Apollo's - * validation rules. Validate the document first so the WS path enforces the - * same query-depth cap as the HTTP path (#1424 / BE-HARD-33). + * validation rules. Validate first so the WS path enforces the same depth and + * complexity caps as the HTTP path (#1424 / BE-HARD-33). */ -const withDepthLimit = (runner: (args: T) => any) => { - return (args: T) => { - const errors = validate(args.schema, args.document, [depthLimitRule(() => MAX_DEPTH)]); - if (errors.length > 0) { - return { errors }; - } - return runner(args); - }; +const withValidationRules = (runner: (args: ExecutionArgs) => any) => (args: ExecutionArgs) => { + const errors = validate(args.schema, args.document, [ + depthLimitRule(() => MAX_DEPTH), + complexityLimitRule(() => MAX_COMPLEXITY), + ]); + if (errors.length > 0) return { errors }; + return runner(args); }; -const executeWithDepthLimit = withDepthLimit(execute); -const subscribeWithDepthLimit = withDepthLimit(subscribe); - -const app = express(); - -app.get('/health', (_req, res) => res.status(200).send('ok')); - -(async () => { - const graphqlHandlers = await graphQLMiddleware(); - // mount middleware array returned by graphQLMiddleware at /graphql - app.use('/graphql', ...graphqlHandlers); - - const server = http.createServer(app); - - const wsServer = new WebSocketServer({ server, path: '/graphql' }); +export async function createGraphqlWsServer(options: GraphqlWsServerOptions = {}): Promise { + const schema = options.schema ?? buildExecutableSchema(); + const app = express(); + const httpServer = http.createServer(app); + const wsServer = new WebSocketServer({ server: httpServer, path: '/graphql', maxPayload: 64 * 1024 }); - // per-client backpressure queue limit - const QUEUE_LIMIT = 32; + let connections = 0; - useServer( + const serverCleanup = useServer, ConnectionExtra>( { schema, - execute: executeWithDepthLimit, - subscribe: subscribeWithDepthLimit, - roots: { subscription: rootValue, query: rootValue, mutation: rootValue } as any, - onConnect: async (ctx: any) => { - const connectionParams = (ctx.connectionParams || {}) as Record; - const token = (connectionParams.authorization || connectionParams.token || '').replace(/^Bearer\s+/i, ''); - if (!token) throw new Error('Missing auth token'); - try { - const user = jwt.verify(token, JWT_SECRET); - return { user }; - } catch (err) { - throw new Error('Unauthorized'); + execute: withValidationRules(execute), + subscribe: withValidationRules(subscribe), + connectionInitWaitTimeout: options.connectionInitWaitTimeoutMs ?? 5_000, + context: (ctx) => ({ user: ctx.extra.user ?? null }), + onConnect: async (ctx) => { + const params = ctx.connectionParams ?? {}; + const user = await resolveUserFromToken(params.authorization ?? params.token); + if (!user) { + // Returning false closes the socket with 4403 Forbidden. + return false; } - }, - onSubscribe: async (ctx: any, msg: any) => { - // multiplexing: allow clients to subscribe to different rooms via variables - return msg.payload; - }, - onNext: (ctx: any, msg: any, args: any) => { - // no-op: handled by graphql execution - }, - onError: (_ctx: any, _id: any, _payload: any, err: any) => { - console.error('WS error', err); - }, - onClose: (ctx: any, code: any, reason: any) => { - // cleanup handled by graphql-ws - }, - // customize sending to add backpressure dropping stale frames - sendMessage: (socket: any, message: any) => { - // attach a small queue on the socket - const qSymbol = Symbol.for('gqlws_queue'); - // @ts-ignore - if (!socket[qSymbol]) socket[qSymbol] = []; - // @ts-ignore - const q = socket[qSymbol] as any[]; - q.push(message); - if (q.length > QUEUE_LIMIT) { - // drop oldest - q.shift(); + ctx.extra.user = user; + if (user.exp) { + const msUntilExpiry = user.exp * 1000 - Date.now(); + ctx.extra.expiryTimer = setTimeout(() => { + ctx.extra.socket.close(WS_CLOSE_TOKEN_EXPIRED, 'Token expired'); + }, Math.max(0, msUntilExpiry)); + ctx.extra.expiryTimer.unref?.(); } - // if socket is ready, flush - // @ts-ignore - if (socket.readyState === socket.OPEN) { - // flush all - // @ts-ignore - while (q.length) socket.send(JSON.stringify(q.shift())); + connections++; + return true; + }, + onClose: (ctx, code, reason) => { + if (ctx.extra.expiryTimer) clearTimeout(ctx.extra.expiryTimer); + if (ctx.extra.user) { + connections = Math.max(0, connections - 1); + logger.debug?.('[graphql-ws] connection closed', { userId: ctx.extra.user.id, code, reason }); } - return Promise.resolve(); }, - } as any, - wsServer + onError: (_ctx, id, _payload, errors) => { + logger.warn('[graphql-ws] operation error', { id, errors: errors.map((e) => e.message) }); + }, + }, + wsServer, + options.keepAliveMs ?? 12_000 ); - server.listen(PORT, () => console.log(`GraphQL WS server listening on ${PORT}`)); + const graphqlHandlers = await graphQLMiddleware({ + schema, + plugins: [ + ApolloServerPluginDrainHttpServer({ httpServer }), + { + async serverWillStart() { + return { + async drainServer() { + await serverCleanup.dispose(); + }, + }; + }, + }, + ], + }); - // expose a tiny API to publish to rooms (used by other parts of the app) - app.post('/publish/:room', express.json(), (req, res) => { - const room = req.params.room; - pubsub.publish(room, req.body.payload ?? req.body); - res.status(204).end(); + const openSockets = () => { + let total = 0; + for (const client of wsServer.clients) { + if (client.readyState === client.OPEN) total++; + } + return total; + }; + + app.get('/health', (_req, res) => { + res.status(200).json({ status: 'ok', connections, sockets: openSockets() }); }); -})(); + app.use('/graphql', ...graphqlHandlers); -export { }; + await new Promise((resolve) => httpServer.listen(options.port ?? 0, resolve)); + const port = (httpServer.address() as AddressInfo).port; + + return { + httpServer, + wsServer, + port, + stats: () => ({ connections, sockets: openSockets() }), + close: async () => { + await serverCleanup.dispose(); + for (const client of wsServer.clients) client.terminate(); + await new Promise((resolve) => wsServer.close(() => resolve())); + await new Promise((resolve) => httpServer.close(() => resolve())); + }, + }; +} + +const isEntrypoint = /graphqlWsServer\.(ts|js)$/.test(process.argv[1] ?? ''); + +if (isEntrypoint) { + const port = Number(process.env.PORT || process.env.WS_PORT || 4001); + createGraphqlWsServer({ port }) + .then((server) => { + logger.info(`GraphQL HTTP + WS server listening on ${server.port}`); + const shutdown = () => { + server.close().finally(() => process.exit(0)); + }; + process.once('SIGTERM', shutdown); + process.once('SIGINT', shutdown); + }) + .catch((err) => { + logger.error('Failed to start GraphQL WS server', err); + process.exit(1); + }); +} diff --git a/backend/tests/graphqlSubscriptions.test.ts b/backend/tests/graphqlSubscriptions.test.ts new file mode 100644 index 00000000..61d06cf6 --- /dev/null +++ b/backend/tests/graphqlSubscriptions.test.ts @@ -0,0 +1,273 @@ +import { afterAll, beforeAll, describe, expect, it, jest } from '@jest/globals'; +import jwt from 'jsonwebtoken'; +import { performance } from 'perf_hooks'; +import WebSocket from 'ws'; +import { createClient, type Client } from 'graphql-ws'; + +/** + * BE-007: GraphQL subscriptions over graphql-ws — JWT auth on connect, + * per-subscription authorization, sub-50ms push and listener cleanup. + */ + +jest.mock('../src/db/index.js', () => ({ __esModule: true, default: {}, prisma: {} })); +jest.mock('../src/cache/RedisClient.js', () => ({ __esModule: true, default: { getClient: () => null } })); +jest.mock('../src/utils/logger.js', () => ({ + __esModule: true, + default: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }, +})); + +const SECRET = 'graphql-ws-test-secret'; +process.env.ACCESS_TOKEN_SECRET = SECRET; + +import { createGraphqlWsServer, WS_CLOSE_UNAUTHORIZED, type GraphqlWsServerHandle } from '../src/graphqlWsServer.js'; +import { + publishCourseProgress, + publishExerciseStatus, + PubSub, + pubsub, + SUBSCRIBER_BUFFER_LIMIT, + topics, +} from '../src/graphql/pubsub.js'; + +jest.setTimeout(20_000); + +const sign = (claims: Record, options: jwt.SignOptions = { expiresIn: '15m' }) => + jwt.sign(claims, SECRET, options); + +const studentToken = sign({ userId: 'student-1' }); +const otherStudentToken = sign({ userId: 'student-2' }); +const adminToken = sign({ userId: 'admin-1', role: 'admin' }); + +const waitFor = async (predicate: () => boolean, timeoutMs = 3_000) => { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() > deadline) throw new Error('Timed out waiting for condition'); + await new Promise((r) => setTimeout(r, 5)); + } +}; + +describe('PubSub', () => { + it('delivers to matching subscribers and stops after return()', async () => { + const bus = new PubSub(); + const it1 = bus.subscribe<{ n: number }>('t'); + bus.publish('t', { n: 1 }); + await expect(it1.next()).resolves.toEqual({ value: { n: 1 }, done: false }); + + expect(bus.listenerCount('t')).toBe(1); + await it1.return(); + expect(bus.listenerCount('t')).toBe(0); + await expect(it1.next()).resolves.toMatchObject({ done: true }); + }); + + it('applies the per-subscriber filter', async () => { + const bus = new PubSub(); + const iterator = bus.subscribe<{ owner: string }>('t', (p) => p.owner === 'me'); + bus.publish('t', { owner: 'you' }); + bus.publish('t', { owner: 'me' }); + await expect(iterator.next()).resolves.toEqual({ value: { owner: 'me' }, done: false }); + await iterator.return(); + }); + + it('bounds the buffer of a slow subscriber by dropping the oldest events', async () => { + const bus = new PubSub(); + const iterator = bus.subscribe('t'); + for (let i = 0; i < SUBSCRIBER_BUFFER_LIMIT + 10; i++) bus.publish('t', i); + await expect(iterator.next()).resolves.toEqual({ value: 10, done: false }); + await iterator.return(); + }); + + it('resolves a pending next() when the subscription is closed', async () => { + const bus = new PubSub(); + const iterator = bus.subscribe('t'); + const pending = iterator.next(); + await iterator.return(); + await expect(pending).resolves.toMatchObject({ done: true }); + }); +}); + +describe('graphql-ws server', () => { + let server: GraphqlWsServerHandle; + let url: string; + const clients: Client[] = []; + + const connect = (token?: string, closed?: (event: unknown) => void) => { + const client = createClient({ + url, + webSocketImpl: WebSocket, + retryAttempts: 0, + lazy: true, + connectionParams: token ? { authorization: `Bearer ${token}` } : {}, + ...(closed ? { on: { closed } } : {}), + }); + clients.push(client); + return client; + }; + + beforeAll(async () => { + server = await createGraphqlWsServer({ port: 0, keepAliveMs: 1_000 }); + url = `ws://127.0.0.1:${server.port}/graphql`; + }); + + afterAll(async () => { + await Promise.allSettled(clients.map((c) => c.dispose())); + await server.close(); + }); + + const EXERCISE_SUB = `subscription ($studentId: ID!) { + exerciseStatusUpdated(studentId: $studentId) { studentId exerciseId status gasUsed passed total tests { name status } emittedAt } + }`; + + it('rejects connections without a JWT', async () => { + let closeCode: number | undefined; + const client = connect(undefined, (event) => { + closeCode = (event as { code: number }).code; + }); + const iterator = client.iterate({ query: EXERCISE_SUB, variables: { studentId: 'student-1' } }); + await expect(iterator.next()).rejects.toBeDefined(); + expect(closeCode).toBe(WS_CLOSE_UNAUTHORIZED); + }); + + it('rejects connections with a forged or expired JWT', async () => { + const forged = jwt.sign({ userId: 'student-1' }, 'wrong-secret'); + const expired = sign({ userId: 'student-1', exp: Math.floor(Date.now() / 1000) - 60 }, {}); + for (const token of [forged, expired]) { + let closeCode: number | undefined; + const client = connect(token, (event) => { + closeCode = (event as { code: number }).code; + }); + const iterator = client.iterate({ query: EXERCISE_SUB, variables: { studentId: 'student-1' } }); + await expect(iterator.next()).rejects.toBeDefined(); + expect(closeCode).toBe(WS_CLOSE_UNAUTHORIZED); + } + }); + + it('pushes exerciseStatusUpdated to the subscribed student in under 50ms', async () => { + const client = connect(studentToken); + const iterator = client.iterate({ query: EXERCISE_SUB, variables: { studentId: 'student-1' } }); + const first = iterator.next(); + await waitFor(() => pubsub.listenerCount(topics.exerciseStatus('student-1')) === 1); + + const latencies: number[] = []; + let next = first; + for (let i = 0; i < 5; i++) { + const started = performance.now(); + publishExerciseStatus({ + studentId: 'student-1', + exerciseId: 'ex-1', + status: 'PASSED', + gasUsed: 752, + passed: 1, + total: 1, + tests: [{ name: 'adds', status: 'passed', gasUsed: 752, durationMs: 0.1 }], + }); + const { value } = await next; + latencies.push(performance.now() - started); + expect((value as any).data.exerciseStatusUpdated).toMatchObject({ + studentId: 'student-1', + status: 'PASSED', + gasUsed: 752, + tests: [{ name: 'adds', status: 'passed' }], + }); + next = iterator.next(); + } + + expect(Math.max(...latencies)).toBeLessThan(50); + await iterator.return?.(); + }); + + it("forbids subscribing to another student's exercise updates", async () => { + const client = connect(otherStudentToken); + const iterator = client.iterate({ query: EXERCISE_SUB, variables: { studentId: 'student-1' } }); + const { value } = await iterator.next(); + expect((value as any).errors[0]).toMatchObject({ + message: expect.stringMatching(/own/), + extensions: { code: 'FORBIDDEN' }, + }); + expect(pubsub.listenerCount(topics.exerciseStatus('student-1'))).toBe(0); + }); + + it('lets admins subscribe to any student', async () => { + const client = connect(adminToken); + const iterator = client.iterate({ query: EXERCISE_SUB, variables: { studentId: 'student-1' } }); + const next = iterator.next(); + await waitFor(() => pubsub.listenerCount(topics.exerciseStatus('student-1')) === 1); + publishExerciseStatus({ studentId: 'student-1', exerciseId: 'ex-2', status: 'RUNNING' }); + await expect(next).resolves.toMatchObject({ value: { data: { exerciseStatusUpdated: { status: 'RUNNING' } } } }); + await iterator.return?.(); + }); + + it('only sends a student their own courseProgressChanged events', async () => { + const query = `subscription { courseProgressChanged(courseId: "course-1") { studentId percentage } }`; + const client = connect(studentToken); + const iterator = client.iterate({ query }); + const next = iterator.next(); + await waitFor(() => pubsub.listenerCount(topics.courseProgress('course-1')) === 1); + + const base = { courseId: 'course-1', status: 'in_progress', completedLessons: ['l1'] }; + publishCourseProgress({ ...base, studentId: 'student-2', percentage: 90 }); + publishCourseProgress({ ...base, studentId: 'student-1', percentage: 10 }); + + await expect(next).resolves.toMatchObject({ + value: { data: { courseProgressChanged: { studentId: 'student-1', percentage: 10 } } }, + }); + await iterator.return?.(); + }); + + it('broadcasts admin announcements to every subscriber and blocks non-admin publishers', async () => { + const listener = connect(studentToken); + const iterator = listener.iterate({ query: `subscription { newGlobalAnnouncement { title severity publishedBy } }` }); + const next = iterator.next(); + await waitFor(() => pubsub.listenerCount(topics.announcements()) === 1); + + const mutation = `mutation { publishGlobalAnnouncement(input: { title: "Maintenance", message: "Tonight 22:00 UTC", severity: WARNING }) { id } }`; + + const denied = await connect(studentToken).iterate({ query: mutation }).next(); + expect((denied.value as any).errors[0].extensions.code).toBe('FORBIDDEN'); + + const published = await connect(adminToken).iterate({ query: mutation }).next(); + expect((published.value as any).data.publishGlobalAnnouncement.id).toBeTruthy(); + + await expect(next).resolves.toMatchObject({ + value: { data: { newGlobalAnnouncement: { title: 'Maintenance', severity: 'WARNING', publishedBy: 'admin-1' } } }, + }); + await iterator.return?.(); + }); + + it('removes subscription listeners when the client completes the operation', async () => { + const client = connect(studentToken); + const iterator = client.iterate({ query: EXERCISE_SUB, variables: { studentId: 'student-1' } }); + void iterator.next(); + await waitFor(() => pubsub.listenerCount(topics.exerciseStatus('student-1')) === 1); + + await iterator.return?.(); + await waitFor(() => pubsub.listenerCount(topics.exerciseStatus('student-1')) === 0); + }); + + it('removes subscription listeners and the socket when the client disconnects', async () => { + const client = connect(studentToken); + const iterator = client.iterate({ query: `subscription { newGlobalAnnouncement { id } }` }); + void iterator.next().catch(() => undefined); + await waitFor(() => pubsub.listenerCount(topics.announcements()) === 1); + const connectionsBefore = server.stats().connections; + + await client.dispose(); + await waitFor(() => pubsub.listenerCount(topics.announcements()) === 0); + await waitFor(() => server.stats().connections === connectionsBefore - 1); + }); + + it('closes the socket when the JWT expires', async () => { + let closeCode: number | undefined; + const shortLived = sign({ userId: 'student-1', exp: Math.floor(Date.now() / 1000) + 1 }, {}); + const client = connect(shortLived, (event) => { + closeCode = (event as { code: number }).code; + }); + const iterator = client.iterate({ query: `subscription { newGlobalAnnouncement { id } }` }); + const next = iterator.next().catch(() => undefined); + await waitFor(() => pubsub.listenerCount(topics.announcements()) === 1); + + await waitFor(() => closeCode !== undefined, 3_000); + expect(closeCode).toBe(4401); + await next; + await waitFor(() => pubsub.listenerCount(topics.announcements()) === 0); + }); +}); From e7024d03a2b399babf9b186105a6b1fee56d8cdb Mon Sep 17 00:00:00 2001 From: emmzy4tech Date: Wed, 21 Oct 2026 17:34:22 +0100 Subject: [PATCH 4/5] feat(backend): wire contract evaluation worker, queue worker entrypoint and routes Evaluation jobs run in the wasm sandbox and stream status through the exerciseStatusUpdated subscription. Mounts /evaluations and /admin/queues. Co-Authored-By: Claude Opus 5.5 --- backend/src/index.ts | 3 + backend/src/routes/evaluations.routes.ts | 113 ++++++++++++++ backend/src/routes/index.ts | 4 + backend/src/workers/evaluationWorker.ts | 146 ++++++++++++++++++ backend/src/workers/queueWorkers.ts | 38 +++++ backend/tests/queueWorkers.test.ts | 185 +++++++++++++++++++++++ 6 files changed, 489 insertions(+) create mode 100644 backend/src/routes/evaluations.routes.ts create mode 100644 backend/src/workers/evaluationWorker.ts create mode 100644 backend/src/workers/queueWorkers.ts create mode 100644 backend/tests/queueWorkers.test.ts diff --git a/backend/src/index.ts b/backend/src/index.ts index c5a98176..77b7b923 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -24,6 +24,9 @@ app.use(getSentryRequestHandler()); app.use(createCorsMiddleware()); +// Contract evaluation submissions carry base64 WASM, so they may use the full +// 1MB body ceiling enforced by jsonBodySizeLimit instead of the 100kb default. +app.use('/api/v1/evaluations', express.json({ limit: '1mb' })); app.use(express.json()); app.use(express.urlencoded({ extended: true })); app.use(jsonBodySizeLimit); diff --git a/backend/src/routes/evaluations.routes.ts b/backend/src/routes/evaluations.routes.ts new file mode 100644 index 00000000..79fb0c9b --- /dev/null +++ b/backend/src/routes/evaluations.routes.ts @@ -0,0 +1,113 @@ +import { Router, Request, Response } from 'express'; +import { z } from 'zod'; +import { authenticate } from '../auth/auth.middleware.js'; +import { getQueueManager, QUEUE_NAMES, QueueRateLimitError } from '../queues/queueManager.js'; +import { MAX_TEST_CASES } from '../services/wasmExecutionSandbox.js'; +import { MAX_WASM_MODULE_BYTES } from '../utils/wasmValidator.js'; +import { validateRequest } from '../utils/validation.js'; +import { enqueueEvaluation, type EvaluationJobData } from '../workers/evaluationWorker.js'; +import logger from '../utils/logger.js'; + +/** + * Sandboxed contract evaluation (BE-010). Mounted at /api/v1/evaluations. + * + * Submissions are queued rather than run inline so a burst of students can + * never exhaust the API process; progress is streamed through the + * `exerciseStatusUpdated` GraphQL subscription and the final structured + * result can be fetched from GET /:jobId. + */ + +const router: ReturnType = Router(); + +const MAX_BASE64_CHARS = Math.min(1_000_000, Math.ceil((MAX_WASM_MODULE_BYTES * 4) / 3) + 4); + +const wasmValue = z.union([z.number().finite(), z.string().max(64)]); + +export const evaluationSubmissionSchema = z.object({ + exerciseId: z.string().trim().min(1).max(128), + // Bounded by the 1MB request body ceiling (≈730KB of bytecode); the + // sandbox validator separately caps decoded modules at MAX_WASM_MODULE_BYTES. + wasmBase64: z.string().min(1).max(MAX_BASE64_CHARS), + tests: z + .array( + z.object({ + name: z.string().trim().min(1).max(200), + functionName: z.string().trim().min(1).max(128), + args: z.array(wasmValue).max(16).optional(), + expected: z.union([wasmValue, z.null()]).optional(), + }) + ) + .min(1) + .max(MAX_TEST_CASES), +}); + +/** + * @route POST /api/v1/evaluations + * @desc Queue a contract submission for sandboxed evaluation + */ +router.post('/', authenticate, validateRequest(evaluationSubmissionSchema), async (req: Request, res: Response) => { + const body = req.body as z.infer; + try { + const data: EvaluationJobData = { + studentId: req.user!.id, + exerciseId: body.exerciseId, + wasmBase64: body.wasmBase64, + tests: body.tests.map((t) => ({ + name: t.name, + functionName: t.functionName, + ...(t.args ? { args: t.args } : {}), + ...(t.expected !== undefined ? { expected: t.expected } : {}), + })), + }; + const job = await enqueueEvaluation(data); + res.status(202).json({ status: 'success', data: { jobId: job.id, state: 'queued' } }); + } catch (error) { + if (error instanceof QueueRateLimitError) { + res.setHeader('Retry-After', Math.ceil(error.retryAfterMs / 1000)); + res.status(429).json({ status: 'error', error: error.message }); + return; + } + logger.error('[evaluations] Failed to enqueue evaluation', { + error: error instanceof Error ? error.message : String(error), + }); + res.status(500).json({ status: 'error', error: 'Failed to queue evaluation' }); + } +}); + +/** + * @route GET /api/v1/evaluations/:jobId + * @desc Evaluation state and, once finished, the structured sandbox result + */ +router.get('/:jobId', authenticate, async (req: Request, res: Response) => { + try { + const job = await getQueueManager() + .getQueue(QUEUE_NAMES.CONTRACT_EVALUATION) + .getJob(String(req.params.jobId)); + + // Respond 404 for other students' jobs too, so job ids can't be probed. + if (!job || job.data.studentId !== req.user!.id) { + res.status(404).json({ status: 'error', error: 'Evaluation not found' }); + return; + } + + const state = await job.getState(); + res.json({ + status: 'success', + data: { + jobId: job.id, + exerciseId: job.data.exerciseId, + state, + attemptsMade: job.attemptsMade, + result: job.returnvalue ?? null, + failedReason: state === 'failed' ? job.failedReason : null, + }, + }); + } catch (error) { + logger.error('[evaluations] Failed to read evaluation', { + error: error instanceof Error ? error.message : String(error), + }); + res.status(500).json({ status: 'error', error: 'Failed to read evaluation' }); + } +}); + +export default router; diff --git a/backend/src/routes/index.ts b/backend/src/routes/index.ts index 288e9445..9c0e6043 100644 --- a/backend/src/routes/index.ts +++ b/backend/src/routes/index.ts @@ -12,6 +12,7 @@ import certificatesRouter from './certificates.routes.js'; import contractRouter from './contracts.routes.js'; import coursesRouter from './courses.js'; import enrollmentsRouter from './enrollments.js'; +import evaluationsRouter from './evaluations.routes.js'; import exportRouter from './export.routes.js'; import explorerRouter from './generator/explorer.routes.js'; import generatorRouter from './generator/generator.routes.js'; @@ -41,6 +42,7 @@ import metricsRouter from './metrics.routes.js'; import adminCoursesRouter from './admin/courses.routes.js'; import adminDLQRouter from './admin/dlq.routes.js'; +import adminQueuesRouter from './admin/queues.routes.js'; import apiRouter from './api.js'; import contributorProofsRouter from './contributor-proofs.routes.js'; import deployRouter from './deploy.routes.js'; @@ -82,6 +84,7 @@ router.use('/dashboard/activity-log', activityLogRouter); router.use('/auth', authRoutes); router.use('/search', curriculumSearchRouter); router.use('/contracts', contractRouter); +router.use('/evaluations', evaluationsRouter); router.use('/notifications', notificationRouter); router.use('/notifications/preferences', notificationPreferencesRouter); router.use('/security', securityRouter); @@ -98,6 +101,7 @@ router.use('/deploy', deployRouter); router.use('/did', didRouter); router.use('/webhooks', webhooksRouter); router.use('/admin/dlq', adminDLQRouter); +router.use('/admin/queues', adminQueuesRouter); router.use('/admin/courses', adminCoursesRouter); router.use('/policy', policyRouter); router.use('/storage', storageRouter); diff --git a/backend/src/workers/evaluationWorker.ts b/backend/src/workers/evaluationWorker.ts new file mode 100644 index 00000000..38e6d466 --- /dev/null +++ b/backend/src/workers/evaluationWorker.ts @@ -0,0 +1,146 @@ +/** + * Contract Evaluation Worker — BE-010 + * + * Consumes the `contract-evaluation` queue: each job carries a student's + * compiled contract bytecode plus the exercise's test cases, which are run + * inside the WASM execution sandbox (`services/wasmExecutionSandbox.ts`). + * The structured result — per-test outcomes and exact gas consumption — is + * stored as the job's return value and pushed live to the student via the + * `exerciseStatusUpdated` GraphQL subscription (QUEUED → RUNNING → + * PASSED | FAILED | ERROR). + * + * Hostile submissions never throw here: the sandbox reports them as results. + * Only infrastructure faults reject, and those are retried by BullMQ and + * eventually dead-lettered by the queue manager. + */ + +import { UnrecoverableError, type Job, type Worker } from 'bullmq'; +import { publishExerciseStatus, type ExerciseStatus, type ExerciseStatusEvent } from '../graphql/pubsub.js'; +import { getQueueManager, QUEUE_NAMES, type QueueManager } from '../queues/queueManager.js'; +import { + MAX_TEST_CASES, + runWasmEvaluation, + type SandboxRunRequest, + type SandboxRunResult, + type WasmTestCase, +} from '../services/wasmExecutionSandbox.js'; +import logger from '../utils/logger.js'; + +export interface EvaluationJobData { + studentId: string; + exerciseId: string; + wasmBase64: string; + tests: WasmTestCase[]; + userId?: string; +} + +export type EvaluationJobResult = SandboxRunResult; + +type Publish = (event: Omit) => void; +type Run = (request: SandboxRunRequest) => Promise; + +const STATUS_BY_RESULT: Record = { + passed: 'PASSED', + failed: 'FAILED', + error: 'ERROR', +}; + +function assertValidJob(data: EvaluationJobData): void { + if (!data?.studentId || !data.exerciseId) { + throw new UnrecoverableError('Evaluation job is missing studentId or exerciseId'); + } + if (typeof data.wasmBase64 !== 'string' || data.wasmBase64.length === 0) { + throw new UnrecoverableError('Evaluation job is missing wasmBase64'); + } + if (!Array.isArray(data.tests) || data.tests.length === 0 || data.tests.length > MAX_TEST_CASES) { + throw new UnrecoverableError(`Evaluation job must include 1-${MAX_TEST_CASES} test cases`); + } +} + +export function toStatusEvent( + data: Pick, + jobId: string | undefined, + result: SandboxRunResult +): Omit { + return { + studentId: data.studentId, + exerciseId: data.exerciseId, + jobId: jobId ?? null, + status: STATUS_BY_RESULT[result.status], + gasUsed: result.gasUsed, + gasLimit: result.gasLimit, + terminationReason: result.terminationReason, + passed: result.summary.passed, + total: result.summary.total, + tests: result.tests.map((t) => ({ + name: t.name, + status: t.status, + gasUsed: t.gasUsed, + durationMs: t.durationMs, + error: t.error ?? null, + })), + message: result.error ?? null, + }; +} + +export function createEvaluationProcessor(run: Run = runWasmEvaluation, publish: Publish = publishExerciseStatus) { + return async (job: Job): Promise => { + assertValidJob(job.data); + const { studentId, exerciseId } = job.data; + + publish({ studentId, exerciseId, jobId: job.id ?? null, status: 'RUNNING' }); + + try { + const result = await run({ wasmBase64: job.data.wasmBase64, tests: job.data.tests }); + publish(toStatusEvent(job.data, job.id, result)); + logger.info('[evaluationWorker] Evaluation finished', { + jobId: job.id, + studentId, + exerciseId, + status: result.status, + terminationReason: result.terminationReason, + gasUsed: result.gasUsed, + }); + return result; + } catch (err) { + publish({ + studentId, + exerciseId, + jobId: job.id ?? null, + status: 'ERROR', + message: 'Evaluation infrastructure error; the job will be retried.', + }); + throw err; + } + }; +} + +export async function enqueueEvaluation( + data: EvaluationJobData, + manager: QueueManager = getQueueManager(), + publish: Publish = publishExerciseStatus +): Promise> { + const job = await manager.enqueue(QUEUE_NAMES.CONTRACT_EVALUATION, 'evaluate-contract', data, { + userId: data.studentId, + }); + publish({ studentId: data.studentId, exerciseId: data.exerciseId, jobId: job.id ?? null, status: 'QUEUED' }); + return job; +} + +let evaluationWorker: Worker | null = null; + +export function startEvaluationWorker(manager: QueueManager = getQueueManager()) { + if (!evaluationWorker) { + evaluationWorker = manager.createWorker( + QUEUE_NAMES.CONTRACT_EVALUATION, + createEvaluationProcessor() + ); + } + return evaluationWorker; +} + +export async function stopEvaluationWorker(): Promise { + if (!evaluationWorker) return; + await evaluationWorker.close(); + evaluationWorker = null; +} diff --git a/backend/src/workers/queueWorkers.ts b/backend/src/workers/queueWorkers.ts new file mode 100644 index 00000000..d935b5d2 --- /dev/null +++ b/backend/src/workers/queueWorkers.ts @@ -0,0 +1,38 @@ +/** + * Standalone entrypoint for the BullMQ workers managed by QueueManager + * (BE-006 / BE-010). Runs separately from the API so CPU-heavy evaluation and + * rate-limited minting never compete with request handling: + * + * npm run workers:queues (development, via tsx) + * npm run start:workers (production, compiled output) + * + * The dead-letter queue intentionally has no consumer; it is drained only + * through the admin API (/api/v1/admin/queues/dlq). + */ + +import { getQueueManager } from '../queues/queueManager.js'; +import logger from '../utils/logger.js'; +import { startEvaluationWorker, stopEvaluationWorker } from './evaluationWorker.js'; +import { startMintingWorker, stopMintingWorker } from './mintingWorker.js'; + +export function startQueueWorkers(): void { + startMintingWorker(); + startEvaluationWorker(); +} + +export async function stopQueueWorkers(): Promise { + await Promise.allSettled([stopMintingWorker(), stopEvaluationWorker()]); + await getQueueManager().close(); +} + +const isEntrypoint = /queueWorkers\.(ts|js)$/.test(process.argv[1] ?? ''); + +if (isEntrypoint) { + startQueueWorkers(); + const shutdown = (signal: string) => { + logger.info(`[queueWorkers] ${signal} received; draining workers`); + stopQueueWorkers().finally(() => process.exit(0)); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} diff --git a/backend/tests/queueWorkers.test.ts b/backend/tests/queueWorkers.test.ts new file mode 100644 index 00000000..6daa5bb2 --- /dev/null +++ b/backend/tests/queueWorkers.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, it, jest } from '@jest/globals'; +import { UnrecoverableError } from 'bullmq'; + +jest.mock('../src/utils/logger.js', () => ({ + __esModule: true, + default: { info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }, +})); + +jest.mock('../src/blockchain/CertificateBlockchainService.js', () => { + class BlockchainNotConfiguredError extends Error { + constructor(operation: string) { + super(`${operation} is not configured`); + this.name = 'BlockchainNotConfiguredError'; + } + } + return { __esModule: true, BlockchainNotConfiguredError, CertificateBlockchainService: class {} }; +}); + +import { BlockchainNotConfiguredError } from '../src/blockchain/CertificateBlockchainService.js'; +import { createMintProcessor, RpcRateLimitedError, type MintJobData } from '../src/workers/mintingWorker.js'; +import { createEvaluationProcessor, enqueueEvaluation, type EvaluationJobData } from '../src/workers/evaluationWorker.js'; +import type { SandboxRunResult } from '../src/services/wasmExecutionSandbox.js'; + +const mintJob = (data: Partial = {}) => + ({ + id: 'm1', + attemptsMade: 0, + data: { studentId: 's1', courseId: 'c1', metadata: { grade: 'A' }, ...data }, + }) as any; + +describe('mintingWorker processor (BE-006)', () => { + const okMinter = () => ({ + mintCertificate: jest.fn(async () => ({ + success: true, + tokenId: 't1', + transactionHash: 'abc', + contractAddress: 'C1', + })), + }); + + it('mints and returns the transaction details', async () => { + const minter = okMinter(); + const result = await createMintProcessor(minter, () => undefined)(mintJob()); + expect(result).toMatchObject({ tokenId: 't1', transactionHash: 'abc', contractAddress: 'C1' }); + expect(minter.mintCertificate).toHaveBeenCalledWith({ grade: 'A', studentId: 's1', courseId: 'c1' }); + }); + + it('treats an unconfirmed mint (circuit-breaker fallback) as a retryable failure', async () => { + const minter = { + mintCertificate: jest.fn(async () => ({ + success: false, + tokenId: 'x', + transactionHash: 'circuit-breaker-fallback', + contractAddress: 'C1', + })), + }; + const err = await createMintProcessor(minter, () => undefined)(mintJob()).catch((e) => e); + expect(err).toBeInstanceOf(Error); + expect(err).not.toBeInstanceOf(UnrecoverableError); + }); + + it('sends invalid payloads straight to the DLQ via UnrecoverableError', async () => { + const err = await createMintProcessor(okMinter(), () => undefined)(mintJob({ studentId: '' })).catch((e) => e); + expect(err).toBeInstanceOf(UnrecoverableError); + }); + + it('does not burn retries on a misconfigured blockchain', async () => { + const minter = { + mintCertificate: jest.fn(async () => { + throw new BlockchainNotConfiguredError('mintCertificate'); + }), + }; + const err = await createMintProcessor(minter as any, () => undefined)(mintJob()).catch((e) => e); + expect(err).toBeInstanceOf(UnrecoverableError); + }); + + it('pauses the whole queue on a Soroban 429 instead of consuming an attempt', async () => { + const worker = { rateLimit: jest.fn(async () => undefined) }; + const minter = { + mintCertificate: jest.fn(async () => { + throw new RpcRateLimitedError(7_000); + }), + }; + const err = await createMintProcessor(minter as any, () => worker as any)(mintJob()).catch((e) => e); + expect(worker.rateLimit).toHaveBeenCalledWith(7_000); + expect(err.name).toBe('RateLimitError'); + }); + + it('recognises HTTP 429 responses from the RPC client', async () => { + const worker = { rateLimit: jest.fn(async () => undefined) }; + const minter = { + mintCertificate: jest.fn(async () => { + throw Object.assign(new Error('Too Many Requests'), { response: { status: 429 } }); + }), + }; + await createMintProcessor(minter as any, () => worker as any)(mintJob()).catch(() => undefined); + expect(worker.rateLimit).toHaveBeenCalled(); + }); +}); + +const sandboxResult = (overrides: Partial = {}): SandboxRunResult => ({ + executionId: 'e1', + status: 'passed', + terminationReason: 'completed', + gasUsed: 752, + gasLimit: 1_000_000, + memoryPeakBytes: 0, + memoryLimitBytes: 50 * 1024 * 1024, + durationMs: 3, + tests: [{ name: 't', functionName: 'add', status: 'passed', actual: 5, expected: 5, gasUsed: 752, durationMs: 0.1 }], + summary: { total: 1, passed: 1, failed: 0, errored: 0, skipped: 0 }, + ...overrides, +}); + +const evaluationJob = (data: Partial = {}) => + ({ + id: 'j1', + data: { + studentId: 's1', + exerciseId: 'ex1', + wasmBase64: 'AGFzbQEAAAA=', + tests: [{ name: 't', functionName: 'add', args: [2, 3], expected: 5 }], + ...data, + }, + }) as any; + +describe('evaluationWorker processor (BE-010)', () => { + it('runs the sandbox and streams RUNNING then the final status with gas and test outputs', async () => { + const publish = jest.fn(); + const run = jest.fn(async () => sandboxResult()); + const result = await createEvaluationProcessor(run, publish)(evaluationJob()); + + expect(result.gasUsed).toBe(752); + expect(publish).toHaveBeenNthCalledWith(1, expect.objectContaining({ status: 'RUNNING', studentId: 's1' })); + expect(publish).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + status: 'PASSED', + exerciseId: 'ex1', + jobId: 'j1', + gasUsed: 752, + gasLimit: 1_000_000, + passed: 1, + total: 1, + tests: [expect.objectContaining({ name: 't', status: 'passed', gasUsed: 752 })], + }) + ); + }); + + it.each([ + ['failed', 'FAILED'], + ['error', 'ERROR'], + ] as const)('maps a %s sandbox result to %s', async (status, expected) => { + const publish = jest.fn(); + await createEvaluationProcessor(async () => sandboxResult({ status }), publish)(evaluationJob()); + expect(publish).toHaveBeenLastCalledWith(expect.objectContaining({ status: expected })); + }); + + it('rejects malformed jobs as unrecoverable', async () => { + const err = await createEvaluationProcessor(jest.fn() as any, jest.fn())(evaluationJob({ tests: [] })).catch( + (e) => e + ); + expect(err).toBeInstanceOf(UnrecoverableError); + }); + + it('reports infrastructure faults and rethrows so BullMQ retries', async () => { + const publish = jest.fn(); + const run = jest.fn(async () => { + throw new Error('worker pool unavailable'); + }); + await expect(createEvaluationProcessor(run, publish)(evaluationJob())).rejects.toThrow('worker pool unavailable'); + expect(publish).toHaveBeenLastCalledWith(expect.objectContaining({ status: 'ERROR' })); + }); + + it('enqueues per student (for rate limiting) and publishes QUEUED', async () => { + const publish = jest.fn(); + const manager = { enqueue: jest.fn(async () => ({ id: 'j9' })) }; + await enqueueEvaluation(evaluationJob().data, manager as any, publish); + + expect(manager.enqueue).toHaveBeenCalledWith('contract-evaluation', 'evaluate-contract', expect.any(Object), { + userId: 's1', + }); + expect(publish).toHaveBeenCalledWith(expect.objectContaining({ status: 'QUEUED', jobId: 'j9' })); + }); +}); From 7ac3f828b299edd98c05cc54da2db68effc32fb6 Mon Sep 17 00:00:00 2001 From: emmzy4tech Date: Wed, 21 Oct 2026 17:34:23 +0100 Subject: [PATCH 5/5] feat(auth): add webauthn verification core and passkey service (BE-008, partial) Adds webauthn_credentials and webauthn_challenges tables, CBOR/COSE parsing, none/packed/fido-u2f attestation and assertion verification, 60s single-use challenges, and signature-counter clone detection. HTTP endpoints and tests are not included yet. Co-Authored-By: Claude Opus 5.5 --- .../migration.sql | 50 ++ backend/prisma/schema.prisma | 48 ++ backend/src/auth/passkeyService.ts | 543 ++++++++++++++++++ backend/src/auth/webauthn.ts | 434 ++++++++++++++ 4 files changed, 1075 insertions(+) create mode 100644 backend/prisma/migrations/20261021000000_webauthn_passkeys/migration.sql create mode 100644 backend/src/auth/passkeyService.ts create mode 100644 backend/src/auth/webauthn.ts diff --git a/backend/prisma/migrations/20261021000000_webauthn_passkeys/migration.sql b/backend/prisma/migrations/20261021000000_webauthn_passkeys/migration.sql new file mode 100644 index 00000000..ebc9a45e --- /dev/null +++ b/backend/prisma/migrations/20261021000000_webauthn_passkeys/migration.sql @@ -0,0 +1,50 @@ +-- CreateTable +CREATE TABLE "webauthn_credentials" ( + "id" TEXT NOT NULL, + "studentId" TEXT NOT NULL, + "credentialId" TEXT NOT NULL, + "publicKey" BYTEA NOT NULL, + "algorithm" INTEGER NOT NULL, + "signCount" BIGINT NOT NULL DEFAULT 0, + "transports" TEXT[] DEFAULT ARRAY[]::TEXT[], + "aaguid" TEXT, + "attestationFormat" TEXT NOT NULL, + "attestationType" TEXT NOT NULL, + "userVerified" BOOLEAN NOT NULL DEFAULT false, + "backupEligible" BOOLEAN NOT NULL DEFAULT false, + "backedUp" BOOLEAN NOT NULL DEFAULT false, + "deviceName" TEXT, + "cloneDetectedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastUsedAt" TIMESTAMP(3), + + CONSTRAINT "webauthn_credentials_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "webauthn_challenges" ( + "id" TEXT NOT NULL, + "challenge" TEXT NOT NULL, + "ceremony" TEXT NOT NULL, + "studentId" TEXT, + "expiresAt" TIMESTAMP(3) NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "webauthn_challenges_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "webauthn_credentials_credentialId_key" ON "webauthn_credentials"("credentialId"); + +-- CreateIndex +CREATE INDEX "webauthn_credentials_studentId_idx" ON "webauthn_credentials"("studentId"); + +-- CreateIndex +CREATE UNIQUE INDEX "webauthn_challenges_challenge_key" ON "webauthn_challenges"("challenge"); + +-- CreateIndex +CREATE INDEX "webauthn_challenges_expiresAt_idx" ON "webauthn_challenges"("expiresAt"); + +-- AddForeignKey +ALTER TABLE "webauthn_credentials" ADD CONSTRAINT "webauthn_credentials_studentId_fkey" FOREIGN KEY ("studentId") REFERENCES "students"("id") ON DELETE CASCADE ON UPDATE CASCADE; + diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 4ae50562..4d3f85c2 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -39,6 +39,7 @@ model Student { ideas Idea[] votes Vote[] passkeyCredentials PasskeyCredential[] + webauthnCredentials WebAuthnCredential[] hashSimulations HashSimulation[] subscriptions Subscription[] @@ -68,6 +69,53 @@ model PasskeyCredential { @@map("passkey_credentials") } +/// FIDO2/WebAuthn passkey bound to a student (BE-008). +model WebAuthnCredential { + id String @id @default(cuid()) + studentId String + /// base64url credential ID chosen by the authenticator + credentialId String @unique + /// COSE_Key-encoded credential public key + publicKey Bytes + /// COSE algorithm identifier (-7 ES256, -8 EdDSA, -257 RS256, ...) + algorithm Int + /// Authenticator signature counter (uint32, hence BigInt) + signCount BigInt @default(0) + transports String[] @default([]) + aaguid String? + attestationFormat String + /// none | self | basic + attestationType String + userVerified Boolean @default(false) + backupEligible Boolean @default(false) + backedUp Boolean @default(false) + deviceName String? + /// Set when a signature-counter regression indicates a cloned authenticator; the credential is then refused + cloneDetectedAt DateTime? + createdAt DateTime @default(now()) + lastUsedAt DateTime? + + student Student @relation(fields: [studentId], references: [id], onDelete: Cascade) + + @@index([studentId]) + @@map("webauthn_credentials") +} + +/// Single-use WebAuthn challenge nonce with a 60-second TTL (BE-008). +model WebAuthnChallenge { + id String @id @default(cuid()) + /// base64url-encoded 32-byte random nonce + challenge String @unique + /// registration | authentication + ceremony String + studentId String? + expiresAt DateTime + createdAt DateTime @default(now()) + + @@index([expiresAt]) + @@map("webauthn_challenges") +} + model OAuthState { id String @id @default(cuid()) state String @unique diff --git a/backend/src/auth/passkeyService.ts b/backend/src/auth/passkeyService.ts new file mode 100644 index 00000000..272aced0 --- /dev/null +++ b/backend/src/auth/passkeyService.ts @@ -0,0 +1,543 @@ +/** + * Passkey (FIDO2 / WebAuthn) Authentication Service — BE-008 + * + * Implements the relying-party side of the W3C WebAuthn registration (§7.1) + * and authentication (§7.2) ceremonies for platform passkeys (Touch ID, + * Face ID, Windows Hello) and roaming security keys (YubiKey). + * + * Replay protection: every ceremony starts with a 32-byte random challenge + * stored in PostgreSQL with a 60-second TTL. A challenge is consumed with an + * atomic DELETE before anything else is verified, so it can be used at most + * once even under concurrent submissions. + * + * Clone detection: the authenticator's signature counter must strictly + * increase (unless both stored and received counters are zero, which the + * spec allows for authenticators without counters). A regression flags the + * credential (`cloneDetectedAt`) and it is refused from then on. The counter + * update itself is a compare-and-set on the stored value, so two concurrent + * assertions replaying the same counter cannot both succeed. + */ + +import { randomBytes } from 'crypto'; +import prisma from '../db/index.js'; +import { + decodeAttestationObject, + fromBase64Url, + parseAuthenticatorData, + parseCoseKey, + sha256, + SUPPORTED_ALGORITHMS, + toBase64Url, + verifyAttestation, + verifySignature, + WebAuthnError, + type AuthenticatorData, +} from './webauthn.js'; + +export const CHALLENGE_TTL_MS = 60_000; +const CHALLENGE_BYTES = 32; + +type Ceremony = 'registration' | 'authentication'; +type UserVerification = 'required' | 'preferred' | 'discouraged'; + +export interface PasskeyConfig { + rpId: string; + rpName: string; + /** Exact origins (scheme://host[:port]) allowed to run ceremonies. */ + origins: string[]; + userVerification: UserVerification; + attestation: 'none' | 'direct'; + challengeTtlMs: number; +} + +export function loadPasskeyConfig(env: NodeJS.ProcessEnv = process.env): PasskeyConfig { + const frontend = env.FRONTEND_URL || 'http://localhost:3000'; + const origins = (env.WEBAUTHN_ORIGINS || frontend) + .split(',') + .map((o) => o.trim().replace(/\/+$/, '')) + .filter(Boolean); + let defaultRpId = 'localhost'; + try { + defaultRpId = new URL(frontend).hostname; + } catch { + // keep localhost + } + return { + rpId: env.WEBAUTHN_RP_ID || defaultRpId, + rpName: env.WEBAUTHN_RP_NAME || 'Web3 Student Lab', + origins, + userVerification: env.WEBAUTHN_USER_VERIFICATION === 'preferred' ? 'preferred' : 'required', + attestation: env.WEBAUTHN_ATTESTATION === 'direct' ? 'direct' : 'none', + challengeTtlMs: CHALLENGE_TTL_MS, + }; +} + +// --------------------------------------------------------------------------- +// Persistence surface (subset of the Prisma client, so tests can fake it) +// --------------------------------------------------------------------------- + +export interface ChallengeRow { + challenge: string; + ceremony: string; + studentId: string | null; + expiresAt: Date; +} + +export interface CredentialRow { + id: string; + studentId: string; + credentialId: string; + publicKey: Uint8Array; + algorithm: number; + signCount: bigint; + transports: string[]; + aaguid: string | null; + attestationFormat: string; + attestationType: string; + userVerified: boolean; + backupEligible: boolean; + backedUp: boolean; + deviceName: string | null; + cloneDetectedAt: Date | null; + createdAt: Date; + lastUsedAt: Date | null; +} + +export interface StudentRow { + id: string; + email: string; + firstName: string; + lastName: string; +} + +export interface PasskeyDb { + webAuthnChallenge: { + create(args: { data: ChallengeRow }): Promise; + delete(args: { where: { challenge: string } }): Promise; + deleteMany(args: { where: { expiresAt: { lt: Date } } }): Promise<{ count: number }>; + }; + webAuthnCredential: { + findUnique(args: { where: { credentialId: string } }): Promise; + findMany(args: { where: { studentId: string }; orderBy?: { createdAt: 'asc' | 'desc' } }): Promise; + create(args: { data: Omit }): Promise; + updateMany(args: { + where: { id: string; signCount?: bigint; cloneDetectedAt?: null }; + data: Partial>; + }): Promise<{ count: number }>; + deleteMany(args: { where: { id: string; studentId: string } }): Promise<{ count: number }>; + }; + student: { + findUnique(args: { + where: { id: string } | { email: string }; + select: { id: true; email: true; firstName: true; lastName: true }; + }): Promise; + }; +} + +// --------------------------------------------------------------------------- +// WebAuthn JSON shapes (PublicKeyCredential.toJSON(), WebAuthn L3) +// --------------------------------------------------------------------------- + +export interface RegistrationResponseJSON { + id: string; + rawId: string; + type: 'public-key'; + response: { clientDataJSON: string; attestationObject: string; transports?: string[] }; + authenticatorAttachment?: string; + clientExtensionResults?: Record; +} + +export interface AuthenticationResponseJSON { + id: string; + rawId: string; + type: 'public-key'; + response: { clientDataJSON: string; authenticatorData: string; signature: string; userHandle?: string | null }; + authenticatorAttachment?: string; + clientExtensionResults?: Record; +} + +interface CredentialDescriptorJSON { + type: 'public-key'; + id: string; + transports?: string[]; +} + +export interface PasskeyCredentialSummary { + id: string; + credentialId: string; + deviceName: string | null; + algorithm: number; + attestationFormat: string; + attestationType: string; + aaguid: string | null; + transports: string[]; + backedUp: boolean; + signCount: number; + createdAt: Date; + lastUsedAt: Date | null; + cloneDetected: boolean; +} + +const KNOWN_TRANSPORTS = new Set(['usb', 'nfc', 'ble', 'smart-card', 'hybrid', 'internal']); + +const toSummary = (row: CredentialRow): PasskeyCredentialSummary => ({ + id: row.id, + credentialId: row.credentialId, + deviceName: row.deviceName, + algorithm: row.algorithm, + attestationFormat: row.attestationFormat, + attestationType: row.attestationType, + aaguid: row.aaguid, + transports: row.transports, + backedUp: row.backedUp, + signCount: Number(row.signCount), + createdAt: row.createdAt, + lastUsedAt: row.lastUsedAt, + cloneDetected: row.cloneDetectedAt !== null, +}); + +const formatAaguid = (bytes: Buffer): string | null => { + if (bytes.every((b) => b === 0)) return null; + const hex = bytes.toString('hex'); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +}; + +const isPrismaNotFound = (err: unknown) => (err as { code?: string })?.code === 'P2025'; + +interface ClientData { + type: string; + challenge: string; + origin: string; + crossOrigin?: boolean; +} + +export class PasskeyService { + constructor( + private readonly db: PasskeyDb = prisma as unknown as PasskeyDb, + private readonly config: PasskeyConfig = loadPasskeyConfig(), + private readonly now: () => number = Date.now + ) {} + + // ------------------------------------------------------------------------- + // Challenges + // ------------------------------------------------------------------------- + + private async issueChallenge(ceremony: Ceremony, studentId: string | null): Promise { + const nowMs = this.now(); + // Opportunistic cleanup keeps the table tiny without a cron job. + await this.db.webAuthnChallenge.deleteMany({ where: { expiresAt: { lt: new Date(nowMs) } } }).catch(() => undefined); + + const challenge = toBase64Url(randomBytes(CHALLENGE_BYTES)); + await this.db.webAuthnChallenge.create({ + data: { challenge, ceremony, studentId, expiresAt: new Date(nowMs + this.config.challengeTtlMs) }, + }); + return challenge; + } + + /** Atomically deletes the challenge; it is spent whether or not the rest of the ceremony succeeds. */ + private async consumeChallenge(challenge: string, ceremony: Ceremony): Promise { + let row: ChallengeRow; + try { + row = await this.db.webAuthnChallenge.delete({ where: { challenge } }); + } catch (err) { + if (isPrismaNotFound(err)) { + throw new WebAuthnError('challenge-invalid', 'Challenge is unknown or has already been used'); + } + throw err; + } + if (row.ceremony !== ceremony) { + throw new WebAuthnError('challenge-invalid', 'Challenge was issued for a different ceremony'); + } + if (row.expiresAt.getTime() <= this.now()) { + throw new WebAuthnError('challenge-expired', 'Challenge has expired'); + } + return row; + } + + // ------------------------------------------------------------------------- + // Shared verification steps + // ------------------------------------------------------------------------- + + private parseClientData(encoded: string, expectedType: string): { clientData: ClientData; hash: Buffer } { + const raw = fromBase64Url(encoded, 'clientDataJSON'); + let clientData: ClientData; + try { + clientData = JSON.parse(raw.toString('utf8')) as ClientData; + } catch { + throw new WebAuthnError('invalid-client-data', 'clientDataJSON is not valid JSON'); + } + if (clientData.type !== expectedType) { + throw new WebAuthnError('invalid-client-data', `clientData.type must be '${expectedType}'`); + } + if (typeof clientData.challenge !== 'string' || !clientData.challenge) { + throw new WebAuthnError('invalid-client-data', 'clientData.challenge is missing'); + } + if (!this.config.origins.includes(clientData.origin)) { + throw new WebAuthnError('origin-mismatch', `Origin '${clientData.origin}' is not allowed`); + } + if (clientData.crossOrigin === true) { + throw new WebAuthnError('origin-mismatch', 'Cross-origin ceremonies are not allowed'); + } + return { clientData, hash: sha256(raw) }; + } + + private assertAuthenticatorData(authData: AuthenticatorData): void { + if (!authData.rpIdHash.equals(sha256(Buffer.from(this.config.rpId, 'utf8')))) { + throw new WebAuthnError('rp-id-mismatch', 'Authenticator data was produced for a different RP ID'); + } + if (!authData.flags.userPresent) { + throw new WebAuthnError('user-not-present', 'User presence flag is not set'); + } + if (this.config.userVerification === 'required' && !authData.flags.userVerified) { + throw new WebAuthnError('user-not-verified', 'User verification (biometric/PIN) is required'); + } + } + + private assertCredentialEnvelope(response: { id: string; rawId: string; type: string }): void { + if (response.type !== 'public-key') throw new WebAuthnError('invalid-credential', "type must be 'public-key'"); + if (response.id !== response.rawId) throw new WebAuthnError('invalid-credential', 'id and rawId differ'); + fromBase64Url(response.rawId, 'rawId'); + } + + // ------------------------------------------------------------------------- + // Registration (§7.1) + // ------------------------------------------------------------------------- + + async generateRegistrationOptions(studentId: string) { + const student = await this.db.student.findUnique({ + where: { id: studentId }, + select: { id: true, email: true, firstName: true, lastName: true }, + }); + if (!student) throw new WebAuthnError('unknown-user', 'Student not found'); + + const existing = await this.db.webAuthnCredential.findMany({ where: { studentId } }); + const challenge = await this.issueChallenge('registration', studentId); + const displayName = `${student.firstName} ${student.lastName}`.trim() || student.email; + + return { + challenge, + rp: { id: this.config.rpId, name: this.config.rpName }, + // The user handle is the opaque student id — never PII. + user: { id: toBase64Url(Buffer.from(student.id, 'utf8')), name: student.email, displayName }, + pubKeyCredParams: SUPPORTED_ALGORITHMS.map((alg) => ({ type: 'public-key' as const, alg })), + timeout: this.config.challengeTtlMs, + excludeCredentials: existing.map( + (c): CredentialDescriptorJSON => ({ type: 'public-key', id: c.credentialId, transports: c.transports }) + ), + authenticatorSelection: { + // No attachment constraint: platform passkeys and roaming keys (YubiKey) both work. + residentKey: 'preferred' as const, + requireResidentKey: false, + userVerification: this.config.userVerification, + }, + attestation: this.config.attestation, + extensions: { credProps: true }, + }; + } + + async verifyRegistration( + studentId: string, + response: RegistrationResponseJSON, + deviceName?: string + ): Promise { + this.assertCredentialEnvelope(response); + const { clientData, hash } = this.parseClientData(response.response.clientDataJSON, 'webauthn.create'); + const challenge = await this.consumeChallenge(clientData.challenge, 'registration'); + if (challenge.studentId !== studentId) { + throw new WebAuthnError('challenge-invalid', 'Challenge was issued to a different user'); + } + + const { fmt, attStmt, authData: rawAuthData } = decodeAttestationObject( + fromBase64Url(response.response.attestationObject, 'attestationObject') + ); + const authData = parseAuthenticatorData(rawAuthData); + this.assertAuthenticatorData(authData); + + const attested = authData.attestedCredential; + if (!attested) throw new WebAuthnError('invalid-attestation', 'Attested credential data is missing'); + if (toBase64Url(attested.credentialId) !== response.rawId) { + throw new WebAuthnError('invalid-credential', 'Credential ID does not match the attested credential'); + } + + const credentialKey = parseCoseKey(attested.publicKey); + if (!SUPPORTED_ALGORITHMS.includes(credentialKey.alg)) { + throw new WebAuthnError('unsupported-algorithm', `Algorithm ${credentialKey.alg} was not offered`); + } + + const attestation = verifyAttestation({ + fmt, + attStmt, + authData, + clientDataHash: hash, + credentialKey, + now: new Date(this.now()), + }); + + if (await this.db.webAuthnCredential.findUnique({ where: { credentialId: response.rawId } })) { + throw new WebAuthnError('credential-exists', 'This authenticator is already registered'); + } + + const transports = (response.response.transports ?? []).filter((t) => KNOWN_TRANSPORTS.has(t)); + const created = await this.db.webAuthnCredential.create({ + data: { + studentId, + credentialId: response.rawId, + publicKey: attested.publicKey, + algorithm: credentialKey.alg, + signCount: BigInt(authData.signCount), + transports, + aaguid: formatAaguid(attested.aaguid), + attestationFormat: attestation.fmt, + attestationType: attestation.type, + userVerified: authData.flags.userVerified, + backupEligible: authData.flags.backupEligible, + backedUp: authData.flags.backedUp, + deviceName: deviceName?.trim().slice(0, 100) || null, + }, + }); + return toSummary(created); + } + + // ------------------------------------------------------------------------- + // Authentication (§7.2) + // ------------------------------------------------------------------------- + + /** + * With an email, the challenge is bound to that account and its passkeys + * are listed; without one, the client uses a discoverable credential. An + * unknown email yields an unbound challenge so account existence isn't leaked. + */ + async generateAuthenticationOptions(email?: string) { + let studentId: string | null = null; + let allowCredentials: CredentialDescriptorJSON[] = []; + + if (email) { + const student = await this.db.student.findUnique({ + where: { email: email.trim().toLowerCase() }, + select: { id: true, email: true, firstName: true, lastName: true }, + }); + if (student) { + studentId = student.id; + const credentials = await this.db.webAuthnCredential.findMany({ where: { studentId } }); + allowCredentials = credentials + .filter((c) => c.cloneDetectedAt === null) + .map((c) => ({ type: 'public-key', id: c.credentialId, transports: c.transports })); + } + } + + const challenge = await this.issueChallenge('authentication', studentId); + return { + challenge, + rpId: this.config.rpId, + timeout: this.config.challengeTtlMs, + userVerification: this.config.userVerification, + allowCredentials, + }; + } + + async verifyAuthentication( + response: AuthenticationResponseJSON + ): Promise<{ studentId: string; credential: PasskeyCredentialSummary }> { + this.assertCredentialEnvelope(response); + const { clientData, hash } = this.parseClientData(response.response.clientDataJSON, 'webauthn.get'); + const challenge = await this.consumeChallenge(clientData.challenge, 'authentication'); + + const credential = await this.db.webAuthnCredential.findUnique({ where: { credentialId: response.rawId } }); + if (!credential) throw new WebAuthnError('unknown-credential', 'Passkey is not registered'); + if (credential.cloneDetectedAt) { + throw new WebAuthnError('credential-disabled', 'Passkey was disabled after a possible clone was detected'); + } + if (challenge.studentId && challenge.studentId !== credential.studentId) { + throw new WebAuthnError('unknown-credential', 'Passkey does not belong to this account'); + } + + const userHandle = response.response.userHandle; + if (userHandle) { + if (fromBase64Url(userHandle, 'userHandle').toString('utf8') !== credential.studentId) { + throw new WebAuthnError('user-handle-mismatch', 'userHandle does not match the passkey owner'); + } + } else if (!challenge.studentId) { + throw new WebAuthnError('user-handle-mismatch', 'userHandle is required for usernameless sign-in'); + } + + const authData = parseAuthenticatorData(fromBase64Url(response.response.authenticatorData, 'authenticatorData')); + this.assertAuthenticatorData(authData); + if (authData.flags.backupEligible !== credential.backupEligible) { + throw new WebAuthnError('invalid-authenticator-data', 'Backup eligibility changed since registration'); + } + + const key = parseCoseKey(credential.publicKey); + const signed = Buffer.concat([authData.raw, hash]); + if (!verifySignature(credential.algorithm, key.key, signed, fromBase64Url(response.response.signature, 'signature'))) { + throw new WebAuthnError('invalid-signature', 'Assertion signature is invalid'); + } + + await this.updateSignCount(credential, authData); + + return { + studentId: credential.studentId, + credential: toSummary({ + ...credential, + signCount: BigInt(authData.signCount), + backedUp: authData.flags.backedUp, + lastUsedAt: new Date(this.now()), + }), + }; + } + + /** §7.2 step 21: counters must strictly increase unless the authenticator doesn't implement them (both 0). */ + private async updateSignCount(credential: CredentialRow, authData: AuthenticatorData): Promise { + const stored = credential.signCount; + const received = BigInt(authData.signCount); + + if ((received !== 0n || stored !== 0n) && received <= stored) { + await this.db.webAuthnCredential.updateMany({ + where: { id: credential.id }, + data: { cloneDetectedAt: new Date(this.now()) }, + }); + throw new WebAuthnError( + 'clone-detected', + `Signature counter did not increase (stored ${stored}, received ${received}); the passkey has been disabled` + ); + } + + // Compare-and-set: fails if another assertion advanced the counter first. + const updated = await this.db.webAuthnCredential.updateMany({ + where: { id: credential.id, signCount: stored, cloneDetectedAt: null }, + data: { + signCount: received, + lastUsedAt: new Date(this.now()), + backedUp: authData.flags.backedUp, + userVerified: authData.flags.userVerified, + }, + }); + if (updated.count !== 1) { + throw new WebAuthnError('counter-race', 'Passkey was used concurrently; please try again'); + } + } + + // ------------------------------------------------------------------------- + // Credential management + // ------------------------------------------------------------------------- + + async listCredentials(studentId: string): Promise { + const rows = await this.db.webAuthnCredential.findMany({ where: { studentId }, orderBy: { createdAt: 'asc' } }); + return rows.map(toSummary); + } + + async deleteCredential(studentId: string, id: string): Promise { + const { count } = await this.db.webAuthnCredential.deleteMany({ where: { id, studentId } }); + return count === 1; + } +} + +let instance: PasskeyService | null = null; + +export function getPasskeyAuthService(): PasskeyService { + if (!instance) instance = new PasskeyService(); + return instance; +} + +export function setPasskeyAuthService(service: PasskeyService | null): void { + instance = service; +} diff --git a/backend/src/auth/webauthn.ts b/backend/src/auth/webauthn.ts new file mode 100644 index 00000000..7acc86c1 --- /dev/null +++ b/backend/src/auth/webauthn.ts @@ -0,0 +1,434 @@ +/** + * WebAuthn primitives — BE-008 + * + * Dependency-free implementation of the parts of W3C WebAuthn Level 2/3 the + * relying party needs: a strict CBOR decoder (RFC 8949 subset used by CTAP2), + * authenticator-data parsing, COSE_Key → Node KeyObject conversion, signature + * verification, and attestation-statement verification for the `none`, + * `packed` (self and x5c) and `fido-u2f` formats. + * + * All verification failures throw `WebAuthnError` with a stable `code`. + */ + +import { createHash, createPublicKey, verify as cryptoVerify, X509Certificate, type KeyObject } from 'crypto'; + +export class WebAuthnError extends Error { + constructor( + readonly code: string, + message: string + ) { + super(message); + this.name = 'WebAuthnError'; + } +} + +const fail = (code: string, message: string): never => { + throw new WebAuthnError(code, message); +}; + +// --------------------------------------------------------------------------- +// base64url +// --------------------------------------------------------------------------- + +const BASE64URL = /^[A-Za-z0-9_-]*$/; + +export function fromBase64Url(value: unknown, field = 'value'): Buffer { + if (typeof value !== 'string' || !BASE64URL.test(value)) { + fail('invalid-encoding', `${field} must be base64url`); + } + return Buffer.from(value as string, 'base64url'); +} + +export const toBase64Url = (bytes: Uint8Array): string => Buffer.from(bytes).toString('base64url'); + +export const sha256 = (data: Uint8Array): Buffer => createHash('sha256').update(data).digest(); + +// --------------------------------------------------------------------------- +// CBOR (definite-length subset, as mandated by CTAP2 canonical encoding) +// --------------------------------------------------------------------------- + +export type CborValue = number | bigint | string | boolean | null | undefined | Buffer | CborValue[] | CborMap; +export type CborMap = Map; + +const MAX_CBOR_DEPTH = 16; + +export function decodeCbor(input: Uint8Array, offset = 0): { value: CborValue; offset: number } { + const buf = Buffer.from(input.buffer, input.byteOffset, input.byteLength); + let pos = offset; + + const need = (n: number) => { + if (pos + n > buf.length) fail('invalid-cbor', 'unexpected end of CBOR data'); + }; + + const readLength = (info: number): number => { + if (info < 24) return info; + if (info === 24) { + need(1); + return buf.readUInt8(pos++); + } + if (info === 25) { + need(2); + const v = buf.readUInt16BE(pos); + pos += 2; + return v; + } + if (info === 26) { + need(4); + const v = buf.readUInt32BE(pos); + pos += 4; + return v; + } + if (info === 27) { + need(8); + const v = buf.readBigUInt64BE(pos); + pos += 8; + if (v > BigInt(Number.MAX_SAFE_INTEGER)) fail('invalid-cbor', 'CBOR integer too large'); + return Number(v); + } + return fail('invalid-cbor', 'indefinite-length CBOR items are not allowed'); + }; + + const readItem = (depth: number): CborValue => { + if (depth > MAX_CBOR_DEPTH) fail('invalid-cbor', 'CBOR nesting too deep'); + need(1); + const initial = buf.readUInt8(pos++); + const major = initial >> 5; + const info = initial & 0x1f; + + switch (major) { + case 0: + return readLength(info); + case 1: + return -1 - readLength(info); + case 2: { + const len = readLength(info); + need(len); + const bytes = Buffer.from(buf.subarray(pos, pos + len)); + pos += len; + return bytes; + } + case 3: { + const len = readLength(info); + need(len); + const text = buf.toString('utf8', pos, pos + len); + pos += len; + return text; + } + case 4: { + const len = readLength(info); + const items: CborValue[] = []; + for (let i = 0; i < len; i++) items.push(readItem(depth + 1)); + return items; + } + case 5: { + const len = readLength(info); + const map: CborMap = new Map(); + for (let i = 0; i < len; i++) { + const key = readItem(depth + 1); + if (typeof key !== 'number' && typeof key !== 'string') fail('invalid-cbor', 'unsupported CBOR map key'); + if (map.has(key)) fail('invalid-cbor', 'duplicate CBOR map key'); + map.set(key, readItem(depth + 1)); + } + return map; + } + case 6: + readLength(info); // tag number — value is decoded untagged + return readItem(depth + 1); + default: + if (info === 20) return false; + if (info === 21) return true; + if (info === 22) return null; + if (info === 23) return undefined; + return fail('invalid-cbor', `unsupported CBOR simple/float value ${info}`); + } + }; + + const value = readItem(0); + return { value, offset: pos }; +} + +const asMap = (value: CborValue, what: string): CborMap => + value instanceof Map ? value : fail('invalid-cbor', `${what} must be a CBOR map`); + +// --------------------------------------------------------------------------- +// Authenticator data (§6.1) +// --------------------------------------------------------------------------- + +export const FLAG_UP = 0x01; +export const FLAG_UV = 0x04; +export const FLAG_BE = 0x08; +export const FLAG_BS = 0x10; +export const FLAG_AT = 0x40; +export const FLAG_ED = 0x80; + +export interface AuthenticatorData { + raw: Buffer; + rpIdHash: Buffer; + flags: { userPresent: boolean; userVerified: boolean; backupEligible: boolean; backedUp: boolean }; + signCount: number; + attestedCredential?: { aaguid: Buffer; credentialId: Buffer; publicKey: Buffer }; +} + +export function parseAuthenticatorData(raw: Buffer): AuthenticatorData { + if (raw.length < 37) fail('invalid-authenticator-data', 'authenticator data is too short'); + const flags = raw.readUInt8(32); + const backupEligible = (flags & FLAG_BE) !== 0; + const backedUp = (flags & FLAG_BS) !== 0; + if (backedUp && !backupEligible) { + fail('invalid-authenticator-data', 'backup state set on a credential that is not backup eligible'); + } + + const result: AuthenticatorData = { + raw, + rpIdHash: raw.subarray(0, 32), + flags: { userPresent: (flags & FLAG_UP) !== 0, userVerified: (flags & FLAG_UV) !== 0, backupEligible, backedUp }, + signCount: raw.readUInt32BE(33), + }; + + let pos = 37; + if (flags & FLAG_AT) { + if (raw.length < pos + 18) fail('invalid-authenticator-data', 'attested credential data is truncated'); + const aaguid = raw.subarray(pos, pos + 16); + const idLength = raw.readUInt16BE(pos + 16); + pos += 18; + if (idLength > 1023) fail('invalid-authenticator-data', 'credential ID exceeds 1023 bytes'); + if (raw.length < pos + idLength) fail('invalid-authenticator-data', 'credential ID is truncated'); + const credentialId = raw.subarray(pos, pos + idLength); + pos += idLength; + const keyEnd = decodeCbor(raw, pos).offset; + result.attestedCredential = { aaguid, credentialId, publicKey: raw.subarray(pos, keyEnd) }; + pos = keyEnd; + } + if (flags & FLAG_ED) { + pos = decodeCbor(raw, pos).offset; + } + if (pos !== raw.length) fail('invalid-authenticator-data', 'unexpected trailing bytes in authenticator data'); + return result; +} + +// --------------------------------------------------------------------------- +// COSE keys & signatures +// --------------------------------------------------------------------------- + +export const COSE_ALG = { ES256: -7, EdDSA: -8, ES384: -35, ES512: -36, RS256: -257 } as const; +export const SUPPORTED_ALGORITHMS: number[] = Object.values(COSE_ALG); + +const EC2_CURVES: Record = { + 1: { crv: 'P-256', alg: COSE_ALG.ES256, hash: 'sha256' }, + 2: { crv: 'P-384', alg: COSE_ALG.ES384, hash: 'sha384' }, + 3: { crv: 'P-521', alg: COSE_ALG.ES512, hash: 'sha512' }, +}; + +export interface CoseKey { + alg: number; + key: KeyObject; + /** Uncompressed EC point (0x04 || x || y), only for EC2 keys. */ + ecPoint?: Buffer; + crv?: string; +} + +export function parseCoseKey(encoded: Uint8Array): CoseKey { + const map = asMap(decodeCbor(encoded).value, 'credential public key'); + const kty = map.get(1); + const alg = map.get(3); + if (typeof alg !== 'number') return fail('unsupported-key', 'COSE key has no algorithm'); + + const bytes = (label: number, what: string): Buffer => { + const v = map.get(label); + return Buffer.isBuffer(v) ? v : fail('unsupported-key', `COSE key is missing ${what}`); + }; + + if (kty === 2) { + const curve = EC2_CURVES[map.get(-1) as number]; + if (!curve || curve.alg !== alg) return fail('unsupported-key', 'unsupported or inconsistent EC2 curve'); + const x = bytes(-2, 'x'); + const y = bytes(-3, 'y'); + const key = createPublicKey({ + key: { kty: 'EC', crv: curve.crv, x: toBase64Url(x), y: toBase64Url(y) }, + format: 'jwk', + }); + return { alg, key, ecPoint: Buffer.concat([Buffer.from([0x04]), x, y]), crv: curve.crv }; + } + if (kty === 1) { + if (map.get(-1) !== 6 || alg !== COSE_ALG.EdDSA) return fail('unsupported-key', 'only Ed25519 OKP keys are supported'); + const key = createPublicKey({ key: { kty: 'OKP', crv: 'Ed25519', x: toBase64Url(bytes(-2, 'x')) }, format: 'jwk' }); + return { alg, key }; + } + if (kty === 3) { + if (alg !== COSE_ALG.RS256) return fail('unsupported-key', 'only RS256 RSA keys are supported'); + const key = createPublicKey({ + key: { kty: 'RSA', n: toBase64Url(bytes(-1, 'n')), e: toBase64Url(bytes(-2, 'e')) }, + format: 'jwk', + }); + return { alg, key }; + } + return fail('unsupported-key', `unsupported COSE key type ${String(kty)}`); +} + +const hashForAlg = (alg: number): string | null => { + switch (alg) { + case COSE_ALG.ES256: + case COSE_ALG.RS256: + return 'sha256'; + case COSE_ALG.ES384: + return 'sha384'; + case COSE_ALG.ES512: + return 'sha512'; + case COSE_ALG.EdDSA: + return null; + default: + return fail('unsupported-algorithm', `unsupported COSE algorithm ${alg}`); + } +}; + +/** WebAuthn ECDSA signatures are ASN.1 DER encoded; Node's default dsaEncoding is 'der'. */ +export function verifySignature(alg: number, key: KeyObject, data: Buffer, signature: Buffer): boolean { + try { + return cryptoVerify(hashForAlg(alg), data, key, signature); + } catch (err) { + if (err instanceof WebAuthnError) throw err; + return false; + } +} + +// --------------------------------------------------------------------------- +// Attestation statements (§8) +// --------------------------------------------------------------------------- + +export type AttestationType = 'none' | 'self' | 'basic'; + +export interface AttestationResult { + fmt: string; + type: AttestationType; +} + +/** id-fido-gen-ce-aaguid (1.3.6.1.4.1.45724.1.1.4) DER encoding. */ +const AAGUID_EXTENSION_OID = Buffer.from([0x06, 0x0b, 0x2b, 0x06, 0x01, 0x04, 0x01, 0x82, 0xe5, 0x1c, 0x01, 0x01, 0x04]); + +/** Extracts the AAGUID certificate extension value, if present (§8.2.1). */ +export function readAaguidExtension(certDer: Buffer): { aaguid: Buffer; critical: boolean } | null { + const at = certDer.indexOf(AAGUID_EXTENSION_OID); + if (at === -1) return null; + let pos = at + AAGUID_EXTENSION_OID.length; + let critical = false; + if (certDer[pos] === 0x01 && certDer[pos + 1] === 0x01) { + critical = certDer[pos + 2] !== 0x00; + pos += 3; + } + // extnValue OCTET STRING wrapping an OCTET STRING of 16 bytes. + if (certDer[pos] !== 0x04 || certDer[pos + 2] !== 0x04 || certDer[pos + 3] !== 0x10) { + return fail('invalid-attestation', 'malformed AAGUID certificate extension'); + } + return { aaguid: certDer.subarray(pos + 4, pos + 20), critical }; +} + +function parseCertificateChain(x5c: CborValue): X509Certificate[] { + if (!Array.isArray(x5c) || x5c.length === 0) return fail('invalid-attestation', 'x5c must be a non-empty array'); + return x5c.map((der) => + Buffer.isBuffer(der) ? new X509Certificate(der) : fail('invalid-attestation', 'x5c entries must be byte strings') + ); +} + +function assertChainSignatures(chain: X509Certificate[], now: Date): void { + chain.forEach((cert, i) => { + if (now < new Date(cert.validFrom) || now > new Date(cert.validTo)) { + fail('invalid-attestation', 'attestation certificate is outside its validity period'); + } + const issuer = chain[i + 1]; + if (issuer && !cert.verify(issuer.publicKey)) { + fail('invalid-attestation', 'attestation certificate chain signature is invalid'); + } + }); +} + +export interface AttestationInput { + fmt: string; + attStmt: CborMap; + authData: AuthenticatorData; + clientDataHash: Buffer; + credentialKey: CoseKey; + now?: Date; +} + +export function verifyAttestation(input: AttestationInput): AttestationResult { + const { fmt, attStmt, authData, clientDataHash, credentialKey } = input; + const now = input.now ?? new Date(); + const signedData = Buffer.concat([authData.raw, clientDataHash]); + const credential = authData.attestedCredential!; + + if (fmt === 'none') { + if (attStmt.size !== 0) fail('invalid-attestation', "'none' attestation must have an empty statement"); + return { fmt, type: 'none' }; + } + + if (fmt === 'packed') { + const alg = attStmt.get('alg'); + const sig = attStmt.get('sig'); + if (typeof alg !== 'number' || !Buffer.isBuffer(sig)) fail('invalid-attestation', 'packed statement needs alg and sig'); + + const x5c = attStmt.get('x5c'); + if (x5c === undefined) { + // Self attestation: signed by the credential key itself. + if (alg !== credentialKey.alg) fail('invalid-attestation', 'self attestation alg does not match the credential'); + if (!verifySignature(alg as number, credentialKey.key, signedData, sig as Buffer)) { + fail('invalid-attestation', 'packed self-attestation signature is invalid'); + } + return { fmt, type: 'self' }; + } + + const chain = parseCertificateChain(x5c); + const leaf = chain[0]!; + if (!verifySignature(alg as number, leaf.publicKey, signedData, sig as Buffer)) { + fail('invalid-attestation', 'packed attestation signature is invalid'); + } + // §8.2.1 attestation certificate requirements. + if (!/OU=Authenticator Attestation/.test(leaf.subject)) { + fail('invalid-attestation', "attestation certificate subject OU must be 'Authenticator Attestation'"); + } + if (leaf.ca) fail('invalid-attestation', 'attestation certificate must not be a CA'); + const ext = readAaguidExtension(leaf.raw); + if (ext) { + if (ext.critical) fail('invalid-attestation', 'AAGUID extension must not be critical'); + if (!ext.aaguid.equals(credential.aaguid)) fail('invalid-attestation', 'certificate AAGUID does not match'); + } + assertChainSignatures(chain, now); + return { fmt, type: 'basic' }; + } + + if (fmt === 'fido-u2f') { + const sig = attStmt.get('sig'); + const chain = parseCertificateChain(attStmt.get('x5c')); + if (chain.length !== 1 || !Buffer.isBuffer(sig)) fail('invalid-attestation', 'fido-u2f needs one cert and a sig'); + const leaf = chain[0]!; + if (leaf.publicKey.asymmetricKeyDetails?.namedCurve !== 'prime256v1') { + fail('invalid-attestation', 'fido-u2f attestation key must be P-256'); + } + if (credentialKey.alg !== COSE_ALG.ES256 || !credentialKey.ecPoint) { + fail('invalid-attestation', 'fido-u2f credentials must be ES256'); + } + const verificationData = Buffer.concat([ + Buffer.from([0x00]), + authData.rpIdHash, + clientDataHash, + credential.credentialId, + credentialKey.ecPoint!, + ]); + if (!verifySignature(COSE_ALG.ES256, leaf.publicKey, verificationData, sig as Buffer)) { + fail('invalid-attestation', 'fido-u2f attestation signature is invalid'); + } + assertChainSignatures(chain, now); + return { fmt, type: 'basic' }; + } + + return fail('unsupported-attestation', `attestation format '${fmt}' is not supported`); +} + +export function decodeAttestationObject(raw: Buffer): { fmt: string; attStmt: CborMap; authData: Buffer } { + const { value, offset } = decodeCbor(raw); + if (offset !== raw.length) fail('invalid-cbor', 'trailing bytes after attestation object'); + const map = asMap(value, 'attestation object'); + const fmt = map.get('fmt'); + const authData = map.get('authData'); + if (typeof fmt !== 'string' || !Buffer.isBuffer(authData)) fail('invalid-attestation', 'malformed attestation object'); + return { fmt: fmt as string, attStmt: asMap(map.get('attStmt') ?? new Map(), 'attStmt'), authData: authData as Buffer }; +}