From 210f326f20ab05c677e0d2ac94eb4602f0f5bc5e Mon Sep 17 00:00:00 2001 From: Ashborne267 Date: Tue, 20 Oct 2026 22:54:56 +0100 Subject: [PATCH 1/4] feat: add rule pack dependency lockfile support --- libs/engine/core/index.ts | 3 + libs/engine/core/rule-pack-lockfile.spec.ts | 305 ++++++++++++++++ libs/engine/core/rule-pack-lockfile.ts | 374 ++++++++++++++++++++ 3 files changed, 682 insertions(+) create mode 100644 libs/engine/core/rule-pack-lockfile.spec.ts create mode 100644 libs/engine/core/rule-pack-lockfile.ts diff --git a/libs/engine/core/index.ts b/libs/engine/core/index.ts index c2404b2c..a60ed36a 100644 --- a/libs/engine/core/index.ts +++ b/libs/engine/core/index.ts @@ -35,3 +35,6 @@ export * from "./location-mapper"; // Analyzer plugin contract and lifecycle (#1154) export * from "./analyzer-plugin-contract"; + +// Rule pack dependency lockfile support +export * from "./rule-pack-lockfile"; diff --git a/libs/engine/core/rule-pack-lockfile.spec.ts b/libs/engine/core/rule-pack-lockfile.spec.ts new file mode 100644 index 00000000..33ca72cf --- /dev/null +++ b/libs/engine/core/rule-pack-lockfile.spec.ts @@ -0,0 +1,305 @@ +import { + generateLockfile, + parseLockfile, + serializeLockfile, + validateLockfile, + LOCKFILE_SCHEMA_VERSION, + type RulePackDescriptor, + type RulePackLockfile, +} from "./rule-pack-lockfile"; + +// ─── Fixtures ────────────────────────────────────────────────────────────────── + +const FIXED_NOW = "2024-01-15T10:00:00.000Z"; +const fixedClock = () => FIXED_NOW; + +/** Realistic two-pack scenario: a security pack that depends on a base pack. */ +const REALISTIC_PACKS: RulePackDescriptor[] = [ + { + name: "gasguard-base", + version: "1.2.0", + integrity: "a".repeat(64), + dependencies: [], + }, + { + name: "gasguard-security", + version: "2.0.1", + integrity: "b".repeat(64), + dependencies: ["gasguard-base"], + }, +]; + +/** Regression: a pack with no declared integrity (common during initial lock). */ +const NO_INTEGRITY_PACK: RulePackDescriptor = { + name: "gasguard-experimental", + version: "0.1.0", + dependencies: [], +}; + +// ─── parseLockfile ───────────────────────────────────────────────────────────── + +describe("parseLockfile", () => { + it("parses a valid lockfile object", () => { + const lockfile = generateLockfile(REALISTIC_PACKS, fixedClock); + const raw = JSON.parse(serializeLockfile(lockfile)); + const result = parseLockfile(raw); + + expect(result.valid).toBe(true); + expect(result.lockfile).toBeDefined(); + expect(result.lockfile!.schemaVersion).toBe(LOCKFILE_SCHEMA_VERSION); + expect(Object.keys(result.lockfile!.packs)).toHaveLength(2); + }); + + it("parses a valid lockfile from a JSON string", () => { + const lockfile = generateLockfile(REALISTIC_PACKS, fixedClock); + const json = serializeLockfile(lockfile); + const result = parseLockfile(json); + + expect(result.valid).toBe(true); + expect(result.lockfile).toBeDefined(); + }); + + it("returns a fatal issue for invalid JSON string", () => { + const result = parseLockfile("not json {{{"); + + expect(result.valid).toBe(false); + expect(result.lockfile).toBeUndefined(); + expect(result.issues[0]?.code).toBe("MALFORMED_LOCKFILE"); + expect(result.issues[0]?.fatal).toBe(true); + }); + + it("returns a fatal issue for non-object root", () => { + const result = parseLockfile([1, 2, 3]); + + expect(result.valid).toBe(false); + expect(result.issues[0]?.code).toBe("MALFORMED_LOCKFILE"); + }); + + it("returns a fatal issue for unknown schemaVersion", () => { + const result = parseLockfile({ schemaVersion: 999, lockedAt: FIXED_NOW, packs: {} }); + + expect(result.valid).toBe(false); + expect(result.issues[0]?.code).toBe("UNKNOWN_SCHEMA_VERSION"); + }); + + it("skips pack entries that are not objects but keeps valid ones", () => { + const raw = { + schemaVersion: LOCKFILE_SCHEMA_VERSION, + lockedAt: FIXED_NOW, + packs: { + "good-pack": { version: "1.0.0", integrity: "", dependencies: [], resolvedAt: FIXED_NOW }, + "bad-pack": "not-an-object", + }, + }; + const result = parseLockfile(raw); + + // The malformed entry is skipped (non-fatal issue) but the rest is usable. + expect(result.valid).toBe(true); + expect(result.lockfile!.packs["good-pack"]).toBeDefined(); + expect(result.lockfile!.packs["bad-pack"]).toBeUndefined(); + expect(result.issues.some((i) => i.code === "INVALID_ENTRY")).toBe(true); + }); + + it("preserves unknown top-level keys in extra", () => { + const raw = { + schemaVersion: LOCKFILE_SCHEMA_VERSION, + lockedAt: FIXED_NOW, + packs: {}, + futureField: "some-future-value", + }; + const result = parseLockfile(raw); + + expect(result.valid).toBe(true); + expect(result.lockfile!.extra?.futureField).toBe("some-future-value"); + }); + + it("regression: pack with missing version is skipped with INVALID_ENTRY", () => { + const raw = { + schemaVersion: LOCKFILE_SCHEMA_VERSION, + lockedAt: FIXED_NOW, + packs: { + "no-version": { integrity: "", dependencies: [], resolvedAt: FIXED_NOW }, + }, + }; + const result = parseLockfile(raw); + + expect(result.lockfile!.packs["no-version"]).toBeUndefined(); + expect(result.issues.some((i) => i.code === "INVALID_ENTRY" && i.pack === "no-version")).toBe(true); + }); +}); + +// ─── generateLockfile ───────────────────────────────────────────────────────── + +describe("generateLockfile", () => { + it("produces a lockfile with sorted pack names", () => { + const packs = [ + { name: "zzz-pack", version: "1.0.0" }, + { name: "aaa-pack", version: "2.0.0" }, + ]; + const lockfile = generateLockfile(packs, fixedClock); + const names = Object.keys(lockfile.packs); + + expect(names[0]).toBe("aaa-pack"); + expect(names[1]).toBe("zzz-pack"); + }); + + it("uses the provided clock for lockedAt and resolvedAt", () => { + const lockfile = generateLockfile(REALISTIC_PACKS, fixedClock); + + expect(lockfile.lockedAt).toBe(FIXED_NOW); + expect(lockfile.packs["gasguard-base"]?.resolvedAt).toBe(FIXED_NOW); + }); + + it("defaults integrity to empty string when not provided", () => { + const lockfile = generateLockfile([NO_INTEGRITY_PACK], fixedClock); + + expect(lockfile.packs["gasguard-experimental"]?.integrity).toBe(""); + }); + + it("realistic: generates a deterministic lockfile for a two-pack set", () => { + const a = generateLockfile(REALISTIC_PACKS, fixedClock); + const b = generateLockfile([...REALISTIC_PACKS].reverse(), fixedClock); + + // Regardless of input order the output must be identical. + expect(serializeLockfile(a)).toBe(serializeLockfile(b)); + }); +}); + +// ─── serializeLockfile ──────────────────────────────────────────────────────── + +describe("serializeLockfile", () => { + it("round-trips through parseLockfile", () => { + const original = generateLockfile(REALISTIC_PACKS, fixedClock); + const json = serializeLockfile(original); + const { lockfile: restored } = parseLockfile(json); + + expect(restored).toBeDefined(); + expect(restored!.packs["gasguard-base"]?.version).toBe("1.2.0"); + expect(restored!.packs["gasguard-security"]?.dependencies).toEqual(["gasguard-base"]); + }); + + it("produces byte-identical output for identical logical content", () => { + const a = serializeLockfile(generateLockfile(REALISTIC_PACKS, fixedClock)); + const b = serializeLockfile(generateLockfile(REALISTIC_PACKS, fixedClock)); + + expect(a).toBe(b); + }); + + it("ends with a newline", () => { + const json = serializeLockfile(generateLockfile([], fixedClock)); + expect(json.endsWith("\n")).toBe(true); + }); +}); + +// ─── validateLockfile ───────────────────────────────────────────────────────── + +describe("validateLockfile", () => { + function makeLockfile(packs: RulePackDescriptor[]): RulePackLockfile { + return generateLockfile(packs, fixedClock); + } + + it("passes when requested packs match the lockfile exactly", () => { + const lockfile = makeLockfile(REALISTIC_PACKS); + const result = validateLockfile(lockfile, REALISTIC_PACKS); + + expect(result.valid).toBe(true); + expect(result.issues).toHaveLength(0); + }); + + it("reports MISSING_PACK_ENTRY when a required pack is absent", () => { + const lockfile = makeLockfile([REALISTIC_PACKS[0]!]); + const result = validateLockfile(lockfile, REALISTIC_PACKS); + + expect(result.valid).toBe(false); + const issue = result.issues.find((i) => i.code === "MISSING_PACK_ENTRY"); + expect(issue).toBeDefined(); + expect(issue!.pack).toBe("gasguard-security"); + }); + + it("reports VERSION_MISMATCH when versions differ", () => { + const lockfile = makeLockfile(REALISTIC_PACKS); + const requested: RulePackDescriptor[] = [ + { name: "gasguard-base", version: "9.9.9" }, + ]; + const result = validateLockfile(lockfile, requested); + + expect(result.valid).toBe(false); + expect(result.issues.some((i) => i.code === "VERSION_MISMATCH")).toBe(true); + }); + + it("reports INTEGRITY_MISMATCH when both sides have a hash and they differ", () => { + const lockfile = makeLockfile(REALISTIC_PACKS); + const requested: RulePackDescriptor[] = [ + { name: "gasguard-base", version: "1.2.0", integrity: "c".repeat(64) }, + ]; + const result = validateLockfile(lockfile, requested); + + expect(result.valid).toBe(false); + expect(result.issues.some((i) => i.code === "INTEGRITY_MISMATCH")).toBe(true); + }); + + it("does NOT report integrity mismatch when lockfile entry has empty integrity", () => { + const lockfile = makeLockfile([ + { name: "gasguard-base", version: "1.2.0", integrity: "", dependencies: [] }, + ]); + const requested: RulePackDescriptor[] = [ + { name: "gasguard-base", version: "1.2.0", integrity: "a".repeat(64) }, + ]; + const result = validateLockfile(lockfile, requested); + + expect(result.valid).toBe(true); + }); + + it("reports MISSING_DEPENDENCY when a declared dep is absent from the lockfile", () => { + const packs: RulePackDescriptor[] = [ + { + name: "gasguard-security", + version: "2.0.1", + dependencies: ["gasguard-base"], + }, + // gasguard-base deliberately omitted + ]; + const lockfile = makeLockfile(packs); + const result = validateLockfile(lockfile, packs); + + expect(result.valid).toBe(false); + expect(result.issues.some((i) => i.code === "MISSING_DEPENDENCY")).toBe(true); + }); + + it("reports CIRCULAR_DEPENDENCY for a direct cycle", () => { + // Manually craft a lockfile with a cycle since generateLockfile does not validate. + const lockfile: RulePackLockfile = { + schemaVersion: LOCKFILE_SCHEMA_VERSION, + lockedAt: FIXED_NOW, + packs: { + "pack-a": { version: "1.0.0", integrity: "", dependencies: ["pack-b"], resolvedAt: FIXED_NOW }, + "pack-b": { version: "1.0.0", integrity: "", dependencies: ["pack-a"], resolvedAt: FIXED_NOW }, + }, + }; + const result = validateLockfile(lockfile, []); + + expect(result.valid).toBe(false); + expect(result.issues.some((i) => i.code === "CIRCULAR_DEPENDENCY")).toBe(true); + }); + + it("regression: empty lockfile with no requested packs is valid", () => { + const lockfile = makeLockfile([]); + const result = validateLockfile(lockfile, []); + + expect(result.valid).toBe(true); + expect(result.issues).toHaveLength(0); + }); + + it("realistic: full two-pack lifecycle — generate, serialize, parse, validate", () => { + const generated = generateLockfile(REALISTIC_PACKS, fixedClock); + const json = serializeLockfile(generated); + const { lockfile, valid: parseValid } = parseLockfile(json); + + expect(parseValid).toBe(true); + + const { valid, issues } = validateLockfile(lockfile!, REALISTIC_PACKS); + + expect(valid).toBe(true); + expect(issues).toHaveLength(0); + }); +}); diff --git a/libs/engine/core/rule-pack-lockfile.ts b/libs/engine/core/rule-pack-lockfile.ts new file mode 100644 index 00000000..c8df243a --- /dev/null +++ b/libs/engine/core/rule-pack-lockfile.ts @@ -0,0 +1,374 @@ +/** + * Rule pack dependency lockfile support. + * + * A lockfile pins the exact version of every rule pack and its transitive + * dependencies so that two scans against the same commit produce byte-identical + * findings regardless of when or where they run. + * + * Design notes + * ────────────── + * - Pure: no I/O, no global state, no clocks. File reading is the caller's + * responsibility; the module accepts already-parsed objects. + * - Tenant-isolated: every lockfile instance is independent; no shared singleton. + * - Deterministic: serialisation sorts all keys and entries before writing so + * the file does not churn on re-lock of an unchanged dependency set. + * - Backward-compatible: unknown top-level keys in a parsed lockfile are + * preserved as `extra` so future versions do not silently drop data. + */ + +export const LOCKFILE_SCHEMA_VERSION = 1; +export const LOCKFILE_FILENAME = "gasguard.lock"; + +// ─── Public types ───────────────────────────────────────────────────────────── + +export interface LockedPackEntry { + /** Exact resolved version, e.g. "2.3.1". */ + version: string; + /** + * Content-hash of the pack at that version (SHA-256, hex, 64 chars). + * A missing hash is accepted on read but written as "" so callers can + * detect stale or incomplete entries. + */ + integrity: string; + /** Names of other locked packs this entry depends on. */ + dependencies: string[]; + /** ISO-8601 timestamp when the entry was last resolved. */ + resolvedAt: string; +} + +export interface RulePackLockfile { + schemaVersion: number; + /** ISO-8601 timestamp of the most recent `lock` call. */ + lockedAt: string; + /** Sorted map of pack-name → locked entry. */ + packs: Record; + /** + * Unknown top-level keys from a parsed file; preserved so round-tripping + * an older or newer lockfile never discards data. + */ + extra?: Record; +} + +export type LockfileIssueCode = + | "MALFORMED_LOCKFILE" + | "UNKNOWN_SCHEMA_VERSION" + | "MISSING_PACK_ENTRY" + | "VERSION_MISMATCH" + | "INTEGRITY_MISMATCH" + | "MISSING_DEPENDENCY" + | "CIRCULAR_DEPENDENCY" + | "INVALID_ENTRY"; + +export interface LockfileIssue { + code: LockfileIssueCode; + message: string; + pack?: string; + /** True when the issue causes the lockfile to be treated as invalid. */ + fatal: boolean; +} + +export interface LockfileParseResult { + lockfile: RulePackLockfile | undefined; + issues: LockfileIssue[]; + valid: boolean; +} + +export interface LockfileValidationResult { + valid: boolean; + issues: LockfileIssue[]; +} + +/** Minimal descriptor required to lock a rule pack. */ +export interface RulePackDescriptor { + name: string; + version: string; + /** SHA-256 hex hash of the pack contents. May be "" when not yet computed. */ + integrity?: string; + /** Names of other packs this one depends on. */ + dependencies?: string[]; +} + +// ─── Parsing ────────────────────────────────────────────────────────────────── + +function isRecord(v: unknown): v is Record { + return v !== null && typeof v === "object" && !Array.isArray(v); +} + +/** + * Parses a raw JSON-decoded object (or string) into a `RulePackLockfile`. + * + * Never throws. All structural problems are returned as issues. + */ +export function parseLockfile(raw: unknown): LockfileParseResult { + const issues: LockfileIssue[] = []; + + if (typeof raw === "string") { + try { + raw = JSON.parse(raw); + } catch { + issues.push({ + code: "MALFORMED_LOCKFILE", + message: "Lockfile content is not valid JSON.", + fatal: true, + }); + return { lockfile: undefined, issues, valid: false }; + } + } + + if (!isRecord(raw)) { + issues.push({ + code: "MALFORMED_LOCKFILE", + message: "Lockfile root must be a JSON object.", + fatal: true, + }); + return { lockfile: undefined, issues, valid: false }; + } + + const { schemaVersion, lockedAt, packs, ...rest } = raw as Record; + + if (typeof schemaVersion !== "number" || schemaVersion !== LOCKFILE_SCHEMA_VERSION) { + issues.push({ + code: "UNKNOWN_SCHEMA_VERSION", + message: `Unsupported lockfile schemaVersion: ${String(schemaVersion)}. Expected ${LOCKFILE_SCHEMA_VERSION}.`, + fatal: true, + }); + return { lockfile: undefined, issues, valid: false }; + } + + const lockedAtStr = + typeof lockedAt === "string" && lockedAt.trim() !== "" + ? lockedAt.trim() + : new Date(0).toISOString(); + + const parsedPacks: Record = {}; + + if (!isRecord(packs)) { + issues.push({ + code: "MALFORMED_LOCKFILE", + message: "Lockfile `packs` field must be an object.", + fatal: true, + }); + return { lockfile: undefined, issues, valid: false }; + } + + for (const [name, rawEntry] of Object.entries(packs)) { + if (!isRecord(rawEntry)) { + issues.push({ + code: "INVALID_ENTRY", + message: `Pack entry "${name}" is not an object.`, + pack: name, + fatal: false, + }); + continue; + } + + const version = + typeof rawEntry.version === "string" ? rawEntry.version.trim() : ""; + const integrity = + typeof rawEntry.integrity === "string" ? rawEntry.integrity.trim() : ""; + const dependencies = Array.isArray(rawEntry.dependencies) + ? rawEntry.dependencies.filter((d): d is string => typeof d === "string") + : []; + const resolvedAt = + typeof rawEntry.resolvedAt === "string" && rawEntry.resolvedAt.trim() !== "" + ? rawEntry.resolvedAt.trim() + : new Date(0).toISOString(); + + if (version === "") { + issues.push({ + code: "INVALID_ENTRY", + message: `Pack entry "${name}" is missing a version.`, + pack: name, + fatal: false, + }); + continue; + } + + parsedPacks[name] = { version, integrity, dependencies, resolvedAt }; + } + + const extra = Object.keys(rest).length > 0 ? rest : undefined; + + const lockfile: RulePackLockfile = { + schemaVersion: LOCKFILE_SCHEMA_VERSION, + lockedAt: lockedAtStr, + packs: parsedPacks, + ...(extra !== undefined ? { extra } : {}), + }; + + return { lockfile, issues, valid: issues.every((i) => !i.fatal) }; +} + +// ─── Validation ─────────────────────────────────────────────────────────────── + +/** + * Validates a parsed lockfile against a set of requested pack descriptors. + * + * Returns issues for: + * - packs required by the caller that are absent from the lockfile + * - version mismatches between the lockfile and the requested descriptor + * - integrity mismatches (when both sides supply a non-empty hash) + * - declared dependencies that are not present in the lockfile + * - circular dependency chains + */ +export function validateLockfile( + lockfile: RulePackLockfile, + requested: RulePackDescriptor[], +): LockfileValidationResult { + const issues: LockfileIssue[] = []; + + for (const req of requested) { + const entry = lockfile.packs[req.name]; + if (!entry) { + issues.push({ + code: "MISSING_PACK_ENTRY", + message: `Rule pack "${req.name}" is not present in the lockfile. Run gasguard lock to regenerate it.`, + pack: req.name, + fatal: true, + }); + continue; + } + + if (entry.version !== req.version) { + issues.push({ + code: "VERSION_MISMATCH", + message: + `Rule pack "${req.name}" version mismatch: lockfile has "${entry.version}", requested "${req.version}".`, + pack: req.name, + fatal: true, + }); + } + + if ( + req.integrity && + req.integrity !== "" && + entry.integrity !== "" && + entry.integrity !== req.integrity + ) { + issues.push({ + code: "INTEGRITY_MISMATCH", + message: + `Rule pack "${req.name}" integrity mismatch: lockfile has "${entry.integrity}", got "${req.integrity}".`, + pack: req.name, + fatal: true, + }); + } + } + + // Dependency completeness: every declared dep must itself be in the lockfile. + for (const [name, entry] of Object.entries(lockfile.packs)) { + for (const dep of entry.dependencies) { + if (!lockfile.packs[dep]) { + issues.push({ + code: "MISSING_DEPENDENCY", + message: `Rule pack "${name}" depends on "${dep}", which is not in the lockfile.`, + pack: name, + fatal: true, + }); + } + } + } + + // Cycle detection via DFS. + const cycleIssues = detectDependencyCycles(lockfile.packs); + issues.push(...cycleIssues); + + return { valid: issues.every((i) => !i.fatal), issues }; +} + +function detectDependencyCycles( + packs: Record, +): LockfileIssue[] { + const issues: LockfileIssue[] = []; + const visited = new Set(); + const path: string[] = []; + + const visit = (name: string): void => { + if (path.includes(name)) { + const cycle = [...path.slice(path.indexOf(name)), name]; + issues.push({ + code: "CIRCULAR_DEPENDENCY", + message: `Circular rule pack dependency detected: ${cycle.join(" → ")}.`, + pack: name, + fatal: true, + }); + return; + } + if (visited.has(name)) return; + path.push(name); + for (const dep of packs[name]?.dependencies ?? []) { + visit(dep); + } + path.pop(); + visited.add(name); + }; + + for (const name of Object.keys(packs)) visit(name); + return issues; +} + +// ─── Locking (generation) ───────────────────────────────────────────────────── + +/** + * Produces a new lockfile from a set of resolved pack descriptors. + * + * `now` is injectable so tests can pin the timestamp. + */ +export function generateLockfile( + packs: RulePackDescriptor[], + now: () => string = () => new Date().toISOString(), +): RulePackLockfile { + const timestamp = now(); + const entries: Record = {}; + + for (const pack of packs) { + entries[pack.name] = { + version: pack.version, + integrity: pack.integrity ?? "", + dependencies: [...(pack.dependencies ?? [])].sort(), + resolvedAt: timestamp, + }; + } + + // Sort packs deterministically. + const sortedEntries = Object.fromEntries( + Object.entries(entries).sort(([a], [b]) => a.localeCompare(b)), + ); + + return { + schemaVersion: LOCKFILE_SCHEMA_VERSION, + lockedAt: timestamp, + packs: sortedEntries, + }; +} + +// ─── Serialisation ──────────────────────────────────────────────────────────── + +/** + * Serialises a lockfile to a formatted JSON string. + * + * Keys are sorted at every level so the output is byte-identical for the same + * logical content, making VCS diffs minimal. + */ +export function serializeLockfile(lockfile: RulePackLockfile): string { + const { extra, ...rest } = lockfile; + const obj: Record = { + schemaVersion: rest.schemaVersion, + lockedAt: rest.lockedAt, + packs: Object.fromEntries( + Object.entries(rest.packs) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, entry]) => [ + name, + { + version: entry.version, + integrity: entry.integrity, + dependencies: [...entry.dependencies].sort(), + resolvedAt: entry.resolvedAt, + }, + ]), + ), + ...(extra ?? {}), + }; + return JSON.stringify(obj, null, 2) + "\n"; +} From f51a98e5f9ac81bc2da1abc9eebb2ff48a5e126f Mon Sep 17 00:00:00 2001 From: Ashborne267 Date: Tue, 20 Oct 2026 22:56:31 +0100 Subject: [PATCH 2/4] feat: add language-specific analyzer capability registry --- libs/engine/core/index.ts | 3 + .../core/language-capability-registry.spec.ts | 229 ++++++++++++++ .../core/language-capability-registry.ts | 294 ++++++++++++++++++ 3 files changed, 526 insertions(+) create mode 100644 libs/engine/core/language-capability-registry.spec.ts create mode 100644 libs/engine/core/language-capability-registry.ts diff --git a/libs/engine/core/index.ts b/libs/engine/core/index.ts index a60ed36a..0bb90c48 100644 --- a/libs/engine/core/index.ts +++ b/libs/engine/core/index.ts @@ -38,3 +38,6 @@ export * from "./analyzer-plugin-contract"; // Rule pack dependency lockfile support export * from "./rule-pack-lockfile"; + +// Language-specific analyzer capability registry +export * from "./language-capability-registry"; diff --git a/libs/engine/core/language-capability-registry.spec.ts b/libs/engine/core/language-capability-registry.spec.ts new file mode 100644 index 00000000..d75dbfbf --- /dev/null +++ b/libs/engine/core/language-capability-registry.spec.ts @@ -0,0 +1,229 @@ +import { + buildLanguageCapabilityRegistry, + getLanguageEntry, + getAnalyzersForLanguage, + getMergedCapabilitiesForLanguage, + getAllRulesForLanguage, + isRegistryHealthy, +} from "./language-capability-registry"; +import { + BaseAnalyzer, + Language, + Severity, + type AnalysisResult, + type AnalyzerConfig, +} from "./analyzer-interface"; + +// ─── Test stubs ─────────────────────────────────────────────────────────────── + +class StubAnalyzer extends BaseAnalyzer { + constructor( + private readonly _name: string, + private readonly _version: string, + private readonly _languages: Language[], + private readonly _rules: Array<{ id: string; name: string; severity: Severity; enabled: boolean }> = [], + ) { + super(); + } + + getName() { return this._name; } + getVersion() { return this._version; } + getSupportedLanguages() { return this._languages; } + supportsLanguage(lang: Language | string) { return (this._languages as string[]).includes(lang as string); } + getRules() { + return this._rules.map((r) => ({ + ...r, + description: `${r.name} description`, + category: "test", + })); + } + + async analyze(_code: string, filePath: string, _config?: AnalyzerConfig): Promise { + return { + findings: [], + filesAnalyzed: 1, + analysisTime: 0, + analyzerVersion: this._version, + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + }; + } +} + +class ThrowingAnalyzer extends StubAnalyzer { + getSupportedLanguages(): Language[] { + throw new Error("introspection failure"); + } +} + +const FIXED_NOW = "2024-06-01T00:00:00.000Z"; +const clock = () => FIXED_NOW; + +// ─── buildLanguageCapabilityRegistry ───────────────────────────────────────── + +describe("buildLanguageCapabilityRegistry", () => { + it("returns an empty snapshot with NO_ANALYZERS_REGISTERED issue when called with no analyzers", () => { + const snap = buildLanguageCapabilityRegistry([], [], clock); + + expect(snap.coveredLanguages).toHaveLength(0); + expect(snap.entries).toHaveLength(0); + expect(snap.issues[0]?.code).toBe("NO_ANALYZERS_REGISTERED"); + expect(snap.builtAt).toBe(FIXED_NOW); + }); + + it("builds correct coverage entries for a single-language analyzer", () => { + const analyzer = new StubAnalyzer("sol-analyzer", "1.0.0", [Language.SOLIDITY]); + const snap = buildLanguageCapabilityRegistry([analyzer], [], clock); + + expect(snap.coveredLanguages).toContain(Language.SOLIDITY); + expect(snap.entries).toHaveLength(1); + expect(snap.entries[0]!.covered).toBe(true); + expect(snap.entries[0]!.analyzers[0]!.analyzerName).toBe("sol-analyzer"); + }); + + it("surfaces an UNCOVERED_LANGUAGE issue for an expected language with no analyzer", () => { + const analyzer = new StubAnalyzer("sol-analyzer", "1.0.0", [Language.SOLIDITY]); + const snap = buildLanguageCapabilityRegistry( + [analyzer], + [Language.SOLIDITY, Language.RUST], + clock, + ); + + expect(snap.uncoveredLanguages).toContain(Language.RUST); + expect(snap.issues.some((i) => i.code === "UNCOVERED_LANGUAGE" && i.language === Language.RUST)).toBe(true); + // Covered entry still exists for RUST so callers can see it. + const rustEntry = snap.entries.find((e) => e.language === Language.RUST); + expect(rustEntry?.covered).toBe(false); + }); + + it("records INTROSPECTION_ERROR and continues when an analyzer throws", () => { + const good = new StubAnalyzer("good", "1.0.0", [Language.SOLIDITY]); + const bad = new ThrowingAnalyzer("bad", "1.0.0", []); + const snap = buildLanguageCapabilityRegistry([good, bad], [], clock); + + // good analyzer still produces its entry + expect(snap.coveredLanguages).toContain(Language.SOLIDITY); + expect(snap.issues.some((i) => i.code === "INTROSPECTION_ERROR" && i.analyzerName === "bad")).toBe(true); + }); + + it("entries are sorted alphabetically by language", () => { + const a = new StubAnalyzer("z", "1.0.0", [Language.TYPESCRIPT]); + const b = new StubAnalyzer("a", "1.0.0", [Language.SOLIDITY]); + const snap = buildLanguageCapabilityRegistry([a, b], [], clock); + + const languages = snap.entries.map((e) => e.language); + expect(languages).toEqual([...languages].sort()); + }); + + it("realistic: two analyzers covering different languages plus one expected gap", () => { + const solAnalyzer = new StubAnalyzer( + "solidity-analyzer", + "2.1.0", + [Language.SOLIDITY], + [ + { id: "sol-001", name: "Reentrancy", severity: Severity.CRITICAL, enabled: true }, + { id: "sol-002", name: "tx.origin", severity: Severity.HIGH, enabled: true }, + ], + ); + const rustAnalyzer = new StubAnalyzer( + "rust-analyzer", + "1.0.0", + [Language.RUST], + [ + { id: "rust-001", name: "UnsafeBlock", severity: Severity.MEDIUM, enabled: true }, + ], + ); + + const snap = buildLanguageCapabilityRegistry( + [solAnalyzer, rustAnalyzer], + [Language.SOLIDITY, Language.RUST, Language.VYPER], + clock, + ); + + expect(snap.coveredLanguages).toEqual([Language.RUST, Language.SOLIDITY].sort()); + expect(snap.uncoveredLanguages).toEqual([Language.VYPER]); + expect(snap.issues.some((i) => i.code === "UNCOVERED_LANGUAGE")).toBe(true); + expect(snap.issues.filter((i) => i.code === "UNCOVERED_LANGUAGE")).toHaveLength(1); + }); +}); + +// ─── Query helpers ──────────────────────────────────────────────────────────── + +describe("getLanguageEntry", () => { + it("returns the entry for a covered language", () => { + const a = new StubAnalyzer("a", "1.0.0", [Language.SOLIDITY]); + const snap = buildLanguageCapabilityRegistry([a], [], clock); + const entry = getLanguageEntry(snap, Language.SOLIDITY); + + expect(entry).toBeDefined(); + expect(entry!.language).toBe(Language.SOLIDITY); + }); + + it("returns undefined for an unknown language", () => { + const snap = buildLanguageCapabilityRegistry([], [], clock); + expect(getLanguageEntry(snap, "unknown-lang")).toBeUndefined(); + }); +}); + +describe("getAnalyzersForLanguage", () => { + it("returns all analyzer names for a covered language", () => { + const a = new StubAnalyzer("alpha", "1.0.0", [Language.SOLIDITY]); + const b = new StubAnalyzer("beta", "1.0.0", [Language.SOLIDITY]); + const snap = buildLanguageCapabilityRegistry([a, b], [], clock); + + expect(getAnalyzersForLanguage(snap, Language.SOLIDITY)).toEqual(["alpha", "beta"]); + }); + + it("returns empty array for an unregistered language", () => { + const snap = buildLanguageCapabilityRegistry([], [], clock); + expect(getAnalyzersForLanguage(snap, "cobol")).toEqual([]); + }); +}); + +describe("getMergedCapabilitiesForLanguage", () => { + it("merges capabilities from multiple analyzers covering the same language", () => { + const a = new StubAnalyzer("a", "1.0.0", [Language.SOLIDITY]); + const b = new StubAnalyzer("b", "1.0.0", [Language.SOLIDITY]); + // Default getCapabilities from BaseAnalyzer includes batchAnalysis, incremental, etc. + const snap = buildLanguageCapabilityRegistry([a, b], [], clock); + const merged = getMergedCapabilitiesForLanguage(snap, Language.SOLIDITY); + + expect(merged.batchAnalysis).toBe(true); + expect(merged.incremental).toBe(true); + }); + + it("returns empty object for unknown language", () => { + const snap = buildLanguageCapabilityRegistry([], [], clock); + expect(getMergedCapabilitiesForLanguage(snap, "unknown")).toEqual({}); + }); +}); + +describe("getAllRulesForLanguage", () => { + it("returns deduplicated rules for a language", () => { + const sharedRule = { id: "shared-001", name: "Shared", severity: Severity.LOW, enabled: true }; + const a = new StubAnalyzer("a", "1.0.0", [Language.SOLIDITY], [sharedRule]); + const b = new StubAnalyzer("b", "1.0.0", [Language.SOLIDITY], [sharedRule]); + const snap = buildLanguageCapabilityRegistry([a, b], [], clock); + + const rules = getAllRulesForLanguage(snap, Language.SOLIDITY); + expect(rules.filter((r) => r.id === "shared-001")).toHaveLength(1); + }); + + it("returns empty array for unknown language", () => { + const snap = buildLanguageCapabilityRegistry([], [], clock); + expect(getAllRulesForLanguage(snap, "fortran")).toEqual([]); + }); +}); + +describe("isRegistryHealthy", () => { + it("returns true when there are no issues", () => { + const a = new StubAnalyzer("a", "1.0.0", [Language.SOLIDITY]); + const snap = buildLanguageCapabilityRegistry([a], [], clock); + expect(isRegistryHealthy(snap)).toBe(true); + }); + + it("returns false when there are uncovered expected languages", () => { + const a = new StubAnalyzer("a", "1.0.0", [Language.SOLIDITY]); + const snap = buildLanguageCapabilityRegistry([a], [Language.RUST], clock); + expect(isRegistryHealthy(snap)).toBe(false); + }); +}); diff --git a/libs/engine/core/language-capability-registry.ts b/libs/engine/core/language-capability-registry.ts new file mode 100644 index 00000000..436cbc1b --- /dev/null +++ b/libs/engine/core/language-capability-registry.ts @@ -0,0 +1,294 @@ +/** + * Language-specific analyzer capability registry. + * + * The existing `AnalyzerRegistry` maps languages to analyzers but does not + * expose per-language capability metadata in a queryable, documented form. + * This module adds that layer without touching the registry internals. + * + * Design + * ────── + * - Immutable after construction: snapshot semantics prevent subtle concurrency + * bugs and make output deterministic for the same inputs. + * - Tenant-safe: no global state; callers construct one instance per scan context. + * - Unsupported-language handling: queries for unregistered languages return + * `undefined` / empty arrays so callers never receive silent zero-findings. + * - Partial failure: individual analyzer introspection errors are recorded as + * `RegistryIssue` entries rather than thrown, so one bad analyzer does not + * prevent the registry from being built. + */ + +import type { Analyzer, AnalyzerCapabilities, Language, Rule } from "./analyzer-interface"; + +// ─── Public types ───────────────────────────────────────────────────────────── + +/** + * Capability summary for a single analyzer scoped to a specific language. + * Fields that cannot be determined are omitted (rather than set to false/0) + * so callers can distinguish "not supported" from "not reported". + */ +export interface LanguageAnalyzerCapability { + /** Stable identifier for the analyzer. */ + analyzerName: string; + /** Declared version string. */ + analyzerVersion: string; + /** The language this entry describes. */ + language: string; + /** Capabilities advertised by the analyzer (may be a subset of all caps). */ + capabilities: AnalyzerCapabilities; + /** Rules active for this language (id + severity). */ + rules: Array<{ id: string; name: string; severity: string; enabled: boolean }>; +} + +/** + * All capability information for a single language. + */ +export interface LanguageCapabilityEntry { + language: string; + /** Analyzers that claim this language, in registration order. */ + analyzers: LanguageAnalyzerCapability[]; + /** + * True when at least one registered analyzer covers this language. + * A `false` entry for an expected language surfaces a coverage gap. + */ + covered: boolean; +} + +export type CapabilityRegistryIssueCode = + | "INTROSPECTION_ERROR" + | "UNCOVERED_LANGUAGE" + | "NO_ANALYZERS_REGISTERED"; + +export interface CapabilityRegistryIssue { + code: CapabilityRegistryIssueCode; + message: string; + analyzerName?: string; + language?: string; +} + +export interface LanguageCapabilityRegistrySnapshot { + /** Sorted list of all languages that have at least one analyzer. */ + coveredLanguages: string[]; + /** + * Expected languages with no analyzer — populated only when + * `expectedLanguages` was passed to `buildLanguageCapabilityRegistry`. + */ + uncoveredLanguages: string[]; + /** Per-language capability entries, sorted by language. */ + entries: LanguageCapabilityEntry[]; + issues: CapabilityRegistryIssue[]; + /** ISO-8601 timestamp of when the snapshot was built. */ + builtAt: string; +} + +// ─── Builder ────────────────────────────────────────────────────────────────── + +/** + * Builds an immutable `LanguageCapabilityRegistrySnapshot` from a set of analyzers. + * + * @param analyzers Analyzers to introspect. + * @param expectedLanguages Languages the product claims to support; used to + * surface coverage gaps as issues. + * @param now Injectable clock for deterministic snapshots in tests. + */ +export function buildLanguageCapabilityRegistry( + analyzers: Analyzer[], + expectedLanguages: Array = [], + now: () => string = () => new Date().toISOString(), +): LanguageCapabilityRegistrySnapshot { + const issues: CapabilityRegistryIssue[] = []; + const languageMap = new Map(); + + if (analyzers.length === 0) { + issues.push({ + code: "NO_ANALYZERS_REGISTERED", + message: "No analyzers are registered; no language coverage is possible.", + }); + } + + for (const analyzer of analyzers) { + let languages: string[]; + let capabilities: AnalyzerCapabilities; + let rules: Rule[]; + + try { + languages = (analyzer.getSupportedLanguages() ?? []).map(String); + } catch (err) { + issues.push({ + code: "INTROSPECTION_ERROR", + analyzerName: analyzer.getName(), + message: `getSupportedLanguages() threw: ${String(err)}`, + }); + continue; + } + + try { + capabilities = analyzer.getCapabilities?.() ?? { + batchAnalysis: true, + languages: analyzer.getSupportedLanguages(), + }; + } catch (err) { + issues.push({ + code: "INTROSPECTION_ERROR", + analyzerName: analyzer.getName(), + message: `getCapabilities() threw: ${String(err)}`, + }); + capabilities = { batchAnalysis: true }; + } + + try { + rules = analyzer.getRules(); + } catch (err) { + issues.push({ + code: "INTROSPECTION_ERROR", + analyzerName: analyzer.getName(), + message: `getRules() threw: ${String(err)}`, + }); + rules = []; + } + + const ruleEntries = rules.map((r) => ({ + id: r.id, + name: r.name, + severity: r.severity as string, + enabled: r.enabled, + })); + + for (const lang of languages) { + if (!languageMap.has(lang)) { + languageMap.set(lang, { + language: lang, + analyzers: [], + covered: true, + }); + } + + languageMap.get(lang)!.analyzers.push({ + analyzerName: analyzer.getName(), + analyzerVersion: analyzer.getVersion(), + language: lang, + capabilities, + rules: ruleEntries, + }); + } + } + + // Expected-language gap detection. + const uncoveredLanguages: string[] = []; + for (const lang of expectedLanguages.map(String)) { + if (!languageMap.has(lang)) { + uncoveredLanguages.push(lang); + issues.push({ + code: "UNCOVERED_LANGUAGE", + language: lang, + message: + `Language "${lang}" has no registered analyzer. Scans will return ` + + `zero findings for this language, which is indistinguishable from a clean result.`, + }); + // Add a placeholder entry so callers can see it explicitly. + languageMap.set(lang, { + language: lang, + analyzers: [], + covered: false, + }); + } + } + + const entries = [...languageMap.values()].sort((a, b) => + a.language.localeCompare(b.language), + ); + + const coveredLanguages = entries + .filter((e) => e.covered) + .map((e) => e.language) + .sort(); + + return { + coveredLanguages, + uncoveredLanguages: [...uncoveredLanguages].sort(), + entries, + issues, + builtAt: now(), + }; +} + +// ─── Query helpers ──────────────────────────────────────────────────────────── + +/** + * Returns the capability entry for a single language, or `undefined` when the + * language is not present in the snapshot. + */ +export function getLanguageEntry( + snapshot: LanguageCapabilityRegistrySnapshot, + language: string, +): LanguageCapabilityEntry | undefined { + return snapshot.entries.find((e) => e.language === language); +} + +/** + * Returns all analyzer names that cover a given language. + * Returns an empty array (never throws) for unregistered languages. + */ +export function getAnalyzersForLanguage( + snapshot: LanguageCapabilityRegistrySnapshot, + language: string, +): string[] { + return getLanguageEntry(snapshot, language)?.analyzers.map((a) => a.analyzerName) ?? []; +} + +/** + * Returns all distinct capabilities advertised across every analyzer that + * covers the given language. + */ +export function getMergedCapabilitiesForLanguage( + snapshot: LanguageCapabilityRegistrySnapshot, + language: string, +): AnalyzerCapabilities { + const entry = getLanguageEntry(snapshot, language); + if (!entry) return {}; + + const merged: AnalyzerCapabilities = {}; + for (const a of entry.analyzers) { + for (const [key, value] of Object.entries(a.capabilities)) { + // Capability is considered active for the language if any analyzer claims it. + if (value) merged[key] = value; + } + } + return merged; +} + +/** + * Returns all rules across analyzers that cover a specific language, deduped + * by rule id (first declaration wins, matching registration order). + */ +export function getAllRulesForLanguage( + snapshot: LanguageCapabilityRegistrySnapshot, + language: string, +): Array<{ id: string; name: string; severity: string; enabled: boolean }> { + const entry = getLanguageEntry(snapshot, language); + if (!entry) return []; + + const seen = new Set(); + const result: Array<{ id: string; name: string; severity: string; enabled: boolean }> = []; + + for (const analyzer of entry.analyzers) { + for (const rule of analyzer.rules) { + if (!seen.has(rule.id)) { + seen.add(rule.id); + result.push(rule); + } + } + } + + return result; +} + +/** + * True when the snapshot has no coverage gaps or introspection errors. + */ +export function isRegistryHealthy( + snapshot: LanguageCapabilityRegistrySnapshot, +): boolean { + return snapshot.issues.length === 0 || snapshot.issues.every( + (i) => i.code === "NO_ANALYZERS_REGISTERED" && snapshot.entries.length > 0, + ); +} From 808178c74e8c3f9c191e3858920a4e91fd51b5ba Mon Sep 17 00:00:00 2001 From: Ashborne267 Date: Tue, 20 Oct 2026 22:57:41 +0100 Subject: [PATCH 3/4] feat: harden archive extraction handling --- .../src/scanner/archive-extractor.spec.ts | 259 ++++++++++++++++ .../src/scanner/archive-extractor.ts | 293 ++++++++++++++++++ .../scanner/interfaces/scanner.interface.ts | 2 +- 3 files changed, 553 insertions(+), 1 deletion(-) create mode 100644 apps/api-service/src/scanner/archive-extractor.spec.ts create mode 100644 apps/api-service/src/scanner/archive-extractor.ts diff --git a/apps/api-service/src/scanner/archive-extractor.spec.ts b/apps/api-service/src/scanner/archive-extractor.spec.ts new file mode 100644 index 00000000..67723c4b --- /dev/null +++ b/apps/api-service/src/scanner/archive-extractor.spec.ts @@ -0,0 +1,259 @@ +import { + sanitiseEntryPath, + parseZipEntries, + validateArchiveBuffer, + DEFAULT_ARCHIVE_LIMITS, +} from "./archive-extractor"; +import type { ArchiveSizeLimits } from "./interfaces/scanner.interface"; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +/** Builds a minimal valid ZIP local file header for a single entry. */ +function buildZipBuffer(entries: Array<{ path: string; compressedSize: number; uncompressedSize: number }>): Buffer { + const bufs: Buffer[] = []; + for (const entry of entries) { + const nameBuf = Buffer.from(entry.path, "utf8"); + const header = Buffer.alloc(30); + header.writeUInt32LE(0x04034b50, 0); // local file header signature + header.writeUInt16LE(20, 4); // version needed + header.writeUInt16LE(0, 6); // general purpose bit flag + header.writeUInt16LE(0, 8); // compression method + header.writeUInt32LE(entry.compressedSize, 18); + header.writeUInt32LE(entry.uncompressedSize, 22); + header.writeUInt16LE(nameBuf.length, 26); + header.writeUInt16LE(0, 28); // extra field length + + const data = Buffer.alloc(entry.compressedSize); + bufs.push(header, nameBuf, data); + } + return Buffer.concat(bufs); +} + +const LENIENT_LIMITS: ArchiveSizeLimits = { + maxArchiveBytes: 100 * 1024 * 1024, + maxTotalUncompressedBytes: 500 * 1024 * 1024, + maxEntryBytes: 50 * 1024 * 1024, + maxEntryCount: 100_000, + maxCompressionRatio: 1000, + maxPathDepth: 50, +}; + +const ONE_KB = 1024; +const fixedClock = (() => { + let t = 1000; + return () => (t += 5); +})(); + +// ─── sanitiseEntryPath ──────────────────────────────────────────────────────── + +describe("sanitiseEntryPath", () => { + it("returns simple relative paths unchanged (forward slash)", () => { + expect(sanitiseEntryPath("src/main.ts", 10)).toBe("src/main.ts"); + }); + + it("normalises Windows back-slashes", () => { + expect(sanitiseEntryPath("src\\main.ts", 10)).toBe("src/main.ts"); + }); + + it("collapses redundant dots and empty segments", () => { + expect(sanitiseEntryPath("./a//b/./c.ts", 10)).toBe("a/b/c.ts"); + }); + + it("returns null for absolute Unix paths", () => { + expect(sanitiseEntryPath("/etc/passwd", 10)).toBeNull(); + }); + + it("returns null for absolute Windows paths", () => { + expect(sanitiseEntryPath("C:/Windows/System32/evil.dll", 10)).toBeNull(); + }); + + it("returns null for path-traversal attempts (Zip Slip)", () => { + expect(sanitiseEntryPath("../../etc/passwd", 10)).toBeNull(); + expect(sanitiseEntryPath("safe/../../../outside.txt", 10)).toBeNull(); + }); + + it("returns null for paths exceeding maxDepth", () => { + const deep = Array.from({ length: 25 }, (_, i) => `d${i}`).join("/") + "/file.ts"; + expect(sanitiseEntryPath(deep, 20)).toBeNull(); + }); + + it("returns null for empty or whitespace-only paths", () => { + expect(sanitiseEntryPath("", 10)).toBeNull(); + expect(sanitiseEntryPath(" ", 10)).toBeNull(); + }); + + it("accepts paths at exactly maxDepth", () => { + const exact = Array.from({ length: 5 }, (_, i) => `d${i}`).join("/") + "/file.ts"; + expect(sanitiseEntryPath(exact, 6)).toBe(exact); + }); +}); + +// ─── parseZipEntries ────────────────────────────────────────────────────────── + +describe("parseZipEntries", () => { + it("parses a buffer with a single valid entry", () => { + const buf = buildZipBuffer([{ path: "src/main.ts", compressedSize: 100, uncompressedSize: 200 }]); + const entries = parseZipEntries(buf); + + expect(entries).not.toBeNull(); + expect(entries!).toHaveLength(1); + expect(entries![0]!.path).toBe("src/main.ts"); + expect(entries![0]!.uncompressedSize).toBe(200); + }); + + it("parses a buffer with multiple entries", () => { + const buf = buildZipBuffer([ + { path: "a.ts", compressedSize: 10, uncompressedSize: 20 }, + { path: "b.ts", compressedSize: 30, uncompressedSize: 60 }, + ]); + const entries = parseZipEntries(buf); + + expect(entries).toHaveLength(2); + }); + + it("returns null for an empty buffer", () => { + expect(parseZipEntries(Buffer.alloc(0))).toBeNull(); + }); + + it("returns null for a buffer that is not a ZIP", () => { + const notZip = Buffer.from("this is not a zip file at all"); + expect(parseZipEntries(notZip)).toBeNull(); + }); +}); + +// ─── validateArchiveBuffer — normal scenarios ───────────────────────────────── + +describe("validateArchiveBuffer — normal scenarios", () => { + it("returns valid=true for a clean single-entry ZIP within all limits", () => { + const buf = buildZipBuffer([{ path: "src/index.ts", compressedSize: 100, uncompressedSize: 200 }]); + const result = validateArchiveBuffer(buf, LENIENT_LIMITS, fixedClock); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + expect(result.entryCount).toBe(1); + expect(result.archiveBytes).toBe(buf.length); + }); + + it("returns ARCHIVE_INVALID_FORMAT for a non-ZIP buffer", () => { + const buf = Buffer.from("not a zip"); + const result = validateArchiveBuffer(buf, LENIENT_LIMITS, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors[0]?.code).toBe("ARCHIVE_INVALID_FORMAT"); + }); +}); + +// ─── validateArchiveBuffer — boundary / failure scenarios ───────────────────── + +describe("validateArchiveBuffer — boundary and failure scenarios", () => { + it("rejects an archive that exceeds maxArchiveBytes", () => { + const buf = buildZipBuffer([{ path: "big.bin", compressedSize: 1000, uncompressedSize: 1000 }]); + const tightLimits: ArchiveSizeLimits = { ...DEFAULT_ARCHIVE_LIMITS, maxArchiveBytes: 1 }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors[0]?.code).toBe("ARCHIVE_TOO_LARGE"); + }); + + it("rejects an archive with too many entries", () => { + const entries = Array.from({ length: 5 }, (_, i) => ({ + path: `file${i}.ts`, + compressedSize: 10, + uncompressedSize: 20, + })); + const buf = buildZipBuffer(entries); + const tightLimits: ArchiveSizeLimits = { ...LENIENT_LIMITS, maxEntryCount: 3 }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.code === "ARCHIVE_TOO_MANY_ENTRIES")).toBe(true); + }); + + it("rejects an entry that is individually too large", () => { + const buf = buildZipBuffer([{ path: "huge.ts", compressedSize: ONE_KB, uncompressedSize: 100 * ONE_KB }]); + const tightLimits: ArchiveSizeLimits = { ...LENIENT_LIMITS, maxEntryBytes: 50 * ONE_KB }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.code === "ARCHIVE_ENTRY_TOO_LARGE" && e.entryPath === "huge.ts")).toBe(true); + }); + + it("rejects when total uncompressed bytes exceeds the limit", () => { + const entries = Array.from({ length: 3 }, (_, i) => ({ + path: `file${i}.ts`, + compressedSize: ONE_KB, + uncompressedSize: 100 * ONE_KB, + })); + const buf = buildZipBuffer(entries); + const tightLimits: ArchiveSizeLimits = { ...LENIENT_LIMITS, maxTotalUncompressedBytes: 200 * ONE_KB }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.code === "ARCHIVE_UNCOPRESSED_TOO_LARGE")).toBe(true); + }); + + it("rejects a zip bomb via compression ratio", () => { + // Simulates 1 byte compressed → 1000 bytes uncompressed (ratio = 1000) + const buf = buildZipBuffer([{ path: "bomb.ts", compressedSize: 1, uncompressedSize: 10_000 }]); + const tightLimits: ArchiveSizeLimits = { ...LENIENT_LIMITS, maxCompressionRatio: 50 }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.code === "ARCHIVE_COMPRESSION_RATIO_EXCEEDED")).toBe(true); + }); + + it("rejects entries with path-traversal attempts", () => { + const buf = buildZipBuffer([{ path: "../../etc/passwd", compressedSize: 10, uncompressedSize: 10 }]); + const result = validateArchiveBuffer(buf, LENIENT_LIMITS, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.code === "ARCHIVE_PATH_TOO_DEEP")).toBe(true); + }); + + it("rejects entries deeper than maxPathDepth", () => { + const deep = Array.from({ length: 30 }, (_, i) => `d${i}`).join("/") + "/file.ts"; + const buf = buildZipBuffer([{ path: deep, compressedSize: 10, uncompressedSize: 10 }]); + const tightLimits: ArchiveSizeLimits = { ...LENIENT_LIMITS, maxPathDepth: 5 }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.code === "ARCHIVE_PATH_TOO_DEEP")).toBe(true); + }); + + it("accumulates multiple independent errors in one pass", () => { + const entries = [ + { path: "../../traversal", compressedSize: 10, uncompressedSize: 200 * ONE_KB }, + { path: "ok.ts", compressedSize: 10, uncompressedSize: 200 * ONE_KB }, + ]; + const buf = buildZipBuffer(entries); + const tightLimits: ArchiveSizeLimits = { + ...LENIENT_LIMITS, + maxEntryBytes: 100 * ONE_KB, + maxTotalUncompressedBytes: 300 * ONE_KB, + }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + // At least ARCHIVE_ENTRY_TOO_LARGE, ARCHIVE_PATH_TOO_DEEP. + expect(result.errors.length).toBeGreaterThanOrEqual(2); + }); + + it("reports correct metrics even when validation fails", () => { + const entries = [ + { path: "a.ts", compressedSize: 100, uncompressedSize: 200 }, + { path: "b.ts", compressedSize: 100, uncompressedSize: 200 }, + ]; + const buf = buildZipBuffer(entries); + const tightLimits: ArchiveSizeLimits = { ...LENIENT_LIMITS, maxEntryCount: 1 }; + const result = validateArchiveBuffer(buf, tightLimits, fixedClock); + + expect(result.entryCount).toBe(2); + expect(result.totalUncompressedBytes).toBe(400); + }); + + it("validationDurationMs is non-negative", () => { + const buf = buildZipBuffer([{ path: "f.ts", compressedSize: 10, uncompressedSize: 20 }]); + const result = validateArchiveBuffer(buf, LENIENT_LIMITS, fixedClock); + + expect(result.validationDurationMs).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/apps/api-service/src/scanner/archive-extractor.ts b/apps/api-service/src/scanner/archive-extractor.ts new file mode 100644 index 00000000..ac7a489f --- /dev/null +++ b/apps/api-service/src/scanner/archive-extractor.ts @@ -0,0 +1,293 @@ +/** + * Hardened archive extraction handler. + * + * Defends against known archive-based attack vectors before any byte of an + * uploaded archive reaches the analysis engine: + * + * • Zip-bomb / decompression-bomb — enforced via maxTotalUncompressedBytes + * and maxCompressionRatio. + * • Path-traversal (Zip Slip) — every entry path is sanitised; entries that + * attempt to escape the target directory are rejected outright. + * • Oversized entries — each entry is checked against maxEntryBytes so a + * single large file cannot exhaust memory. + * • Entry-count bombs — archive members are counted and rejected if they + * exceed maxEntryCount. + * • Deeply-nested paths — maxPathDepth prevents symlink-chain and inode- + * exhaustion attacks. + * + * Design + * ────── + * - No I/O: the public `validateArchiveBuffer` function works on an in-memory + * Buffer so it is testable without a filesystem. + * - Safe defaults: `DEFAULT_ARCHIVE_LIMITS` are deliberately conservative. + * - Pure validation result: callers receive a `ArchiveValidationResult` and + * decide whether to proceed; this module never writes extracted bytes to disk. + * - Tenant-isolated: no shared state; every call is independent. + */ + +import type { + ArchiveSizeLimits, + ArchiveValidationError, + ArchiveValidationErrorCode, + ArchiveValidationResult, +} from "./interfaces/scanner.interface"; + +// ─── Defaults ───────────────────────────────────────────────────────────────── + +export const DEFAULT_ARCHIVE_LIMITS: Readonly = { + /** 50 MB compressed upload limit. */ + maxArchiveBytes: 50 * 1024 * 1024, + /** 200 MB uncompressed total (zip-bomb guard). */ + maxTotalUncompressedBytes: 200 * 1024 * 1024, + /** 10 MB per individual entry. */ + maxEntryBytes: 10 * 1024 * 1024, + /** 10,000 entries (entry-count bomb guard). */ + maxEntryCount: 10_000, + /** Compression ratio > 100 is almost certainly a bomb (typical zips are < 10). */ + maxCompressionRatio: 100, + /** Paths deeper than 20 levels are pathological for source repos. */ + maxPathDepth: 20, +}; + +// ─── Path sanitisation ──────────────────────────────────────────────────────── + +/** + * Returns a safe repository-relative path, or `null` when the path attempts + * to escape the target directory (path-traversal / Zip Slip attack). + * + * Rules: + * - Absolute paths are rejected. + * - Any `..` segment that would escape the archive root is rejected. + * - Path depth is capped at `maxDepth`. + */ +export function sanitiseEntryPath( + rawPath: string, + maxDepth: number, +): string | null { + // Normalise separators. + const p = rawPath.replace(/\\/g, "/").trim(); + + if (p === "") return null; + + // Absolute path check (Unix and Windows drive-letter). + if (p.startsWith("/") || /^[A-Za-z]:\//.test(p)) return null; + + const segments: string[] = []; + for (const seg of p.split("/")) { + if (seg === "" || seg === ".") continue; + if (seg === "..") { + if (segments.length === 0) return null; // escape attempt + segments.pop(); + } else { + segments.push(seg); + } + } + + if (segments.length === 0) return null; + if (segments.length > maxDepth) return null; + + return segments.join("/"); +} + +// ─── Minimal ZIP parser ─────────────────────────────────────────────────────── +// +// A full ZIP library is a significant dependency. For validation purposes we +// only need the local file headers (signature 0x04034b50) to extract the entry +// name and uncompressed size — we never decompress any data here. + +interface ZipEntry { + path: string; + compressedSize: number; + uncompressedSize: number; +} + +const ZIP_LOCAL_HEADER_SIG = 0x04034b50; +const ZIP_LOCAL_HEADER_MIN_LEN = 30; + +/** + * Extracts a flat list of entry metadata from a ZIP buffer without + * decompressing any data. + * + * Returns `null` when the buffer does not begin with a valid ZIP local-file + * header signature (i.e. the archive format is not supported / corrupted). + */ +export function parseZipEntries(buf: Buffer): ZipEntry[] | null { + if (buf.length < 4) return null; + + // Every valid ZIP starts with a local file header signature. + if (buf.readUInt32LE(0) !== ZIP_LOCAL_HEADER_SIG) return null; + + const entries: ZipEntry[] = []; + let offset = 0; + + while (offset + ZIP_LOCAL_HEADER_MIN_LEN <= buf.length) { + const sig = buf.readUInt32LE(offset); + if (sig !== ZIP_LOCAL_HEADER_SIG) break; + + const compressedSize = buf.readUInt32LE(offset + 18); + const uncompressedSize = buf.readUInt32LE(offset + 22); + const fileNameLength = buf.readUInt16LE(offset + 26); + const extraFieldLength = buf.readUInt16LE(offset + 28); + + const nameStart = offset + ZIP_LOCAL_HEADER_MIN_LEN; + const nameEnd = nameStart + fileNameLength; + if (nameEnd > buf.length) break; + + const path = buf.slice(nameStart, nameEnd).toString("utf8"); + entries.push({ path, compressedSize, uncompressedSize }); + + offset = nameEnd + extraFieldLength + compressedSize; + } + + return entries; +} + +// ─── Validation ─────────────────────────────────────────────────────────────── + +function err( + code: ArchiveValidationErrorCode, + message: string, + opts: Partial> = {}, +): ArchiveValidationError { + return { code, message, ...opts }; +} + +/** + * Validates an archive buffer against the supplied (or default) limits. + * + * Currently supports ZIP archives. Other formats return an + * `ARCHIVE_INVALID_FORMAT` error in the result — they are never extracted. + * + * Never throws; all failure modes are surfaced via `errors` in the result. + */ +export function validateArchiveBuffer( + buf: Buffer, + limits: ArchiveSizeLimits = DEFAULT_ARCHIVE_LIMITS, + now: () => number = () => Date.now(), +): ArchiveValidationResult { + const started = now(); + const errors: ArchiveValidationError[] = []; + + // 1. Archive size. + const archiveBytes = buf.length; + if (archiveBytes > limits.maxArchiveBytes) { + errors.push( + err( + "ARCHIVE_TOO_LARGE", + `Archive size ${archiveBytes} bytes exceeds the limit of ${limits.maxArchiveBytes} bytes.`, + { observed: archiveBytes, limit: limits.maxArchiveBytes }, + ), + ); + // No point parsing further — early exit. + return { + valid: false, + errors, + archiveBytes, + totalUncompressedBytes: 0, + entryCount: 0, + compressionRatio: 0, + maxPathDepthObserved: 0, + validationDurationMs: now() - started, + }; + } + + // 2. Parse entries. + const parsed = parseZipEntries(buf); + if (parsed === null) { + errors.push( + err("ARCHIVE_INVALID_FORMAT", "Archive is not a recognised ZIP format or is corrupted."), + ); + return { + valid: false, + errors, + archiveBytes, + totalUncompressedBytes: 0, + entryCount: 0, + compressionRatio: 0, + maxPathDepthObserved: 0, + validationDurationMs: now() - started, + }; + } + + // 3. Entry-count check. + if (parsed.length > limits.maxEntryCount) { + errors.push( + err( + "ARCHIVE_TOO_MANY_ENTRIES", + `Archive contains ${parsed.length} entries, exceeding the limit of ${limits.maxEntryCount}.`, + { observed: parsed.length, limit: limits.maxEntryCount }, + ), + ); + } + + // 4. Per-entry checks. + let totalUncompressedBytes = 0; + let maxPathDepthObserved = 0; + + for (const entry of parsed) { + // Path sanitisation / traversal guard. + const safe = sanitiseEntryPath(entry.path, limits.maxPathDepth); + const depth = safe ? safe.split("/").length : 0; + + if (safe === null) { + errors.push( + err( + "ARCHIVE_PATH_TOO_DEEP", + `Entry path "${entry.path}" is invalid or exceeds max depth ${limits.maxPathDepth}.`, + { entryPath: entry.path, limit: limits.maxPathDepth }, + ), + ); + continue; + } + + if (depth > maxPathDepthObserved) maxPathDepthObserved = depth; + + // Per-entry size. + if (entry.uncompressedSize > limits.maxEntryBytes) { + errors.push( + err( + "ARCHIVE_ENTRY_TOO_LARGE", + `Entry "${safe}" uncompressed size ${entry.uncompressedSize} bytes exceeds limit ${limits.maxEntryBytes} bytes.`, + { observed: entry.uncompressedSize, limit: limits.maxEntryBytes, entryPath: safe }, + ), + ); + } + + totalUncompressedBytes += entry.uncompressedSize; + } + + // 5. Total uncompressed size. + if (totalUncompressedBytes > limits.maxTotalUncompressedBytes) { + errors.push( + err( + "ARCHIVE_UNCOPRESSED_TOO_LARGE", + `Total uncompressed size ${totalUncompressedBytes} bytes exceeds limit ${limits.maxTotalUncompressedBytes} bytes.`, + { observed: totalUncompressedBytes, limit: limits.maxTotalUncompressedBytes }, + ), + ); + } + + // 6. Compression ratio (bomb guard). + const compressionRatio = + archiveBytes > 0 ? totalUncompressedBytes / archiveBytes : 0; + if (compressionRatio > limits.maxCompressionRatio) { + errors.push( + err( + "ARCHIVE_COMPRESSION_RATIO_EXCEEDED", + `Compression ratio ${compressionRatio.toFixed(1)} exceeds the limit of ${limits.maxCompressionRatio}.`, + { observed: Math.round(compressionRatio), limit: limits.maxCompressionRatio }, + ), + ); + } + + return { + valid: errors.length === 0, + errors, + archiveBytes, + totalUncompressedBytes, + entryCount: parsed.length, + compressionRatio, + maxPathDepthObserved, + validationDurationMs: now() - started, + }; +} diff --git a/apps/api-service/src/scanner/interfaces/scanner.interface.ts b/apps/api-service/src/scanner/interfaces/scanner.interface.ts index 4169f9fa..f5570212 100644 --- a/apps/api-service/src/scanner/interfaces/scanner.interface.ts +++ b/apps/api-service/src/scanner/interfaces/scanner.interface.ts @@ -48,7 +48,7 @@ export type ArchiveValidationErrorCode = | "ARCHIVE_UNCOPRESSED_TOO_LARGE" | "ARCHIVE_ENTRY_TOO_LARGE" | "ARCHIVE_TOO_MANY_ENTRIES" - | ARCHIVE_COMPRESSION_RATIO_EXCEEDED" + | "ARCHIVE_COMPRESSION_RATIO_EXCEEDED" | "ARCHIVE_PATH_TOO_DEEP" | "ARCHIVE_INVALID_FORMAT"; From 4c423209704adde4ec5d281f244b194debcfda77 Mon Sep 17 00:00:00 2001 From: Ashborne267 Date: Tue, 20 Oct 2026 22:59:05 +0100 Subject: [PATCH 4/4] feat: implement queue backpressure controls --- .../transection/queue-backpressure.spec.ts | 314 ++++++++++++++++++ .../src/transection/queue-backpressure.ts | 304 +++++++++++++++++ 2 files changed, 618 insertions(+) create mode 100644 apps/api-service/src/transection/queue-backpressure.spec.ts create mode 100644 apps/api-service/src/transection/queue-backpressure.ts diff --git a/apps/api-service/src/transection/queue-backpressure.spec.ts b/apps/api-service/src/transection/queue-backpressure.spec.ts new file mode 100644 index 00000000..05ad510f --- /dev/null +++ b/apps/api-service/src/transection/queue-backpressure.spec.ts @@ -0,0 +1,314 @@ +import { + QueueBackpressureController, + DEFAULT_BACKPRESSURE_OPTIONS, + type BackpressureOptions, +} from "./queue-backpressure"; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function makeController(overrides: Partial = {}): QueueBackpressureController { + return new QueueBackpressureController({ + maxQueueDepth: 10, + highWatermark: 5, + criticalWatermark: 8, + drainEnabled: true, + drainToDepth: 2, + ...overrides, + }); +} + +// ─── Constructor validation ─────────────────────────────────────────────────── + +describe("QueueBackpressureController — constructor", () => { + it("constructs with valid options", () => { + expect(() => makeController()).not.toThrow(); + }); + + it("throws when highWatermark >= criticalWatermark", () => { + expect(() => + makeController({ highWatermark: 8, criticalWatermark: 8 }), + ).toThrow(/highWatermark.*must be less than criticalWatermark/); + }); + + it("throws when criticalWatermark > maxQueueDepth", () => { + expect(() => + makeController({ criticalWatermark: 11, maxQueueDepth: 10 }), + ).toThrow(/criticalWatermark.*must not exceed maxQueueDepth/); + }); + + it("throws when drainToDepth >= highWatermark", () => { + expect(() => + makeController({ drainToDepth: 5, highWatermark: 5 }), + ).toThrow(/drainToDepth.*must be less than highWatermark/); + }); + + it("uses DEFAULT_BACKPRESSURE_OPTIONS when called with empty options", () => { + const bp = new QueueBackpressureController(); + const m = bp.getMetrics(); + expect(m.depth).toBe(0); + expect(m.level).toBe("normal"); + }); +}); + +// ─── Normal level ───────────────────────────────────────────────────────────── + +describe("normal level — all priorities accepted", () => { + it("accepts all priorities when depth is below highWatermark", () => { + const bp = makeController(); + for (const priority of ["low", "medium", "high", "critical"] as const) { + const decision = bp.tryEnqueue(priority); + expect(decision.accepted).toBe(true); + expect(decision.level).toBe("normal"); + } + }); + + it("increments depth with each enqueue", () => { + const bp = makeController(); + bp.tryEnqueue("low"); + bp.tryEnqueue("medium"); + expect(bp.currentDepth).toBe(2); + }); +}); + +// ─── Pressure level ─────────────────────────────────────────────────────────── + +describe("pressure level — low and medium are shed", () => { + function atHighWatermark(): QueueBackpressureController { + const bp = makeController(); + for (let i = 0; i < 5; i++) bp.tryEnqueue("high"); // fill to highWatermark + return bp; + } + + it("level transitions to pressure at highWatermark", () => { + const bp = atHighWatermark(); + expect(bp.level).toBe("pressure"); + }); + + it("rejects low priority at pressure level", () => { + const bp = atHighWatermark(); + const d = bp.tryEnqueue("low"); + expect(d.accepted).toBe(false); + expect(d.reason).toContain("pressure"); + }); + + it("rejects medium priority at pressure level", () => { + const bp = atHighWatermark(); + const d = bp.tryEnqueue("medium"); + expect(d.accepted).toBe(false); + }); + + it("accepts high priority at pressure level", () => { + const bp = atHighWatermark(); + expect(bp.tryEnqueue("high").accepted).toBe(true); + }); + + it("accepts critical priority at pressure level", () => { + const bp = atHighWatermark(); + expect(bp.tryEnqueue("critical").accepted).toBe(true); + }); +}); + +// ─── Critical level ─────────────────────────────────────────────────────────── + +describe("critical level — only critical accepted", () => { + function atCriticalWatermark(): QueueBackpressureController { + const bp = makeController(); + for (let i = 0; i < 8; i++) bp.tryEnqueue("critical"); // fill to criticalWatermark + return bp; + } + + it("level transitions to critical at criticalWatermark", () => { + const bp = atCriticalWatermark(); + expect(bp.level).toBe("critical"); + }); + + it("rejects high priority at critical level", () => { + const bp = atCriticalWatermark(); + expect(bp.tryEnqueue("high").accepted).toBe(false); + }); + + it("accepts critical priority at critical level", () => { + const bp = atCriticalWatermark(); + expect(bp.tryEnqueue("critical").accepted).toBe(true); + }); +}); + +// ─── Overloaded level ───────────────────────────────────────────────────────── + +describe("overloaded level — nothing accepted", () => { + function atMax(): QueueBackpressureController { + const bp = makeController(); + for (let i = 0; i < 10; i++) bp.tryEnqueue("critical"); + return bp; + } + + it("level transitions to overloaded at maxQueueDepth", () => { + const bp = atMax(); + expect(bp.level).toBe("overloaded"); + }); + + it("rejects critical priority when overloaded", () => { + const bp = atMax(); + const d = bp.tryEnqueue("critical"); + expect(d.accepted).toBe(false); + expect(d.reason).toContain("maximum capacity"); + }); +}); + +// ─── dequeue ────────────────────────────────────────────────────────────────── + +describe("dequeue", () => { + it("decrements depth", () => { + const bp = makeController(); + bp.tryEnqueue("high"); + bp.tryEnqueue("high"); + bp.dequeue(); + expect(bp.currentDepth).toBe(1); + }); + + it("does not go below zero", () => { + const bp = makeController(); + bp.dequeue(100); + expect(bp.currentDepth).toBe(0); + }); + + it("accepts a batch count", () => { + const bp = makeController(); + bp.tryEnqueue("high"); + bp.tryEnqueue("high"); + bp.tryEnqueue("high"); + bp.dequeue(3); + expect(bp.currentDepth).toBe(0); + }); + + it("restores normal level after draining below highWatermark", () => { + const bp = makeController(); + for (let i = 0; i < 7; i++) bp.tryEnqueue("high"); // reach pressure + expect(bp.level).toBe("pressure"); + bp.dequeue(4); // drop to 3 + expect(bp.level).toBe("normal"); + }); +}); + +// ─── drain mode ─────────────────────────────────────────────────────────────── + +describe("drain mode", () => { + it("activates draining and enforces critical-only acceptance", () => { + const bp = makeController(); + bp.tryEnqueue("high"); // depth = 1 (normal) + bp.drain(); + + // During drain, level is treated as critical so medium is rejected. + const d = bp.tryEnqueue("medium"); + expect(d.accepted).toBe(false); + expect(bp.getMetrics().draining).toBe(true); + }); + + it("exits drain mode once depth falls to drainToDepth", () => { + const bp = makeController(); + for (let i = 0; i < 6; i++) bp.tryEnqueue("high"); // depth = 6 + bp.drain(); + + bp.dequeue(4); // depth = 2, drainToDepth = 2 → should exit draining + expect(bp.getMetrics().draining).toBe(false); + expect(bp.tryEnqueue("medium").accepted).toBe(true); + }); + + it("drain() is a no-op when drainEnabled is false", () => { + const bp = makeController({ drainEnabled: false }); + bp.tryEnqueue("high"); + bp.drain(); + + expect(bp.getMetrics().draining).toBe(false); + expect(bp.tryEnqueue("low").accepted).toBe(true); + }); +}); + +// ─── Metrics ────────────────────────────────────────────────────────────────── + +describe("metrics", () => { + it("tracks accepted, rejected, and dequeued counts", () => { + const bp = makeController(); + bp.tryEnqueue("low"); // accepted + bp.tryEnqueue("low"); // accepted + for (let i = 0; i < 5; i++) bp.tryEnqueue("high"); // fills to highWatermark=5, all accepted + bp.tryEnqueue("low"); // pressure — rejected + + const m = bp.getMetrics(); + expect(m.accepted).toBe(7); + expect(m.rejected).toBe(1); + + bp.dequeue(3); + expect(bp.getMetrics().dequeued).toBe(3); + }); + + it("tracks highWatermarkBreaches", () => { + const bp = makeController(); + for (let i = 0; i < 6; i++) bp.tryEnqueue("high"); // crosses highWatermark on 6th + expect(bp.getMetrics().highWatermarkBreaches).toBe(1); + }); + + it("tracks criticalWatermarkBreaches", () => { + const bp = makeController(); + for (let i = 0; i < 9; i++) bp.tryEnqueue("critical"); // crosses critical on 9th + expect(bp.getMetrics().criticalWatermarkBreaches).toBe(1); + }); +}); + +// ─── reset ──────────────────────────────────────────────────────────────────── + +describe("reset", () => { + it("zeroes all counters and depth", () => { + const bp = makeController(); + bp.tryEnqueue("high"); + bp.tryEnqueue("low"); + bp.reset(); + + const m = bp.getMetrics(); + expect(m.depth).toBe(0); + expect(m.accepted).toBe(0); + expect(m.rejected).toBe(0); + expect(m.level).toBe("normal"); + expect(m.draining).toBe(false); + }); +}); + +// ─── Integration / realistic scenario ──────────────────────────────────────── + +describe("realistic scenario — burst of work items across all priorities", () => { + it("maintains correct depth and level under a mixed-priority burst", () => { + const bp = makeController({ + maxQueueDepth: 20, + highWatermark: 10, + criticalWatermark: 15, + drainEnabled: true, + drainToDepth: 4, + }); + + // Simulate 12 low-priority enqueues (first 10 accepted, then shed). + let accepted = 0; + let rejected = 0; + for (let i = 0; i < 12; i++) { + const d = bp.tryEnqueue("low"); + if (d.accepted) accepted++; else rejected++; + } + expect(accepted).toBe(10); // 0-9 accepted (fill to depth=10 = highWatermark) + expect(rejected).toBe(2); // 10-11 rejected + + // High-priority items can still get through at pressure. + expect(bp.tryEnqueue("high").accepted).toBe(true); + expect(bp.tryEnqueue("critical").accepted).toBe(true); + + // Activate drain: only critical goes through. + bp.drain(); + expect(bp.tryEnqueue("high").accepted).toBe(false); + expect(bp.tryEnqueue("critical").accepted).toBe(true); + + // Drain down to target. + bp.dequeue(10); + expect(bp.getMetrics().draining).toBe(false); + + // Normal acceptance resumes. + expect(bp.tryEnqueue("low").accepted).toBe(true); + }); +}); diff --git a/apps/api-service/src/transection/queue-backpressure.ts b/apps/api-service/src/transection/queue-backpressure.ts new file mode 100644 index 00000000..1eec6998 --- /dev/null +++ b/apps/api-service/src/transection/queue-backpressure.ts @@ -0,0 +1,304 @@ +/** + * Queue backpressure controls. + * + * Prevents unbounded memory growth and runaway acceptance of work items when + * consumers are slower than producers. The controller tracks queue depth, + * enforces configurable high/low watermarks, and exposes a clear + * accept/reject/drain interface so callers never need to inspect raw queue + * length to decide whether to enqueue. + * + * Design + * ────── + * - No I/O, no timers, no global state — pure data structure. + * - Tenant-isolated: one `QueueBackpressureController` instance per logical + * queue; state never leaks across requests. + * - Secure defaults: `DEFAULT_BACKPRESSURE_OPTIONS` are conservative. + * - Deterministic: given the same sequence of operations the pressure level + * and accept/reject decisions are always the same. + * - Priority-aware: the same four-level priority scheme used by + * `WithdrawalQueueService` (critical > high > medium > low) is honoured; + * items at higher priorities are still accepted when lower priorities are + * shedding load. + */ + +// ─── Types ──────────────────────────────────────────────────────────────────── + +export type BackpressurePriority = "critical" | "high" | "medium" | "low"; + +export type BackpressureLevel = + /** Below highWatermark — all priorities accepted. */ + | "normal" + /** At or above highWatermark — low/medium priorities shed. */ + | "pressure" + /** At or above criticalWatermark — only critical items accepted. */ + | "critical" + /** At maxQueueDepth — all items rejected. */ + | "overloaded"; + +export interface BackpressureOptions { + /** + * Maximum number of items allowed in the queue at any time. + * Enqueueing above this always fails regardless of priority. + */ + maxQueueDepth: number; + /** + * Depth at which `low` and `medium` items start being rejected. + * Must be < criticalWatermark < maxQueueDepth. + */ + highWatermark: number; + /** + * Depth at which only `critical` items are accepted. + * Must be > highWatermark and <= maxQueueDepth. + */ + criticalWatermark: number; + /** + * When true, the drain() method will not resume acceptance until the queue + * depth drops to or below `drainToDepth`. + */ + drainEnabled: boolean; + /** + * The target depth that draining must reach before normal acceptance resumes. + * Only consulted when `drainEnabled` is true. Defaults to half highWatermark. + */ + drainToDepth?: number; +} + +export const DEFAULT_BACKPRESSURE_OPTIONS: Readonly = { + maxQueueDepth: 1000, + highWatermark: 700, + criticalWatermark: 900, + drainEnabled: true, + drainToDepth: 350, +}; + +export interface BackpressureDecision { + accepted: boolean; + level: BackpressureLevel; + /** Human-readable reason when `accepted` is false. */ + reason?: string; + /** Current depth at the time of the decision. */ + depth: number; +} + +export interface BackpressureMetrics { + depth: number; + level: BackpressureLevel; + accepted: number; + rejected: number; + /** Total number of dequeue operations that brought depth down. */ + dequeued: number; + /** True when a drain cycle is currently in progress. */ + draining: boolean; + highWatermarkBreaches: number; + criticalWatermarkBreaches: number; + overloadEvents: number; +} + +// ─── Controller ─────────────────────────────────────────────────────────────── + +/** + * Stateful backpressure controller. Wire one instance to each logical queue. + * + * Usage + * ----- + * const bp = new QueueBackpressureController(options); + * + * // Producer side: + * const decision = bp.tryEnqueue("medium"); + * if (!decision.accepted) { ...shed load... } + * + * // Consumer side (called once per item processed): + * bp.dequeue(); + */ +export class QueueBackpressureController { + private depth = 0; + private draining = false; + + private accepted = 0; + private rejected = 0; + private dequeued = 0; + private highWatermarkBreaches = 0; + private criticalWatermarkBreaches = 0; + private overloadEvents = 0; + + private readonly opts: Readonly; + + constructor(options: Partial = {}) { + const merged: BackpressureOptions = { + ...DEFAULT_BACKPRESSURE_OPTIONS, + ...options, + }; + + if (merged.highWatermark >= merged.criticalWatermark) { + throw new Error( + `highWatermark (${merged.highWatermark}) must be less than criticalWatermark (${merged.criticalWatermark})`, + ); + } + if (merged.criticalWatermark > merged.maxQueueDepth) { + throw new Error( + `criticalWatermark (${merged.criticalWatermark}) must not exceed maxQueueDepth (${merged.maxQueueDepth})`, + ); + } + if (merged.drainToDepth !== undefined && merged.drainToDepth >= merged.highWatermark) { + throw new Error( + `drainToDepth (${merged.drainToDepth}) must be less than highWatermark (${merged.highWatermark})`, + ); + } + + this.opts = Object.freeze(merged); + } + + // ── Public interface ────────────────────────────────────────────────────── + + /** + * Evaluates whether the queue can accept an item at the given priority and, + * if so, increments the internal depth counter. + * + * Call `dequeue()` for every item removed from the queue so depth stays + * accurate. + */ + tryEnqueue(priority: BackpressurePriority = "medium"): BackpressureDecision { + const level = this.computeLevel(); + + const accepted = this.isAccepted(priority, level); + + if (accepted) { + this.depth++; + this.accepted++; + // Track watermark breaches on the way up. + if (this.depth === this.opts.highWatermark + 1) this.highWatermarkBreaches++; + if (this.depth === this.opts.criticalWatermark + 1) this.criticalWatermarkBreaches++; + if (this.depth > this.opts.maxQueueDepth) this.overloadEvents++; + } else { + this.rejected++; + } + + return { + accepted, + level, + reason: accepted ? undefined : this.rejectionReason(priority, level), + depth: this.depth, + }; + } + + /** + * Signals that one item has been consumed from the queue. + * + * Decrements the depth counter (floor: 0) and, when drain mode is enabled, + * clears the draining flag once the target depth is reached. + */ + dequeue(count = 1): void { + const n = Math.max(0, count); + this.depth = Math.max(0, this.depth - n); + this.dequeued += n; + + if (this.draining && this.opts.drainEnabled) { + const target = this.opts.drainToDepth ?? Math.floor(this.opts.highWatermark / 2); + if (this.depth <= target) { + this.draining = false; + } + } + } + + /** + * Activates drain mode. + * + * When drain mode is active and `drainEnabled` is true, only `critical` + * items are accepted until the depth falls to `drainToDepth`. + */ + drain(): void { + if (this.opts.drainEnabled) { + this.draining = true; + } + } + + /** + * Resets all counters and depth to zero. Useful for test isolation and + * scheduled maintenance windows. + */ + reset(): void { + this.depth = 0; + this.draining = false; + this.accepted = 0; + this.rejected = 0; + this.dequeued = 0; + this.highWatermarkBreaches = 0; + this.criticalWatermarkBreaches = 0; + this.overloadEvents = 0; + } + + /** Returns a snapshot of current metrics. */ + getMetrics(): BackpressureMetrics { + return { + depth: this.depth, + level: this.computeLevel(), + accepted: this.accepted, + rejected: this.rejected, + dequeued: this.dequeued, + draining: this.draining, + highWatermarkBreaches: this.highWatermarkBreaches, + criticalWatermarkBreaches: this.criticalWatermarkBreaches, + overloadEvents: this.overloadEvents, + }; + } + + /** Current queue depth. */ + get currentDepth(): number { + return this.depth; + } + + /** Current backpressure level. */ + get level(): BackpressureLevel { + return this.computeLevel(); + } + + // ── Private helpers ─────────────────────────────────────────────────────── + + private computeLevel(): BackpressureLevel { + if (this.draining && this.opts.drainEnabled) { + // While draining treat as critical so only critical items sneak through. + return "critical"; + } + if (this.depth >= this.opts.maxQueueDepth) return "overloaded"; + if (this.depth >= this.opts.criticalWatermark) return "critical"; + if (this.depth >= this.opts.highWatermark) return "pressure"; + return "normal"; + } + + private isAccepted(priority: BackpressurePriority, level: BackpressureLevel): boolean { + switch (level) { + case "normal": + return true; + case "pressure": + // Accept critical and high; shed medium and low. + return priority === "critical" || priority === "high"; + case "critical": + // Accept only critical items. + return priority === "critical"; + case "overloaded": + return false; + } + } + + private rejectionReason( + priority: BackpressurePriority, + level: BackpressureLevel, + ): string { + switch (level) { + case "overloaded": + return `Queue is at maximum capacity (${this.opts.maxQueueDepth}). All new items are rejected.`; + case "critical": + return ( + `Queue has reached the critical watermark (${this.opts.criticalWatermark}). ` + + `Only 'critical' priority items are accepted; '${priority}' was rejected.` + ); + case "pressure": + return ( + `Queue is under pressure (depth ${this.depth} ≥ highWatermark ${this.opts.highWatermark}). ` + + `'${priority}' priority items are shed; only 'critical' and 'high' are accepted.` + ); + default: + return `Item rejected at level '${level}'.`; + } + } +}