diff --git a/scripts/publication-safety.test.ts b/scripts/publication-safety.test.ts new file mode 100644 index 0000000..32c337d --- /dev/null +++ b/scripts/publication-safety.test.ts @@ -0,0 +1,175 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { checkPublication, trackedPublicationIssues } from "./check-publication.ts"; +import { assertPublicMessage, publicationIssues } from "./publication-safety.ts"; + +test("synthetic private content is refused without echoing its value", () => { + for (const value of [ + "/" + "Users/fictional/project", + "github_" + "pat_" + "SYNTHETIC".repeat(8), + "FixtureReviewer" + " 9.9 Max review", + "# Session" + " report", + "Includes 12" + " merged Engine pull requests", + ]) { + expect(publicationIssues(value).length).toBeGreaterThan(0); + try { + assertPublicMessage({ output: { summary: value } }); + throw new Error("Expected rejection"); + } catch (error) { + expect(String(error)).toContain("Public message refused:"); + expect(String(error)).not.toContain(value); + } + } +}); + +test("public AI, agent and spec terminology and technical provenance remain valid", () => { + expect(() => + assertPublicMessage([ + "AI integration uses a documented API.", + "The agent runs validation; this spec defines the public schema.", + { source_repository: "rootform-dev/engine", source_commit: "a".repeat(40) }, + "The binary handoff verifies exact provenance.", + ]), + ).not.toThrow(); +}); + +test("serialized and URL-encoded metadata is inspected before sending", () => { + expect(() => assertPublicMessage(undefined)).not.toThrow(); + const path = "/" + "Users/fictional/session"; + const token = "github_" + "pat_" + "SYNTHETIC".repeat(8); + for (const value of [path, token]) { + for (const serialized of [ + encodeURIComponent(value), + encodeURIComponent(encodeURIComponent(value)), + JSON.stringify({ value }).replaceAll("/", "\\/"), + JSON.stringify({ value }).replaceAll("/", "\\u002f"), + { encoding: "base64", content: btoa(value) }, + { encoding: "base64", content: btoa(JSON.stringify({ value }).replaceAll("/", "\\u002f")) }, + JSON.stringify({ API_TOKEN: value }), + { message: "Update public fixture", content: btoa(value) }, + ]) { + try { + assertPublicMessage(serialized); + throw new Error("Expected rejection"); + } catch (error) { + expect(String(error)).toContain("Public message refused:"); + expect(String(error)).not.toContain(value); + } + } + } + expect(() => + assertPublicMessage({ encoding: "base64", content: btoa("Public fixture") }), + ).not.toThrow(); + expect(() => assertPublicMessage({ encoding: "base64", content: "?" })).toThrow( + "uninspectable-content", + ); +}); + +test("nested JSON and rendered metadata cannot hide private text", () => { + const credential = "SYNTHETIC".repeat(8); + const path = "/" + "Users/fictional/session"; + for (const value of [ + { body: JSON.stringify({ API_KEY: credential }) }, + '"api_key": "' + credential + '"', + { body: JSON.stringify({ CLOUDFLARE_API_KEY: credential }) }, + { encoding: "base64", content: Buffer.from(path, "utf16le").toString("base64") }, + ["/", ";Users/", ";fictional/", ";session"].join(""), + "notes/" + "fictional-session/report.json", + ]) + expect(() => assertPublicMessage(value)).toThrow("Public message refused:"); + for (const value of [ + "ROOTFORM_DATADOG_CLOUDFLARE_KEY_SENTINEL", + "ROOTFORM_DATADOG_FASTLY_KEY_SENTINEL", + "ROOTFORM_HCP_DATADOG_API_SENTINEL", + "ROOTFORM_ATLAS_OBSERVABILITY_SECRET", + ]) + expect(() => assertPublicMessage('api_key = "' + value + '"')).not.toThrow(); +}); + +test("tracked ignored files and staged bytes cannot escape the publication scan", () => { + const directory = mkdtempSync(join(tmpdir(), "publication-fixture-")); + const git = (...args: string[]) => execFileSync("git", args, { cwd: directory }); + try { + git("init", "-q"); + writeFileSync(join(directory, ".gitignore"), "artifacts/\n"); + mkdirSync(join(directory, "artifacts")); + writeFileSync(join(directory, "artifacts/tracked.txt"), "/" + "Users/fictional/hidden"); + git("add", ".gitignore"); + git("add", "-f", "artifacts/tracked.txt"); + expect(trackedPublicationIssues(directory)[0]?.rule).toBe("personal-path"); + writeFileSync(join(directory, "artifacts/tracked.txt"), "Clean working copy.\n"); + expect(() => checkPublication(["--repo", directory, "--staged"])).toThrow(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("large staged files are inspected and Git failures never echo their input", () => { + const directory = mkdtempSync(join(tmpdir(), "publication-large-fixture-")); + const git = (...args: string[]) => execFileSync("git", args, { cwd: directory }); + try { + git("init", "-q"); + writeFileSync( + join(directory, "bundle.js"), + `${"x".repeat(2 * 1024 * 1024)}\n/${"Users/fictional/hidden"}`, + ); + git("add", "bundle.js"); + expect(trackedPublicationIssues(directory, ":")[0]?.rule).toBe("personal-path"); + const revision = "/" + "Users/fictional/invalid"; + expect(() => trackedPublicationIssues(directory, revision)).toThrow( + "cannot read tracked Git content", + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("staged symlink modes cannot be concealed by the working copy", () => { + const directory = mkdtempSync(join(tmpdir(), "publication-link-fixture-")); + const git = (...args: string[]) => execFileSync("git", args, { cwd: directory }); + try { + git("init", "-q"); + symlinkSync("nonexistent-fixture", join(directory, "alias")); + git("add", "alias"); + rmSync(join(directory, "alias")); + writeFileSync(join(directory, "alias"), "Regular working copy.\n"); + expect(trackedPublicationIssues(directory)).toEqual([]); + expect(trackedPublicationIssues(directory, ":")[0]?.rule).toBe("symlink"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("named credentials are checked recursively while exact synthetic literals remain valid", () => { + const credential = "SYNTHETIC".repeat(8); + for (const payload of [ + { API_TOKEN: credential }, + { encoding: "base64", content: btoa(JSON.stringify({ api_key: credential })) }, + { + encoding: "base64", + content: btoa( + JSON.stringify({ encoding: "base64", content: btoa("github_" + "pat_" + credential) }), + ), + }, + ]) + expect(() => assertPublicMessage(payload)).toThrow("credential"); + for (const value of [ + "ROOTFORM_DATADOG_CLOUDFLARE_KEY_SENTINEL", + "ROOTFORM_DATADOG_FASTLY_KEY_SENTINEL", + "ROOTFORM_HCP_DATADOG_API_SENTINEL", + "ROOTFORM_ATLAS_OBSERVABILITY_SECRET", + ]) { + expect(() => assertPublicMessage({ api_key: value })).not.toThrow(); + expect(() => assertPublicMessage({ api_key: value + "_CHANGED" })).toThrow("credential"); + } + const path = "/" + "Users/fictional/session"; + expect(() => + assertPublicMessage({ + encoding: "base64", + content: btoa(JSON.stringify({ value: path }).replaceAll("/", "\\u002f")), + }), + ).toThrow("personal-path"); +}); diff --git a/scripts/publication-safety.ts b/scripts/publication-safety.ts index ea63fe5..dacbca3 100644 --- a/scripts/publication-safety.ts +++ b/scripts/publication-safety.ts @@ -1,5 +1,19 @@ export type PublicationIssue = { rule: string; line: number }; +// Exact non-credential literals used by public synthetic provider fixtures. +const syntheticCredentialValues = new Set([ + "fixture000000000000000000000000000000000", + "ROOTFORM_DATADOG_PROVIDER_API_KEY_SENTINEL", + "ROOTFORM_NEWRELIC_BOUNDARY_SECRET_SENTINEL", + "ROOTFORM_NEWRELIC_PROVIDER_KEY_SENTINEL", + "ROOTFORM_OKTA_BOUNDARY_SECRET_SENTINEL", + "ROOTFORM_OKTA_PROVIDER_TOKEN_SENTINEL", + "ROOTFORM_DATADOG_CLOUDFLARE_KEY_SENTINEL", + "ROOTFORM_DATADOG_FASTLY_KEY_SENTINEL", + "ROOTFORM_HCP_DATADOG_API_SENTINEL", + "ROOTFORM_ATLAS_OBSERVABILITY_SECRET", +]); + const rules: Array<[string, RegExp]> = [ [ "personal-path", @@ -7,7 +21,7 @@ const rules: Array<[string, RegExp]> = [ ], [ "credential", - /BEGIN (?:RSA|OPENSSH|EC|DSA)? ?PRIVATE KEY|(?:github_pat_|gh[pousr]_|npm_)[A-Za-z0-9_]{12,}|\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|xox[abprs]-[A-Za-z0-9-]{12,}|AIza[A-Za-z0-9_-]{35}|\bBearer\s+[A-Za-z0-9._~+/-]{24,}=*|\b(?:CLOUDFLARE_API_TOKEN|AWS_SECRET_ACCESS_KEY|OPENAI_API_KEY|API_TOKEN|API_KEY)\s*[:=]\s*["\x27]?[A-Za-z0-9_+./~-]{24,}/gu, + /BEGIN (?:RSA|OPENSSH|EC|DSA)? ?PRIVATE KEY|(?:github_pat_|gh[pousr]_|npm_)[A-Za-z0-9_]{12,}|\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|xox[abprs]-[A-Za-z0-9-]{12,}|AIza[A-Za-z0-9_-]{35}|\bBearer\s+[A-Za-z0-9._~+/-]{24,}=*/gu, ], ["private-workspace", /\b(?:rootform-dev\/)?notes\/[A-Za-z0-9][A-Za-z0-9._/-]*/gu], [ @@ -22,22 +36,129 @@ const rules: Array<[string, RegExp]> = [ /** Return only rule and line; never retain or display the matched value. */ export function publicationIssues(text: string): PublicationIssue[] { - return rules.flatMap(([rule, pattern]) => - [...text.matchAll(pattern)].map((match) => ({ - rule, - line: text.slice(0, match.index).split("\n").length, - })), - ); + const found: PublicationIssue[] = []; + const scan = (source: string) => { + for (let depth = 0; depth < 3; depth++) { + const decoded = source + .replace(/(?:%[0-9a-f]{2})+/giu, (encoded) => { + try { + return decodeURIComponent(encoded); + } catch { + return encoded; + } + }) + .replace( + /&(?:#(x[0-9a-f]+|[0-9]+)|amp|sol|colon|quot|apos);/giu, + (entity, numeric: string | undefined) => { + if (numeric) { + const code = numeric.toLowerCase().startsWith("x") + ? Number.parseInt(numeric.slice(1), 16) + : Number(numeric); + return code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : entity; + } + return ( + ( + { + "&": "&", + "/": "/", + ":": ":", + """: '"', + "'": "'", + } as Record + )[entity.toLowerCase()] ?? entity + ); + }, + ) + .replaceAll("\0", ""); + if (decoded === source) break; + source = decoded; + } + for (const [rule, pattern] of rules) + for (const match of source.matchAll(pattern)) + found.push({ rule, line: source.slice(0, match.index).split("\n").length }); + const namedCredential = + /\b(?:CLOUDFLARE_API_(?:TOKEN|KEY)|AWS_SECRET_ACCESS_KEY|OPENAI_API_KEY|API_TOKEN|API_KEY)["']?\s*[:=]\s*["']?([A-Za-z0-9_+./~=-]{24,})/giu; + for (const match of source.matchAll(namedCredential)) + if (!syntheticCredentialValues.has(match[1] ?? "")) + found.push({ rule: "credential", line: source.slice(0, match.index).split("\n").length }); + }; + const inspect = (value: unknown, depth: number) => { + if (depth > 128) { + found.push({ rule: "uninspectable-content", line: 1 }); + return; + } + if (typeof value === "string") { + scan(value); + if (/^\s*[[{]/u.test(value)) { + let nested: unknown; + try { + nested = JSON.parse(value); + } catch { + return; + } + inspect(nested, depth + 1); + } + } else if (Array.isArray(value)) for (const item of value) inspect(item, depth + 1); + else if (value !== null && typeof value === "object") { + const payload = value as Record; + if ( + typeof payload.content === "string" && + (payload.encoding === "base64" || ("message" in payload && payload.encoding === undefined)) + ) { + try { + if ( + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test( + payload.content, + ) + ) + throw new Error("Invalid encoding"); + const decoded = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode( + Uint8Array.from(atob(payload.content), (character) => character.charCodeAt(0)), + ); + inspect(decoded, depth + 1); + } catch { + found.push({ rule: "uninspectable-content", line: 1 }); + } + } + for (const [key, item] of Object.entries(payload)) { + if ( + /^(?:CLOUDFLARE_API_(?:TOKEN|KEY)|AWS_SECRET_ACCESS_KEY|OPENAI_API_KEY|API_TOKEN|API_KEY)$/iu.test( + key, + ) && + typeof item === "string" && + !syntheticCredentialValues.has(item) && + /^[A-Za-z0-9_+./~=-]{24,}$/u.test(item) + ) + found.push({ rule: "credential", line: 1 }); + scan(key); + inspect(item, depth + 1); + } + } + }; + scan(text); + // Parsing valid JSON reveals escapes without interpreting regex or source-code literals. + let structured: unknown; + try { + structured = JSON.parse(text); + } catch { + return found; + } + inspect(structured, 0); + return found; } /** Validate all outbound strings before a bot makes its first write. */ export function assertPublicMessage(value: unknown): void { - if (typeof value === "string") { - const issue = publicationIssues(value)[0]; - if (issue) throw new Error(`Public message refused: ${issue.rule}`); - } else if (Array.isArray(value)) { - for (const item of value) assertPublicMessage(item); - } else if (value !== null && typeof value === "object") { - for (const item of Object.values(value)) assertPublicMessage(item); + if (value === undefined) return; + let text: string; + try { + const serialized = typeof value === "string" ? value : JSON.stringify(value); + if (typeof serialized !== "string") throw new Error("Invalid payload"); + text = serialized; + } catch { + throw new Error("Public message refused: uninspectable-content"); } + const issues = publicationIssues(text); + const issue = issues[0]; + if (issue) throw new Error(`Public message refused: ${issue.rule}`); } diff --git a/scripts/verify.ts b/scripts/verify.ts index 56e2f9c..049f7a1 100644 --- a/scripts/verify.ts +++ b/scripts/verify.ts @@ -56,6 +56,7 @@ try { for (const command of [ ["bun", "scripts/check-publication.ts"], + ["bun", "test", "scripts/publication-safety.test.ts"], ["gitleaks", "git", "--no-banner", "--redact", "--config", ".gitleaks.toml", "."], ["bun", "run", "check:format"], ["bun", "run", "build:vscode"], diff --git a/vscode/test/extensionHost.cjs b/vscode/test/extensionHost.cjs index 6c3ac2c..27a0beb 100644 --- a/vscode/test/extensionHost.cjs +++ b/vscode/test/extensionHost.cjs @@ -365,14 +365,35 @@ async function exerciseCompletionMatrix(firstRoot) { } } -async function rootformLogContains(message) { +async function rootformLogBodies() { const logs = path.join(process.env.ROOTFORM_TEST_USER_DATA, "logs"); + const bodies = []; for (const relative of await fs.readdir(logs, { recursive: true })) { if (relative.endsWith("Rootform.log")) { - if ((await fs.readFile(path.join(logs, relative), "utf8")).includes(message)) return true; + bodies.push(await fs.readFile(path.join(logs, relative), "utf8")); } } - return false; + return bodies; +} + +async function rootformLogContains(message) { + return (await rootformLogBodies()).some((body) => body.includes(message)); +} + +async function rootformLogOccurrences(message) { + return (await rootformLogBodies()).reduce( + (count, body) => count + body.split(message).length - 1, + 0, + ); +} + +function rootformChildProcesses(matches) { + return cp + .execFileSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" }) + .split("\n") + .map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line)) + .filter((match) => match && Number(match[2]) === process.pid && matches(match[3])) + .map((match) => Number(match[1])); } function referencePosition(document) { @@ -745,6 +766,46 @@ async function run() { }); if (process.platform === "darwin" || process.platform === "linux") { + // A configured absolute path that does not exist must report the + // not-found remediation once, leave no Rootform process behind, and + // recover when the provided executable path is restored. + const absentServer = path.join(testRoot, "absent-server", "rootform"); + assert.equal(path.isAbsolute(absentServer), true, "The absent path must be absolute."); + await assert.rejects(fs.access(absentServer), "The absent path fixture must not exist."); + await settings.update("server.path", absentServer, vscode.ConfigurationTarget.Global); + const notFound = `Rootform executable "${absentServer}" was not found. Install Rootform or set "rootform.server.path" to a local executable.`; + await until("nonexistent executable reported with its remediation", () => + rootformLogContains(notFound), + ); + await delay(500); + assert.equal( + await rootformLogOccurrences(`Rootform executable "${absentServer}" was not found.`), + 1, + "A nonexistent executable must be reported once, without a retry loop.", + ); + await until( + "no Rootform process left behind while the executable is absent", + () => rootformChildProcesses((command) => command.endsWith(" lsp")).length === 0, + 5000, + ); + assert.deepEqual( + rootformChildProcesses((command) => command.includes(absentServer)), + [], + "A nonexistent executable must not start a process.", + ); + await settings.update("server.path", rootformBinary, vscode.ConfigurationTarget.Global); + await until( + "hover after restoring the provided executable path", + async () => + ( + await vscode.commands.executeCommand( + "vscode.executeHoverProvider", + rule.uri, + referencePosition(rule), + ) + )?.length > 0, + ); + // A process that exits before initialize must fail once, then recover when // the configured executable is corrected. Use a local recording fixture. const incompatible = path.join(testRoot, "incompatible-server"); @@ -817,18 +878,9 @@ async function run() { } } const children = () => - cp - .execFileSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" }) - .split("\n") - .map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line)) - .filter(Boolean) - .filter( - (item) => - Number(item[2]) === process.pid && - item[3].includes(rootformBinary) && - item[3].endsWith(" lsp"), - ) - .map((item) => Number(item[1])); + rootformChildProcesses( + (command) => command.includes(rootformBinary) && command.endsWith(" lsp"), + ); const [serverPid] = await until( "one running local Rootform child", () => children().length === 1 && children(),