diff --git a/CHANGELOG.md b/CHANGELOG.md index efccccf..5d4aa00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,11 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed +- **The weekly watcher files digests instead of one issue per item.** `scripts/watch.js` now opens one checklist + issue per run for arXiv and one for NVD, titled `[watch-digest] …`. Research items are grouped by their likely + `docs/TRIAGE_RULES.md` route: possible benchmarks (rule 3) first, then literature (rule 1). OWASP repo commits and + framework-page changes still get one issue each. In October 2026 four runs opened 98 issues, and the triage rules + route almost all of them to "note and close"; the same runs would now open 8. - **ASVS framework renamed `OWASP ASVS 4.0.3` → `OWASP ASVS 5.0.0`** in the generated data, exports, registry, webapp and docs. The three ASVS mapping files were already translated to 5.0.0 identifiers (#123); this brings the framework's name in line. **Consumers that filter exports by framework name must update the string.** 5.0.0 is diff --git a/scripts/watch.js b/scripts/watch.js index 24715e3..e5516de 100644 --- a/scripts/watch.js +++ b/scripts/watch.js @@ -3,7 +3,8 @@ * OWASP GenAI Crosswalk — External Source Watcher * ────────────────────────────────────────────────────────────────────────── * Monitors OWASP repos, arXiv, NVD, and framework pages for changes. - * Opens GitHub Issues for each new finding. + * Opens one digest issue per run for arXiv and for NVD, and one issue per + * OWASP repo or framework-page change. * * Usage: * node scripts/watch.js # run all watchers @@ -287,6 +288,20 @@ function mapArxivToOwasp(text) { return [...new Set(mappings)]; } +// Which docs/TRIAGE_RULES.md route a paper probably takes. Like the OWASP +// hints, this only orders the digest — the triager still decides. Abstracts +// mention "benchmarks" when they merely evaluate on one, so the abstract only +// counts when the paper says it introduces or releases the benchmark itself. +// `\w+Bench` is case-sensitive on purpose: it catches names like AfterDeleteBench +// without matching "workbench". +const BENCHMARK_TITLE_RE = /\b[Bb]ench(mark(s|ing)?)?\b|\w+Bench\b|\b[Dd]atasets?\b|\b[Tt]est ?[Ss]uite\b/; +const BENCHMARK_ABSTRACT_RE = /\bwe (introduce|present|release|propose|construct|build|curate)\b[^.]{0,80}\b(benchmark|dataset|test ?suite)\b/i; + +function triageRoute(title, abstract) { + if (BENCHMARK_TITLE_RE.test(title) || BENCHMARK_ABSTRACT_RE.test(abstract)) return 'benchmark'; + return 'literature'; +} + /** * Minimal Atom XML parser — extracts blocks and named fields. */ @@ -366,6 +381,16 @@ async function watchArxiv(state) { findings.push({ label: 'new-research', + digest: { + kind: 'research', + id: entry.id, + title: entry.title, + url: `https://arxiv.org/abs/${entry.id}`, + authors: entry.authors, + abstract: entry.summary || '', + hints: owaspHints, + route: triageRoute(entry.title, entry.summary || ''), + }, title: `[new-research] arXiv:${entry.id} — ${entry.title.slice(0, 80)}`, body: [ `## New arXiv Research Paper`, @@ -489,6 +514,14 @@ async function watchNvd(state, sinceOverride) { findings.push({ label: 'new-cve', + digest: { + kind: 'cve', + id: cveId, + url: nvdUrl, + cvss: cvssScore, + description: engDesc, + hint: owaspHint, + }, title: `[new-cve] ${cveId} — ${engDesc.slice(0, 80)}`, body: [ `## New CVE — AI/LLM Related`, @@ -658,6 +691,107 @@ async function watchFrameworks(state) { // ─── Issue Reporter ─────────────────────────────────────────────────────────── +// ─── Weekly Digests ────────────────────────────────────────────────────────── +// +// One issue per paper or CVE buried the tracker: between 2026-10-05 and +// 2026-10-26 the watcher opened 98 issues, and docs/TRIAGE_RULES.md routes +// almost every paper to "note and close". arXiv and NVD findings now go into +// one checklist issue per source per run. OWASP repo commits and framework +// page changes are rare and usually actionable, so they keep their own issues. + +const TRIAGE_RULES_URL = `https://github.com/${GITHUB_REPOSITORY}/blob/main/docs/TRIAGE_RULES.md`; +const DIGEST_EXCERPT_CHARS = 300; +const DIGEST_MAX_BODY = 60_000; // GitHub rejects issue bodies over 65,536 chars + +const excerpt = (text) => { + const t = (text || '').trim(); + return t.length > DIGEST_EXCERPT_CHARS ? `${t.slice(0, DIGEST_EXCERPT_CHARS)}…` : t; +}; + +function researchLine(d) { + const authors = d.authors.length > 3 ? `${d.authors.slice(0, 3).join(', ')} et al.` : d.authors.join(', '); + const hints = d.hints.length ? ` Hints: ${d.hints.join(', ')}.` : ''; + return [ + `- [ ] [arXiv:${d.id}](${d.url}) **${d.title}** — ${authors || 'N/A'}.${hints}`, + ` > ${excerpt(d.abstract)}`, + ].join('\n'); +} + +function cveLine(d) { + return [ + `- [ ] [${d.id}](${d.url}) — CVSS ${d.cvss}. Hint: ${d.hint}.`, + ` > ${excerpt(d.description)}`, + ].join('\n'); +} + +function researchDigest(items, date) { + const benchmarks = items.filter((i) => i.route === 'benchmark'); + const literature = items.filter((i) => i.route !== 'benchmark'); + const section = (heading, list) => list.length + ? [`### ${heading} (${list.length})`, '', ...list.map(researchLine), ''] + : []; + return { + label: 'new-research', + title: `[watch-digest] Research — ${date} (${items.length} paper${items.length === 1 ? '' : 's'})`, + body: [ + `## Weekly research digest — ${date}`, + '', + `${items.length} new arXiv cs.CR paper${items.length === 1 ? '' : 's'} on GenAI security. Route each one with ` + + `[\`docs/TRIAGE_RULES.md\`](${TRIAGE_RULES_URL}), tick it, and close this issue when every box is ticked. ` + + 'Open a separate issue or PR only for an item that leads to work.', + '', + 'The grouping and the OWASP hints are keyword matches, not mappings.', + '', + ...section('Possible benchmarks — rule 3 candidates for `evals/EXTERNAL_BENCHMARKS.md`', benchmarks), + ...section('Literature — rule 1, note and close unless it documents a real-world failure', literature), + ].join('\n'), + }; +} + +function cveDigest(items, date) { + return { + label: 'new-cve', + title: `[watch-digest] CVEs — ${date} (${items.length} CVE${items.length === 1 ? '' : 's'})`, + body: [ + `## Weekly CVE digest — ${date}`, + '', + `${items.length} new NVD CVE${items.length === 1 ? '' : 's'} matching the GenAI keywords. Per ` + + `[\`docs/TRIAGE_RULES.md\`](${TRIAGE_RULES_URL}) § CVEs, a CVE in GenAI tooling becomes an incident ` + + 'record when it names an affected version and has a citable advisory. Tick each one once routed, and close this ' + + 'issue when every box is ticked.', + '', + 'The OWASP hints are keyword matches, not mappings.', + '', + ...items.map(cveLine), + '', + ].join('\n'), + }; +} + +function capBody(finding) { + if (finding.body.length <= DIGEST_MAX_BODY) return finding; + const cut = finding.body.lastIndexOf('\n- [ ]', DIGEST_MAX_BODY); + return { + ...finding, + body: `${finding.body.slice(0, cut)}\n\n_Digest truncated at ${DIGEST_MAX_BODY} characters; run the watcher with ` + + '`--dry-run` for the full list._\n', + }; +} + +/** + * Fold arXiv and NVD findings into one digest finding per source. Everything + * else passes through unchanged, ahead of the digests. + */ +function buildDigests(findings, date) { + const single = findings.filter((f) => !f.digest); + const research = findings.filter((f) => f.digest?.kind === 'research').map((f) => f.digest); + const cves = findings.filter((f) => f.digest?.kind === 'cve').map((f) => f.digest); + const out = [...single]; + if (research.length) out.push(capBody(researchDigest(research, date))); + if (cves.length) out.push(capBody(cveDigest(cves, date))); + return out; +} + /** * Either open GitHub Issues for each finding or print them to stdout. * Returns the count of successfully handled findings. @@ -779,7 +913,7 @@ async function main() { } // ── Report findings ─────────────────────────────────────────────────────── - const issueCount = await reportFindings(allFindings, DRY_RUN); + const issueCount = await reportFindings(buildDigests(allFindings, runDate.slice(0, 10)), DRY_RUN); // ── Update state ────────────────────────────────────────────────────────── if (!DRY_RUN) { @@ -809,4 +943,4 @@ if (require.main === module) { }); } -module.exports = { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp }; +module.exports = { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp, triageRoute, buildDigests, DIGEST_MAX_BODY }; diff --git a/scripts/watch.test.mjs b/scripts/watch.test.mjs index 9984921..7c75f86 100644 --- a/scripts/watch.test.mjs +++ b/scripts/watch.test.mjs @@ -5,6 +5,10 @@ * them outside GenAI security (CBDC settlement, 6G NOMA, MIMO lattices), * because unquoted multi-word terms turned the query into "any cs.CR paper". * Its OWASP hints also pointed at pre-2026 entry ids. These tests pin both. + * + * With the query fixed, volume was still the problem: 98 open issues from four + * runs in October 2026, nearly all routed to "note and close". arXiv and NVD + * findings now fold into one digest issue per source per run. */ import test from 'node:test'; @@ -16,7 +20,9 @@ import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const require = createRequire(import.meta.url); -const { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp } = require(path.join(ROOT, 'scripts', 'watch.js')); +const { + ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp, triageRoute, buildDigests, DIGEST_MAX_BODY, +} = require(path.join(ROOT, 'scripts', 'watch.js')); /** * Field terms followed by a bare word, e.g. `ti:LLM security`. arXiv reads the @@ -74,3 +80,74 @@ test('every hint id exists and still carries the title its rule names', () => { } } }); + +// Titles from the watcher's October 2026 issues. +test('papers that name a benchmark in the title are routed as rule 3 candidates', () => { + for (const title of [ + 'AgentPhishing: A Benchmark of LLM Email Agents Under Adaptive Attack', + 'AfterDeleteBench: Auditing Post-Deletion Information Persistence in LLM Agents', + 'Omni-UBench: Benchmarking Omni-Modal LLM-based Guardrails Across Safety, Utility', + ]) assert.equal(triageRoute(title, ''), 'benchmark', title); +}); + +test('papers that only evaluate on a benchmark stay literature', () => { + assert.equal(triageRoute('Amplifying Latency and Cost of LLM Inference via Rowhammer Attack', + 'We evaluate on three public benchmarks and show a 4x slowdown.'), 'literature'); + assert.equal(triageRoute('Securing the LLM workbench', ''), 'literature'); +}); + +test('a paper that says it releases a dataset is a rule 3 candidate', () => { + assert.equal(triageRoute('Prompt leakage in the wild', + 'We introduce a new dataset of 4,000 leaked system prompts.'), 'benchmark'); +}); + +const paper = (n, title = `Paper ${n}`) => ({ + label: 'new-research', title: `[new-research] arXiv:2610.${n}`, body: 'per-item body', + digest: { kind: 'research', id: `2610.${n}`, title, url: `https://arxiv.org/abs/2610.${n}`, + authors: ['A', 'B', 'C', 'D'], abstract: 'x'.repeat(500), hints: ['LLM01'], route: triageRoute(title, '') }, +}); +const cve = (n) => ({ + label: 'new-cve', title: `[new-cve] CVE-2026-${n}`, body: 'per-item body', + digest: { kind: 'cve', id: `CVE-2026-${n}`, url: `https://nvd.nist.gov/vuln/detail/CVE-2026-${n}`, + cvss: 7.5, description: 'A flaw in an LLM gateway.', hint: '_Manual mapping required_' }, +}); +const framework = { label: 'framework-update', title: '[framework-update] MITRE ATLAS changed', body: 'b' }; + +test('arXiv and NVD findings fold into one digest each; other findings keep their own issue', () => { + const papers = Array.from({ length: 20 }, (_, i) => paper(10000 + i)); + const cves = Array.from({ length: 31 }, (_, i) => cve(100000 + i)); + const out = buildDigests([framework, ...papers, ...cves], '2026-10-26'); + + assert.equal(out.length, 3); + assert.equal(out[0], framework); + const [research, cveIssue] = out.slice(1); + assert.equal(research.label, 'new-research'); + assert.equal(research.title, '[watch-digest] Research — 2026-10-26 (20 papers)'); + assert.equal((research.body.match(/^- \[ \] /gm) || []).length, 20); + assert.equal(cveIssue.label, 'new-cve'); + assert.equal(cveIssue.title, '[watch-digest] CVEs — 2026-10-26 (31 CVEs)'); + assert.equal((cveIssue.body.match(/^- \[ \] /gm) || []).length, 31); +}); + +test('the research digest lists benchmark candidates before literature', () => { + const [digest] = buildDigests([paper(1), paper(2, 'XBench: a benchmark for agents')], '2026-10-26'); + const body = digest.body; + assert.ok(body.indexOf('Possible benchmarks') < body.indexOf('Literature')); + assert.ok(body.indexOf('2610.2') < body.indexOf('2610.1'), 'the benchmark paper is listed first'); + assert.match(body, /A, B, C et al\./); + assert.match(body, /x{300}…/, 'abstracts are cut to an excerpt'); +}); + +test('a run with nothing from arXiv or NVD opens no digest', () => { + assert.deepEqual(buildDigests([framework], '2026-10-26'), [framework]); + assert.deepEqual(buildDigests([], '2026-10-26'), []); +}); + +test('an oversized digest is cut at an item boundary under the GitHub limit', () => { + const papers = Array.from({ length: 400 }, (_, i) => paper(10000 + i)); + const [digest] = buildDigests(papers, '2026-10-26'); + assert.ok(digest.body.length < 65_536); + assert.ok(digest.body.length <= DIGEST_MAX_BODY + 200); + assert.match(digest.body, /Digest truncated/); + assert.match(digest.body, /x{300}…\n\n_Digest truncated/, 'the last item before the note is complete'); +});