From f93798ac54e3590ec0daa1348a91da4ec4831cab Mon Sep 17 00:00:00 2001 From: emmanuelgjr Date: Wed, 28 Oct 2026 11:51:50 -0400 Subject: [PATCH] Watcher: one digest issue per run for arXiv and NVD Four runs in October 2026 opened 98 issues, and docs/TRIAGE_RULES.md routes almost every paper to "note and close". arXiv and NVD findings now fold into one checklist issue per source per run, with research items grouped by likely triage route (possible benchmarks first). OWASP repo commits and framework-page changes keep their own issues. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++ scripts/watch.js | 140 ++++++++++++++++++++++++++++++++++++++++- scripts/watch.test.mjs | 79 ++++++++++++++++++++++- 3 files changed, 220 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efccccf..5d4aa00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,11 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed +- **The weekly watcher files digests instead of one issue per item.** `scripts/watch.js` now opens one checklist + issue per run for arXiv and one for NVD, titled `[watch-digest] …`. Research items are grouped by their likely + `docs/TRIAGE_RULES.md` route: possible benchmarks (rule 3) first, then literature (rule 1). OWASP repo commits and + framework-page changes still get one issue each. In October 2026 four runs opened 98 issues, and the triage rules + route almost all of them to "note and close"; the same runs would now open 8. - **ASVS framework renamed `OWASP ASVS 4.0.3` → `OWASP ASVS 5.0.0`** in the generated data, exports, registry, webapp and docs. The three ASVS mapping files were already translated to 5.0.0 identifiers (#123); this brings the framework's name in line. **Consumers that filter exports by framework name must update the string.** 5.0.0 is diff --git a/scripts/watch.js b/scripts/watch.js index 24715e3..e5516de 100644 --- a/scripts/watch.js +++ b/scripts/watch.js @@ -3,7 +3,8 @@ * OWASP GenAI Crosswalk — External Source Watcher * ────────────────────────────────────────────────────────────────────────── * Monitors OWASP repos, arXiv, NVD, and framework pages for changes. - * Opens GitHub Issues for each new finding. + * Opens one digest issue per run for arXiv and for NVD, and one issue per + * OWASP repo or framework-page change. * * Usage: * node scripts/watch.js # run all watchers @@ -287,6 +288,20 @@ function mapArxivToOwasp(text) { return [...new Set(mappings)]; } +// Which docs/TRIAGE_RULES.md route a paper probably takes. Like the OWASP +// hints, this only orders the digest — the triager still decides. Abstracts +// mention "benchmarks" when they merely evaluate on one, so the abstract only +// counts when the paper says it introduces or releases the benchmark itself. +// `\w+Bench` is case-sensitive on purpose: it catches names like AfterDeleteBench +// without matching "workbench". +const BENCHMARK_TITLE_RE = /\b[Bb]ench(mark(s|ing)?)?\b|\w+Bench\b|\b[Dd]atasets?\b|\b[Tt]est ?[Ss]uite\b/; +const BENCHMARK_ABSTRACT_RE = /\bwe (introduce|present|release|propose|construct|build|curate)\b[^.]{0,80}\b(benchmark|dataset|test ?suite)\b/i; + +function triageRoute(title, abstract) { + if (BENCHMARK_TITLE_RE.test(title) || BENCHMARK_ABSTRACT_RE.test(abstract)) return 'benchmark'; + return 'literature'; +} + /** * Minimal Atom XML parser — extracts blocks and named fields. */ @@ -366,6 +381,16 @@ async function watchArxiv(state) { findings.push({ label: 'new-research', + digest: { + kind: 'research', + id: entry.id, + title: entry.title, + url: `https://arxiv.org/abs/${entry.id}`, + authors: entry.authors, + abstract: entry.summary || '', + hints: owaspHints, + route: triageRoute(entry.title, entry.summary || ''), + }, title: `[new-research] arXiv:${entry.id} — ${entry.title.slice(0, 80)}`, body: [ `## New arXiv Research Paper`, @@ -489,6 +514,14 @@ async function watchNvd(state, sinceOverride) { findings.push({ label: 'new-cve', + digest: { + kind: 'cve', + id: cveId, + url: nvdUrl, + cvss: cvssScore, + description: engDesc, + hint: owaspHint, + }, title: `[new-cve] ${cveId} — ${engDesc.slice(0, 80)}`, body: [ `## New CVE — AI/LLM Related`, @@ -658,6 +691,107 @@ async function watchFrameworks(state) { // ─── Issue Reporter ─────────────────────────────────────────────────────────── +// ─── Weekly Digests ────────────────────────────────────────────────────────── +// +// One issue per paper or CVE buried the tracker: between 2026-10-05 and +// 2026-10-26 the watcher opened 98 issues, and docs/TRIAGE_RULES.md routes +// almost every paper to "note and close". arXiv and NVD findings now go into +// one checklist issue per source per run. OWASP repo commits and framework +// page changes are rare and usually actionable, so they keep their own issues. + +const TRIAGE_RULES_URL = `https://github.com/${GITHUB_REPOSITORY}/blob/main/docs/TRIAGE_RULES.md`; +const DIGEST_EXCERPT_CHARS = 300; +const DIGEST_MAX_BODY = 60_000; // GitHub rejects issue bodies over 65,536 chars + +const excerpt = (text) => { + const t = (text || '').trim(); + return t.length > DIGEST_EXCERPT_CHARS ? `${t.slice(0, DIGEST_EXCERPT_CHARS)}…` : t; +}; + +function researchLine(d) { + const authors = d.authors.length > 3 ? `${d.authors.slice(0, 3).join(', ')} et al.` : d.authors.join(', '); + const hints = d.hints.length ? ` Hints: ${d.hints.join(', ')}.` : ''; + return [ + `- [ ] [arXiv:${d.id}](${d.url}) **${d.title}** — ${authors || 'N/A'}.${hints}`, + ` > ${excerpt(d.abstract)}`, + ].join('\n'); +} + +function cveLine(d) { + return [ + `- [ ] [${d.id}](${d.url}) — CVSS ${d.cvss}. Hint: ${d.hint}.`, + ` > ${excerpt(d.description)}`, + ].join('\n'); +} + +function researchDigest(items, date) { + const benchmarks = items.filter((i) => i.route === 'benchmark'); + const literature = items.filter((i) => i.route !== 'benchmark'); + const section = (heading, list) => list.length + ? [`### ${heading} (${list.length})`, '', ...list.map(researchLine), ''] + : []; + return { + label: 'new-research', + title: `[watch-digest] Research — ${date} (${items.length} paper${items.length === 1 ? '' : 's'})`, + body: [ + `## Weekly research digest — ${date}`, + '', + `${items.length} new arXiv cs.CR paper${items.length === 1 ? '' : 's'} on GenAI security. Route each one with ` + + `[\`docs/TRIAGE_RULES.md\`](${TRIAGE_RULES_URL}), tick it, and close this issue when every box is ticked. ` + + 'Open a separate issue or PR only for an item that leads to work.', + '', + 'The grouping and the OWASP hints are keyword matches, not mappings.', + '', + ...section('Possible benchmarks — rule 3 candidates for `evals/EXTERNAL_BENCHMARKS.md`', benchmarks), + ...section('Literature — rule 1, note and close unless it documents a real-world failure', literature), + ].join('\n'), + }; +} + +function cveDigest(items, date) { + return { + label: 'new-cve', + title: `[watch-digest] CVEs — ${date} (${items.length} CVE${items.length === 1 ? '' : 's'})`, + body: [ + `## Weekly CVE digest — ${date}`, + '', + `${items.length} new NVD CVE${items.length === 1 ? '' : 's'} matching the GenAI keywords. Per ` + + `[\`docs/TRIAGE_RULES.md\`](${TRIAGE_RULES_URL}) § CVEs, a CVE in GenAI tooling becomes an incident ` + + 'record when it names an affected version and has a citable advisory. Tick each one once routed, and close this ' + + 'issue when every box is ticked.', + '', + 'The OWASP hints are keyword matches, not mappings.', + '', + ...items.map(cveLine), + '', + ].join('\n'), + }; +} + +function capBody(finding) { + if (finding.body.length <= DIGEST_MAX_BODY) return finding; + const cut = finding.body.lastIndexOf('\n- [ ]', DIGEST_MAX_BODY); + return { + ...finding, + body: `${finding.body.slice(0, cut)}\n\n_Digest truncated at ${DIGEST_MAX_BODY} characters; run the watcher with ` + + '`--dry-run` for the full list._\n', + }; +} + +/** + * Fold arXiv and NVD findings into one digest finding per source. Everything + * else passes through unchanged, ahead of the digests. + */ +function buildDigests(findings, date) { + const single = findings.filter((f) => !f.digest); + const research = findings.filter((f) => f.digest?.kind === 'research').map((f) => f.digest); + const cves = findings.filter((f) => f.digest?.kind === 'cve').map((f) => f.digest); + const out = [...single]; + if (research.length) out.push(capBody(researchDigest(research, date))); + if (cves.length) out.push(capBody(cveDigest(cves, date))); + return out; +} + /** * Either open GitHub Issues for each finding or print them to stdout. * Returns the count of successfully handled findings. @@ -779,7 +913,7 @@ async function main() { } // ── Report findings ─────────────────────────────────────────────────────── - const issueCount = await reportFindings(allFindings, DRY_RUN); + const issueCount = await reportFindings(buildDigests(allFindings, runDate.slice(0, 10)), DRY_RUN); // ── Update state ────────────────────────────────────────────────────────── if (!DRY_RUN) { @@ -809,4 +943,4 @@ if (require.main === module) { }); } -module.exports = { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp }; +module.exports = { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp, triageRoute, buildDigests, DIGEST_MAX_BODY }; diff --git a/scripts/watch.test.mjs b/scripts/watch.test.mjs index 9984921..7c75f86 100644 --- a/scripts/watch.test.mjs +++ b/scripts/watch.test.mjs @@ -5,6 +5,10 @@ * them outside GenAI security (CBDC settlement, 6G NOMA, MIMO lattices), * because unquoted multi-word terms turned the query into "any cs.CR paper". * Its OWASP hints also pointed at pre-2026 entry ids. These tests pin both. + * + * With the query fixed, volume was still the problem: 98 open issues from four + * runs in October 2026, nearly all routed to "note and close". arXiv and NVD + * findings now fold into one digest issue per source per run. */ import test from 'node:test'; @@ -16,7 +20,9 @@ import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const require = createRequire(import.meta.url); -const { ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp } = require(path.join(ROOT, 'scripts', 'watch.js')); +const { + ARXIV_URL, ARXIV_HINT_RULES, mapArxivToOwasp, triageRoute, buildDigests, DIGEST_MAX_BODY, +} = require(path.join(ROOT, 'scripts', 'watch.js')); /** * Field terms followed by a bare word, e.g. `ti:LLM security`. arXiv reads the @@ -74,3 +80,74 @@ test('every hint id exists and still carries the title its rule names', () => { } } }); + +// Titles from the watcher's October 2026 issues. +test('papers that name a benchmark in the title are routed as rule 3 candidates', () => { + for (const title of [ + 'AgentPhishing: A Benchmark of LLM Email Agents Under Adaptive Attack', + 'AfterDeleteBench: Auditing Post-Deletion Information Persistence in LLM Agents', + 'Omni-UBench: Benchmarking Omni-Modal LLM-based Guardrails Across Safety, Utility', + ]) assert.equal(triageRoute(title, ''), 'benchmark', title); +}); + +test('papers that only evaluate on a benchmark stay literature', () => { + assert.equal(triageRoute('Amplifying Latency and Cost of LLM Inference via Rowhammer Attack', + 'We evaluate on three public benchmarks and show a 4x slowdown.'), 'literature'); + assert.equal(triageRoute('Securing the LLM workbench', ''), 'literature'); +}); + +test('a paper that says it releases a dataset is a rule 3 candidate', () => { + assert.equal(triageRoute('Prompt leakage in the wild', + 'We introduce a new dataset of 4,000 leaked system prompts.'), 'benchmark'); +}); + +const paper = (n, title = `Paper ${n}`) => ({ + label: 'new-research', title: `[new-research] arXiv:2610.${n}`, body: 'per-item body', + digest: { kind: 'research', id: `2610.${n}`, title, url: `https://arxiv.org/abs/2610.${n}`, + authors: ['A', 'B', 'C', 'D'], abstract: 'x'.repeat(500), hints: ['LLM01'], route: triageRoute(title, '') }, +}); +const cve = (n) => ({ + label: 'new-cve', title: `[new-cve] CVE-2026-${n}`, body: 'per-item body', + digest: { kind: 'cve', id: `CVE-2026-${n}`, url: `https://nvd.nist.gov/vuln/detail/CVE-2026-${n}`, + cvss: 7.5, description: 'A flaw in an LLM gateway.', hint: '_Manual mapping required_' }, +}); +const framework = { label: 'framework-update', title: '[framework-update] MITRE ATLAS changed', body: 'b' }; + +test('arXiv and NVD findings fold into one digest each; other findings keep their own issue', () => { + const papers = Array.from({ length: 20 }, (_, i) => paper(10000 + i)); + const cves = Array.from({ length: 31 }, (_, i) => cve(100000 + i)); + const out = buildDigests([framework, ...papers, ...cves], '2026-10-26'); + + assert.equal(out.length, 3); + assert.equal(out[0], framework); + const [research, cveIssue] = out.slice(1); + assert.equal(research.label, 'new-research'); + assert.equal(research.title, '[watch-digest] Research — 2026-10-26 (20 papers)'); + assert.equal((research.body.match(/^- \[ \] /gm) || []).length, 20); + assert.equal(cveIssue.label, 'new-cve'); + assert.equal(cveIssue.title, '[watch-digest] CVEs — 2026-10-26 (31 CVEs)'); + assert.equal((cveIssue.body.match(/^- \[ \] /gm) || []).length, 31); +}); + +test('the research digest lists benchmark candidates before literature', () => { + const [digest] = buildDigests([paper(1), paper(2, 'XBench: a benchmark for agents')], '2026-10-26'); + const body = digest.body; + assert.ok(body.indexOf('Possible benchmarks') < body.indexOf('Literature')); + assert.ok(body.indexOf('2610.2') < body.indexOf('2610.1'), 'the benchmark paper is listed first'); + assert.match(body, /A, B, C et al\./); + assert.match(body, /x{300}…/, 'abstracts are cut to an excerpt'); +}); + +test('a run with nothing from arXiv or NVD opens no digest', () => { + assert.deepEqual(buildDigests([framework], '2026-10-26'), [framework]); + assert.deepEqual(buildDigests([], '2026-10-26'), []); +}); + +test('an oversized digest is cut at an item boundary under the GitHub limit', () => { + const papers = Array.from({ length: 400 }, (_, i) => paper(10000 + i)); + const [digest] = buildDigests(papers, '2026-10-26'); + assert.ok(digest.body.length < 65_536); + assert.ok(digest.body.length <= DIGEST_MAX_BODY + 200); + assert.match(digest.body, /Digest truncated/); + assert.match(digest.body, /x{300}…\n\n_Digest truncated/, 'the last item before the note is complete'); +});