From 967ddc5dd45197756306d7cbfd7f87b9a81fafa3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:36:40 +0000 Subject: [PATCH] Add inbox/ intake: drop a file in a PR, get it processed into corpus/ - inbox/ is a PR-only staging area; validate_corpus.py fails while it holds anything but its README, so nothing unreviewed reaches main. - scripts/process_inbox.py reads each submission + .yaml sidecar, requires an explicit license statement (CC BY-SA/BY 4.0, Apache-2.0, MIT) before vendoring, moves the file into corpus//, appends the MANIFEST entry, and flips any `supersedes:` entry. `status: linked` catalogs without vendoring. Anything else is held with a reason. - process-inbox.yaml runs it on PRs touching inbox/, commits the result to same-repo PR branches and comments the report; fork PRs get a dry-run report in the job summary. - "Propose a corpus resource" issue form for URL-only suggestions. - README/CONTRIBUTING/CODEOWNERS updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015bzez2N6vdSh6qXV7Y44P9 --- .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/propose-resource.yml | 42 +++ .github/workflows/process-inbox.yaml | 88 +++++ .github/workflows/validate-corpus.yaml | 1 + CONTRIBUTING.md | 3 +- README.md | 16 +- inbox/README.md | 52 +++ scripts/process_inbox.py | 350 ++++++++++++++++++++ scripts/validate_corpus.py | 8 + 9 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/propose-resource.yml create mode 100644 .github/workflows/process-inbox.yaml create mode 100644 inbox/README.md create mode 100644 scripts/process_inbox.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0c0b6fc..bc314df 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,4 @@ # Corpus changes require review from a repo member with write access. # See README.md for the corpus update process. +/inbox/ @GangGreenTemperTatum @mamicidal @rocklambros @sclintonowasp /corpus/ @GangGreenTemperTatum @mamicidal @rocklambros @sclintonowasp diff --git a/.github/ISSUE_TEMPLATE/propose-resource.yml b/.github/ISSUE_TEMPLATE/propose-resource.yml new file mode 100644 index 0000000..80808b6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/propose-resource.yml @@ -0,0 +1,42 @@ +name: Propose a corpus resource +description: Suggest an OWASP GenAI Security Project document for the Advisor corpus, without opening a PR. +title: "Resource: " +labels: ["corpus-proposal"] +body: + - type: markdown + attributes: + value: | + A maintainer will check the license and add it through `inbox/`. + If you'd rather do it yourself, see `inbox/README.md`. + - type: input + id: title + attributes: + label: Title + validations: + required: true + - type: input + id: url + attributes: + label: Source URL + description: genai.owasp.org resource page, direct PDF link, or upstream repo path. + validations: + required: true + - type: input + id: initiative + attributes: + label: Initiative + description: e.g. llm-top10, agentic-top10, data-security, mcp-security, red-teaming, governance, incident-response, or a new one. + - type: input + id: supersedes + attributes: + label: Replaces an existing resource? + description: The MANIFEST.yaml id it supersedes, if any. + - type: textarea + id: license + attributes: + label: License + description: Where does the document state its license? (e.g. "CC BY-SA 4.0 on page 2") + - type: textarea + id: notes + attributes: + label: What is it, and when should the Advisor use it? diff --git a/.github/workflows/process-inbox.yaml b/.github/workflows/process-inbox.yaml new file mode 100644 index 0000000..fe05d60 --- /dev/null +++ b/.github/workflows/process-inbox.yaml @@ -0,0 +1,88 @@ +name: PR - Process inbox + +# Turns submissions in inbox/ into corpus/ files + MANIFEST.yaml entries on the +# PR branch, so reviewers see exactly what will merge. See inbox/README.md. + +on: + pull_request: + paths: + - "inbox/**" + +permissions: + contents: write + pull-requests: write + +concurrency: + group: process-inbox-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + process: + runs-on: ubuntu-latest + env: + # Same-repo PRs get the processed files committed back. Fork PRs can't + # be written to, so they get a dry run and the report in the job summary. + SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + # Commits pushed with the default GITHUB_TOKEN don't re-trigger + # workflows. Set CORPUS_BOT_TOKEN (a GitHub App or fine-grained + # token with contents:write) so validate-corpus re-runs on the + # processed commit. + token: ${{ secrets.CORPUS_BOT_TOKEN || github.token }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: pip install -r scripts/requirements.txt + + - name: Process inbox + id: process + run: | + args="--report inbox-report.md" + if [ "$SAME_REPO" != "true" ]; then args="$args --dry-run"; fi + set +e + python scripts/process_inbox.py $args + echo "exit=$?" >> "$GITHUB_OUTPUT" + cat inbox-report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Commit processed submissions + if: env.SAME_REPO == 'true' + run: | + git config user.name "corpus-inbox-bot" + git config user.email "actions@users.noreply.github.com" + git add -A corpus/ inbox/ + if git diff --cached --quiet; then + echo "Nothing to commit." + else + git commit -m "Process inbox submissions into corpus/" + git push + fi + + - name: Comment report on PR + if: env.SAME_REPO == 'true' + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + { + cat inbox-report.md + echo + echo "_Reviewers: confirm the license and the manifest entry before merging._" + } > comment.md + gh api "repos/${{ github.repository }}/issues/$PR/comments" -F body=@comment.md > /dev/null + + - name: Fail if anything was held + if: steps.process.outputs.exit != '0' + run: | + echo "::error::Some inbox submissions were held -- see the report in the job summary / PR comment." + exit 1 + + - name: Validate corpus + if: env.SAME_REPO == 'true' + run: python scripts/validate_corpus.py --repo-root . diff --git a/.github/workflows/validate-corpus.yaml b/.github/workflows/validate-corpus.yaml index 876144d..3be6df5 100644 --- a/.github/workflows/validate-corpus.yaml +++ b/.github/workflows/validate-corpus.yaml @@ -4,6 +4,7 @@ on: pull_request: paths: - "corpus/**" + - "inbox/**" - "scripts/validate_corpus.py" jobs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 086088f..fca2de7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,8 @@ ## Adding a corpus resource See "Adding or updating a resource" in [README.md](./README.md). In short: drop the -file(s) in `corpus//`, add a `corpus/MANIFEST.yaml` entry, open a PR -- +file and a `.yaml` sidecar in `inbox/` and open a PR (see [inbox/README.md](inbox/README.md)), +or do it by hand: file(s) in `corpus//`, a `corpus/MANIFEST.yaml` entry, a PR -- CI validates the manifest, and PDF text extraction runs automatically after merge. ## Changing the tooling diff --git a/README.md b/README.md index c6dd8a9..7ed7dd4 100644 --- a/README.md +++ b/README.md @@ -22,17 +22,31 @@ corpus/ MANIFEST.yaml # catalog of every resource -- the single source of truth /... # vendored content, grouped by initiative _extracted/ # auto-generated PDF text (search index only, never edit) +inbox/ # submission drop zone -- always empty on main scripts/ validate_corpus.py # MANIFEST checks, run by CI on every PR + process_inbox.py # inbox/ -> corpus/ + MANIFEST entry, with license check extract_pdf_text.py # regenerates corpus/_extracted/ from vendored PDFs requirements.txt .github/workflows/ validate-corpus.yaml # PR check + process-inbox.yaml # PR: processes inbox/ submissions onto the PR branch extract-pdf-text.yaml # runs on push to main, commits refreshed _extracted/ ``` ## Adding or updating a resource +**Easiest: use the inbox.** Put the file and a small `.yaml` sidecar in +`inbox/` and open a PR. The `process-inbox` workflow checks the document for an +explicit license statement, moves it into `corpus/`, writes the `MANIFEST.yaml` +entry (and marks anything it supersedes), and comments on the PR with what it +did. See [inbox/README.md](inbox/README.md). Don't have the file, or don't want +to open a PR? Use the **Propose a corpus resource** issue form instead. + +The manual steps below are what the inbox automates, and still apply for +anything it can't handle (e.g. turning an existing `linked` entry into a +vendored one). + 1. **Confirm the license.** Only vendor (copy in) a document whose license is explicitly stated in the source -- most OWASP GenAI Security Project output is CC BY-SA 4.0. If you can't confirm it, catalog it as a `linked` entry instead @@ -70,7 +84,7 @@ scripts/ python scripts/validate_corpus.py ``` -6. **Open a PR.** `validate-corpus.yaml` checks that every entry has its required +6. **Open a PR.** `validate-corpus.yaml` checks that `inbox/` is empty and that every entry has its required fields, ids are unique, `status` is valid, and `path` exists (or is `null` for `linked`). CODEOWNERS review is required for `corpus/`. 7. **After merge,** `extract-pdf-text.yaml` extracts text from every `current` or diff --git a/inbox/README.md b/inbox/README.md new file mode 100644 index 0000000..f26c65c --- /dev/null +++ b/inbox/README.md @@ -0,0 +1,52 @@ +# inbox/ -- submit new corpus resources here + +Drop a new resource (PDF, markdown, JSON, or a folder of them) in this folder in +a pull request. The `process-inbox` workflow checks it and moves it into +`corpus/` with a `MANIFEST.yaml` entry, committed back to your PR branch. A +CODEOWNER then reviews and merges. + +**This folder is always empty on `main`.** CI fails while anything other than +this README is here, so nothing unreviewed is ever published. + +## How to submit + +1. Add the file, e.g. `inbox/My-Guide-v1.0.pdf`. +2. Add a sidecar with the same name and a `.yaml` extension, + e.g. `inbox/My-Guide-v1.0.yaml`: + + ```yaml + title: "My Guide" # defaults to the PDF's metadata title + initiative: mcp-security # required: corpus/ subfolder (new ones OK) + source_url: https://genai.owasp.org/resource/... # or source_repo (+ source_path) + version: "1.0" # defaults to the version in the file name + published: "2026-10-01" # optional + folder: companions # optional subfolder under the initiative + supersedes: some-existing-id # optional: marks that entry superseded + status: current # current (default) | draft | linked + id: my-guide-1.0 # optional: defaults to the file name + notes: > + What it is and when the Advisor should use it. + ``` + +3. Open a PR. The workflow comments with what it did, or why a submission was + held. + +To work locally instead, run `python scripts/process_inbox.py` (add +`--dry-run` to preview) and commit the result. + +## License rules + +The processor only moves a file into `corpus/` when the document itself states +one of these licenses: CC BY-SA 4.0, CC BY 4.0, Apache-2.0 or MIT. A vague +"released under creative commons licenses" doesn't count. + +- **No license statement found:** the file is held. Either confirm the license + with the initiative lead and add a statement, or set `status: linked`. +- **`status: linked`:** catalogs the resource with `path: null` and + `license: UNCONFIRMED`, and deletes the file. The Advisor links to + `source_url` and doesn't quote from it. A sidecar on its own (no file) works + for this too. +- **Any other license (e.g. NC/ND variants):** held for a maintainer decision. + +Reviewers still confirm the license before merging. The detector helps +reviewers; it doesn't replace them. diff --git a/scripts/process_inbox.py b/scripts/process_inbox.py new file mode 100644 index 0000000..c190f39 --- /dev/null +++ b/scripts/process_inbox.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Process submissions in inbox/ into corpus/ + corpus/MANIFEST.yaml. + +Each submission is a file (or folder) at the top of inbox/, plus an optional +sidecar `.yaml` describing it (see inbox/README.md). For each one: + + 1. Look for an explicit license statement in the text (e.g. "CC BY-SA 4.0"). + A vague mention like "released under creative commons licenses" does NOT + count -- see the governance checklist / exploit generation entries in the + manifest for why. + 2. If a known redistributable license is found (or the sidecar sets + `license:` and the detector doesn't contradict it), move the file into + corpus//[/] and append a MANIFEST entry. + 3. If the sidecar says `status: linked`, append a linked entry (path: null) + and delete the file -- it must never be vendored. + 4. Otherwise leave the file in inbox/ and report why. validate_corpus.py + fails while inbox/ is non-empty, so the PR can't merge until resolved. + 5. If the sidecar names `supersedes: `, flip that entry to superseded. + +Writes a markdown report (--report) for the PR comment / job summary. Exits 1 +if anything was left unprocessed. Run from the repo root: + + python scripts/process_inbox.py # process + python scripts/process_inbox.py --dry-run # report only, change nothing +""" +import argparse +import logging +import os +import re +import shutil +import sys +from datetime import datetime, timezone + +import yaml +from pypdf import PdfReader + +INBOX = "inbox" +INBOX_KEEP = {"README.md", ".gitkeep"} +MANIFEST = os.path.join("corpus", "MANIFEST.yaml") + +# Licenses that may be vendored without further review. Anything else that's +# detected is reported, and the submission is held for a maintainer. +VENDORABLE = {"CC-BY-SA-4.0", "CC-BY-4.0", "Apache-2.0", "MIT"} + +LICENSE_PATTERNS = [ + # Order matters: most specific first. + ("CC-BY-NC-SA-4.0", r"cc[\s-]*by[\s-]*nc[\s-]*sa[\s-]*4|attribution[\s-]+noncommercial[\s-]+sharealike\s+4"), + ("CC-BY-NC-4.0", r"cc[\s-]*by[\s-]*nc[\s-]*4|attribution[\s-]+noncommercial\s+4"), + ("CC-BY-ND-4.0", r"cc[\s-]*by[\s-]*nd[\s-]*4|attribution[\s-]+noderivatives\s+4"), + ("CC-BY-SA-4.0", r"cc[\s-]*by[\s-]*sa[\s-]*4|attribution[\s-]+sharealike\s+4"), + ("CC-BY-4.0", r"cc[\s-]*by[\s-]*4\.0|creative\s+commons\s+attribution\s+4"), + ("Apache-2.0", r"apache\s+license,?\s+version\s+2|apache[\s-]+2\.0\s+licen[sc]e"), + ("MIT", r"\bmit\s+licen[sc]e\b"), +] + +FORMAT_BY_EXT = {".pdf": "pdf", ".md": "markdown", ".json": "json"} +TEXT_EXTS = {".md", ".txt", ".json", ".yaml", ".yml"} + +logging.getLogger("pypdf").setLevel(logging.ERROR) + + +def today() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d") + + +def slug(s: str) -> str: + return re.sub(r"[^a-z0-9.]+", "-", s.lower()).strip("-") + + +def text_pages(path: str): + """Yield (location, text) for a file, page by page for PDFs.""" + ext = os.path.splitext(path)[1].lower() + if ext == ".pdf": + for i, page in enumerate(PdfReader(path).pages, start=1): + yield f"page {i}", page.extract_text() or "" + elif ext in TEXT_EXTS: + with open(path, "r", encoding="utf-8", errors="replace") as f: + yield "text", f.read() + + +def files_of(path: str): + if os.path.isdir(path): + for root, _, names in os.walk(path): + for n in sorted(names): + yield os.path.join(root, n) + else: + yield path + + +def detect_licenses(path: str): + """Return {license_id: first 'file @ location'} for explicit statements.""" + found = {} + for f in files_of(path): + for loc, text in text_pages(f): + flat = re.sub(r"\s+", " ", text) + for lic, pat in LICENSE_PATTERNS: + if lic not in found and re.search(pat, flat, re.I): + found[lic] = f"{os.path.basename(f)} @ {loc}" + # "CC BY-SA 4" also matches the CC-BY-4.0 attribution pattern in some + # wordings; a ShareAlike/NC/ND variant always wins over plain CC BY. + if "CC-BY-4.0" in found and any(k.startswith("CC-BY-") and k != "CC-BY-4.0" for k in found): + del found["CC-BY-4.0"] + return found + + +def pdf_title(path: str): + if not path.lower().endswith(".pdf"): + return None + meta = PdfReader(path).metadata + title = (meta.title if meta else None) or "" + return title.strip() or None + + +def guess_version(name: str): + m = re.search(r"v?(\d+(?:\.\d+)+)", name) + return m.group(1) if m else None + + +def guess_format(path: str) -> str: + if os.path.isdir(path): + fmts = {FORMAT_BY_EXT.get(os.path.splitext(f)[1].lower(), "other") for f in files_of(path)} + return fmts.pop() if len(fmts) == 1 and "other" not in fmts else "mixed" + return FORMAT_BY_EXT.get(os.path.splitext(path)[1].lower(), "mixed") + + +def yaml_scalar(v) -> str: + if v is None: + return "null" + return '"' + str(v).replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def render_entry(e: dict) -> str: + """Render a manifest entry in the hand-written style of MANIFEST.yaml.""" + lines = [f" - id: {e['id']}"] + order = ["title", "initiative", "version", "status", "format", "path", "license", + "source_repo", "source_path", "source_url", "vendored_commit", + "vendored_date", "published"] + plain = {"initiative", "status", "format", "license", "source_repo", + "source_path", "source_url", "vendored_commit"} + for k in order: + if k not in e: + continue + v = e[k] + if k == "path": + lines.append(f" path: {v if v else 'null'}") + elif k in plain and v is not None: + lines.append(f" {k}: {v}") + else: + lines.append(f" {k}: {yaml_scalar(v)}") + notes = (e.get("notes") or "").strip() + if notes: + lines.append(" notes: >") + words, line = notes.split(), " " + for w in words: + if len(line) + 1 + len(w) > 78: + lines.append(line) + line = " " + line += " " + w + lines.append(line) + return "\n".join(lines) + "\n" + + +def supersede(manifest_text: str, rid: str): + """Flip `status:` to superseded inside entry `rid`; text-level to keep comments.""" + m = re.search(rf"^ - id: {re.escape(rid)}\s*\n(.*?)(?=^ - id: |\Z)", manifest_text, re.M | re.S) + if not m: + return manifest_text, False + block = m.group(0) + new_block, n = re.subn(r"^( status: )\S+", r"\1superseded", block, count=1, flags=re.M) + return manifest_text[:m.start()] + new_block + manifest_text[m.end():], n == 1 + + +def submissions(repo_root: str): + inbox = os.path.join(repo_root, INBOX) + if not os.path.isdir(inbox): + return [] + names = sorted(os.listdir(inbox)) + out = [] + for n in names: + if n in INBOX_KEEP or n.startswith("."): + continue + if n.endswith((".yaml", ".yml")): + stem = n.rsplit(".", 1)[0] + if any(o != n and (o == stem or o.rsplit(".", 1)[0] == stem) for o in names): + continue # sidecar of another submission + # A sidecar on its own describes a linked-only resource (no file). + out.append(n) + return out + + +def load_sidecar(inbox: str, name: str): + stem = name if os.path.isdir(os.path.join(inbox, name)) else name.rsplit(".", 1)[0] + for cand in (f"{name}.yaml", f"{stem}.yaml", f"{stem}.yml"): + p = os.path.join(inbox, cand) + if os.path.isfile(p) and cand != name: + with open(p, "r", encoding="utf-8") as f: + return p, yaml.safe_load(f) or {} + return None, {} + + +def process(repo_root: str, dry_run: bool): + inbox = os.path.join(repo_root, INBOX) + manifest_path = os.path.join(repo_root, MANIFEST) + with open(manifest_path, "r", encoding="utf-8") as f: + manifest_text = f.read() + existing = {r["id"]: r for r in (yaml.safe_load(manifest_text) or {}).get("resources", [])} + + report, held, done = [], 0, 0 + for name in submissions(repo_root): + src = os.path.join(inbox, name) + is_sidecar_only = name.endswith((".yaml", ".yml")) + if is_sidecar_only: + sidecar_path, meta = src, yaml.safe_load(open(src, encoding="utf-8")) or {} + else: + sidecar_path, meta = load_sidecar(inbox, name) + + problems = [] + status = meta.get("status", "linked" if is_sidecar_only else "current") + initiative = meta.get("initiative") + title = meta.get("title") or (None if is_sidecar_only else pdf_title(src)) + version = str(meta.get("version") or guess_version(name) or "") + rid = meta.get("id") or slug(os.path.splitext(name)[0]) + source_url, source_repo = meta.get("source_url"), meta.get("source_repo") + + if not initiative: + problems.append("sidecar needs `initiative:` (the corpus/ subfolder, e.g. `mcp-security`)") + elif not re.fullmatch(r"[a-z0-9][a-z0-9-]*", str(initiative)): + problems.append(f"`initiative: {initiative}` must be a lowercase folder name") + if not title: + problems.append("sidecar needs `title:` (no title in PDF metadata)") + if not version: + problems.append("sidecar needs `version:` (none found in the file name)") + if not source_url and not source_repo: + problems.append("sidecar needs `source_url:` or `source_repo:`") + if rid in existing: + problems.append(f"id `{rid}` already exists in MANIFEST.yaml -- set a different `id:`") + sup = meta.get("supersedes") + if sup and sup not in existing: + problems.append(f"`supersedes: {sup}` is not an id in MANIFEST.yaml") + if status not in ("current", "draft", "linked"): + problems.append(f"`status: {status}` must be current, draft or linked") + + detected = {} if is_sidecar_only else detect_licenses(src) + claimed = meta.get("license") + license_ = None + lic_note = ", ".join(f"{k} ({v})" for k, v in detected.items()) or "none found" + if status == "linked": + license_ = claimed or "UNCONFIRMED" + elif claimed and detected and claimed not in detected: + problems.append(f"sidecar says `license: {claimed}` but the document states {lic_note}") + elif len(detected) > 1 and not claimed: + problems.append(f"multiple licenses stated ({lic_note}) -- set `license:` in the sidecar") + else: + license_ = claimed or (next(iter(detected)) if detected else None) + if not license_: + problems.append("no explicit license statement found -- confirm it with the " + "initiative lead, or set `status: linked` to catalog it without vendoring") + elif license_ not in VENDORABLE: + problems.append(f"`{license_}` isn't on the auto-vendor list ({', '.join(sorted(VENDORABLE))}) " + "-- needs a maintainer decision, or set `status: linked`") + elif not detected: + problems.append(f"`license: {license_}` is claimed but no license statement was found in " + "the document -- vendoring needs an explicit statement; use `status: linked`") + + folder = meta.get("folder") + dest_rel = None + if status != "linked" and initiative: + dest_rel = "/".join(p for p in ("corpus", initiative, folder, name) if p) + if os.path.isdir(src): + dest_rel += "/" + if os.path.exists(os.path.join(repo_root, dest_rel)): + problems.append(f"`{dest_rel}` already exists") + + entry = { + "id": rid, "title": title, "initiative": initiative, "version": version, + "status": status, "format": "pdf" if is_sidecar_only and status == "linked" else + (meta.get("format") or (None if is_sidecar_only else guess_format(src))), + "path": dest_rel, "license": license_, + } + for k in ("source_repo", "source_path", "source_url", "vendored_commit"): + if meta.get(k): + entry[k] = meta[k] + if status != "linked": + entry["vendored_date"] = today() + entry["published"] = str(meta["published"]) if meta.get("published") else None + notes = (meta.get("notes") or "").strip() + if status == "linked" and license_ == "UNCONFIRMED": + notes = (notes + " " if notes else "") + ("License not confirmed from the document text. " + "Do NOT vendor until confirmed with the initiative lead.") + if sup: + notes = (notes + " " if notes else "") + f"Supersedes {sup}." + entry["notes"] = notes + + report.append(f"### `{name}`\n") + report.append(f"- License detected: {lic_note}") + if problems: + held += 1 + report.append("- **Held in inbox/** -- fix and push again:") + report += [f" - {p}" for p in problems] + report.append("") + continue + + report.append(f"- Action: {'catalog as linked (file removed)' if status == 'linked' else 'move to `' + dest_rel + '`'}") + if sup: + report.append(f"- Marks `{sup}` as superseded") + report.append("\n```yaml\n" + render_entry(entry) + "```\n") + done += 1 + if dry_run: + continue + + if status != "linked": + dest_abs = os.path.join(repo_root, dest_rel) + os.makedirs(os.path.dirname(dest_abs.rstrip("/")), exist_ok=True) + shutil.move(src, dest_abs.rstrip("/")) + elif not is_sidecar_only: + (shutil.rmtree if os.path.isdir(src) else os.remove)(src) + if sidecar_path and os.path.exists(sidecar_path): + os.remove(sidecar_path) + if sup: + manifest_text, _ = supersede(manifest_text, sup) + manifest_text = manifest_text.rstrip("\n") + "\n\n" + render_entry(entry) + existing[rid] = entry + + if not dry_run and done: + with open(manifest_path, "w", encoding="utf-8") as f: + f.write(manifest_text) + + head = f"## Inbox processing{' (dry run)' if dry_run else ''}\n\n{done} processed, {held} held.\n\n" + if not report: + head = "## Inbox processing\n\nNothing in inbox/.\n" + return head + "\n".join(report), held + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--repo-root", default=".") + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--report", help="also write the markdown report to this file") + args = ap.parse_args() + + report, held = process(os.path.abspath(args.repo_root), args.dry_run) + print(report) + if args.report: + with open(args.report, "w", encoding="utf-8") as f: + f.write(report) + sys.exit(1 if held else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_corpus.py b/scripts/validate_corpus.py index 7c937e3..61d0fc5 100644 --- a/scripts/validate_corpus.py +++ b/scripts/validate_corpus.py @@ -71,6 +71,14 @@ def main() -> None: if not str(r[key]).strip(): die(f"resource '{rid}': {key} is empty") + # inbox/ is a staging area only: nothing may merge while it holds a + # submission, or unreviewed (possibly unlicensed) files would land on main. + inbox = os.path.join(repo_root, "inbox") + if os.path.isdir(inbox): + pending = sorted(n for n in os.listdir(inbox) if n not in ("README.md", ".gitkeep")) + if pending: + die(f"inbox/ must be empty before merge -- run scripts/process_inbox.py; pending: {pending}") + print(f"[validate_corpus] OK: {len(resources)} resources validated from {manifest_path}")